From cb3253d23e03732880796b4c5566a9daa9fc5aad Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 24 Dec 2019 15:36:55 +0800 Subject: [PATCH 001/523] build sink sql --- mysql/mysql-side/mysql-all-side/pom.xml | 1 + mysql/mysql-side/mysql-async-side/pom.xml | 1 + mysql/mysql-sink/pom.xml | 1 + .../flink/sql/sink/mysql/MysqlSink.java | 40 +++++---- oracle/oracle-side/oracle-all-side/pom.xml | 1 + oracle/oracle-side/oracle-async-side/pom.xml | 1 + oracle/oracle-sink/pom.xml | 1 + .../flink/sql/sink/oracle/OracleSink.java | 81 ++++++++++++------- .../postgresql-all-side/pom.xml | 1 + .../postgresql-async-side/pom.xml | 1 + postgresql/postgresql-sink/pom.xml | 1 + .../dtstack/flink/sql/sink/rdb/RdbSink.java | 3 + 12 files changed, 87 insertions(+), 46 deletions(-) diff --git a/mysql/mysql-side/mysql-all-side/pom.xml b/mysql/mysql-side/mysql-all-side/pom.xml index cc3df111d..d28791434 100644 --- a/mysql/mysql-side/mysql-all-side/pom.xml +++ b/mysql/mysql-side/mysql-all-side/pom.xml @@ -40,6 +40,7 @@ shade + false diff --git a/mysql/mysql-side/mysql-async-side/pom.xml b/mysql/mysql-side/mysql-async-side/pom.xml index 55110ad3b..2e648fc3b 100644 --- a/mysql/mysql-side/mysql-async-side/pom.xml +++ b/mysql/mysql-side/mysql-async-side/pom.xml @@ -41,6 +41,7 @@ shade + false diff --git a/mysql/mysql-sink/pom.xml b/mysql/mysql-sink/pom.xml index 139a3ea53..feecddb83 100644 --- a/mysql/mysql-sink/pom.xml +++ b/mysql/mysql-sink/pom.xml @@ -39,6 +39,7 @@ shade + false diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java index 9ba6736be..bb780d876 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java @@ -25,8 +25,11 @@ import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; import com.dtstack.flink.sql.util.DtStringUtil; +import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; /** * Date: 2017/2/27 @@ -49,7 +52,8 @@ public RetractJDBCOutputFormat getOutputFormat() { @Override public void buildSql(String scheam, String tableName, List fields) { - buildInsertSql(tableName, fields); + buildReplaceUpsertStatement(tableName, fields); + buildDuplicateUpsertStatement(tableName, fields); } @Override @@ -57,28 +61,34 @@ public String buildUpdateSql(String schema, String tableName, List field return null; } - private void buildInsertSql(String tableName, List fields) { - String sqlTmp = "replace into " + tableName + " (${fields}) values (${placeholder})"; - String fieldsStr = ""; - String placeholder = ""; - - for (String fieldName : fields) { - fieldsStr += ",`" + fieldName + "`"; - placeholder += ",?"; - } + private void buildReplaceUpsertStatement(String tableName, List fields) { + this.sql = getUpsertIntoStatement("REPLACE ", tableName, fields); + } - fieldsStr = fieldsStr.replaceFirst(",", ""); - placeholder = placeholder.replaceFirst(",", ""); + public void buildDuplicateUpsertStatement(String tableName, List fields) { + String updateClause = fields.stream().map(f -> quoteIdentifier(f) + "=IFNULL(VALUES(" + quoteIdentifier(f) + "),"+ quoteIdentifier(f) + ")") + .collect(Collectors.joining(", ")); - sqlTmp = sqlTmp.replace("${fields}", fieldsStr).replace("${placeholder}", placeholder); - this.sql = sqlTmp; + this.sql = getUpsertIntoStatement("INSERT",tableName, fields) + + " ON DUPLICATE KEY UPDATE " + updateClause; } + public String getUpsertIntoStatement(String operator, String tableName, List fields) { + String columns = fields.stream() + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")); + + String placeholders = fields.stream() + .map(f -> "?") + .collect(Collectors.joining(", ")); + + return operator + " INTO " + quoteIdentifier(tableName) + + "(" + columns + ")" + " VALUES (" + placeholders + ")"; + } @Override public String getDriverName() { return MYSQL_DRIVER; } - } diff --git a/oracle/oracle-side/oracle-all-side/pom.xml b/oracle/oracle-side/oracle-all-side/pom.xml index 0685549c4..fdfb1242f 100644 --- a/oracle/oracle-side/oracle-all-side/pom.xml +++ b/oracle/oracle-side/oracle-all-side/pom.xml @@ -40,6 +40,7 @@ shade + false diff --git a/oracle/oracle-side/oracle-async-side/pom.xml b/oracle/oracle-side/oracle-async-side/pom.xml index ba227fd1e..f1746b6dc 100644 --- a/oracle/oracle-side/oracle-async-side/pom.xml +++ b/oracle/oracle-side/oracle-async-side/pom.xml @@ -39,6 +39,7 @@ shade + false diff --git a/oracle/oracle-sink/pom.xml b/oracle/oracle-sink/pom.xml index 7bd52ec72..e581b7dfe 100644 --- a/oracle/oracle-sink/pom.xml +++ b/oracle/oracle-sink/pom.xml @@ -39,6 +39,7 @@ shade + false diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index c30dc9c60..e744ed97b 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -29,6 +29,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Reason: @@ -56,23 +57,15 @@ public void buildSql(String scheam, String tableName, List fields) { } private void buildInsertSql(String scheam, String tableName, List fields) { - - tableName = DtStringUtil.getTableFullPath(scheam,tableName); - - String sqlTmp = "insert into " + tableName + " (${fields}) values (${placeholder})"; - - List adaptFields = Lists.newArrayList(); - fields.forEach(field -> adaptFields.add(DtStringUtil.addQuoteForStr(field))); - - String fieldsStr = StringUtils.join(adaptFields, ","); - String placeholder = ""; - - for (String fieldName : fields) { - placeholder += ",?"; - } - placeholder = placeholder.replaceFirst(",", ""); - sqlTmp = sqlTmp.replace("${fields}", fieldsStr).replace("${placeholder}", placeholder); - this.sql = sqlTmp; + tableName = DtStringUtil.getTableFullPath(scheam, tableName); + String columns = fields.stream() + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")); + + String placeholders = fields.stream() + .map(f -> "?") + .collect(Collectors.joining(", ")); + this.sql = "INSERT INTO " + tableName + "(" + columns + ")" + " VALUES (" + placeholders + ")"; } /** @@ -94,7 +87,8 @@ public String buildUpdateSql(String scheam, String tableName, List field + updateKeySql(realIndexes) + ") "); - String updateSql = getUpdateSql(fieldNames, fullField, "T1", "T2", keyColList(realIndexes)); + String updateSql1 = buildUpdateSqlForAllValue(fieldNames, fullField, "T1", "T2", keyColList(realIndexes)); + String updateSql = buildUpdateSqlForNotnullValue(fieldNames, fullField, "T1", "T2", keyColList(realIndexes)); if (StringUtils.isNotEmpty(updateSql)) { sb.append(" WHEN MATCHED THEN UPDATE SET "); @@ -149,24 +143,47 @@ protected List keyColList(Map> realIndexes) { * @param indexCols index column * @return */ - public String getUpdateSql(List updateColumn, List fullColumn, String leftTable, String rightTable, List indexCols) { + public String buildUpdateSqlForAllValue(List updateColumn, List fullColumn, String leftTable, String rightTable, List indexCols) { String prefixLeft = StringUtils.isBlank(leftTable) ? "" : DtStringUtil.addQuoteForStr(leftTable) + "."; String prefixRight = StringUtils.isBlank(rightTable) ? "" : DtStringUtil.addQuoteForStr(rightTable) + "."; - List list = new ArrayList<>(); - for (String col : fullColumn) { - // filter index column - if (indexCols == null || indexCols.size() == 0 || containsIgnoreCase(indexCols,col)) { - continue; - } - if (containsIgnoreCase(updateColumn,col)) { - list.add(prefixLeft + DtStringUtil.addQuoteForStr(col) + "=" + prefixRight + DtStringUtil.addQuoteForStr(col)); + + String sql = fullColumn.stream().filter(col -> { + return !(indexCols == null || indexCols.size() == 0 || containsIgnoreCase(indexCols, col)); + }).map(col -> { + String leftCol = prefixLeft + DtStringUtil.addQuoteForStr(col); + String rightCol = prefixRight + DtStringUtil.addQuoteForStr(col); + + if (containsIgnoreCase(updateColumn, col)) { + return (leftCol + "=" + rightCol); } else { - list.add(prefixLeft + DtStringUtil.addQuoteForStr(col) + "=null"); + return (leftCol + "=null"); } - } - return StringUtils.join(list, ","); + }).collect(Collectors.joining(",")); + + return sql; } + public String buildUpdateSqlForNotnullValue(List updateColumn, List fullColumn, String leftTable, String rightTable, List indexCols) { + String prefixLeft = StringUtils.isBlank(leftTable) ? "" : DtStringUtil.addQuoteForStr(leftTable) + "."; + String prefixRight = StringUtils.isBlank(rightTable) ? "" : DtStringUtil.addQuoteForStr(rightTable) + "."; + + String sql = fullColumn.stream().filter(col -> { + return !(indexCols == null || indexCols.size() == 0 || containsIgnoreCase(indexCols, col)); + }).map(col -> { + String leftCol = prefixLeft + DtStringUtil.addQuoteForStr(col); + String rightCol = prefixRight + DtStringUtil.addQuoteForStr(col); + + if (containsIgnoreCase(updateColumn, col)) { + return leftCol + "= nvl(" + rightCol + "," + leftCol + ")"; + } + return ""; + }).collect(Collectors.joining(",")); + + return sql; + } + + + /** * build connect sql by index column, such as T1."A"=T2."A" @@ -212,6 +229,8 @@ public boolean containsIgnoreCase(List l, String s) { return false; } - + public String quoteIdentifier(String identifier) { + return "\"" + identifier + "\""; + } } diff --git a/postgresql/postgresql-side/postgresql-all-side/pom.xml b/postgresql/postgresql-side/postgresql-all-side/pom.xml index d076d9300..8ea3b4128 100644 --- a/postgresql/postgresql-side/postgresql-all-side/pom.xml +++ b/postgresql/postgresql-side/postgresql-all-side/pom.xml @@ -40,6 +40,7 @@ shade + false diff --git a/postgresql/postgresql-side/postgresql-async-side/pom.xml b/postgresql/postgresql-side/postgresql-async-side/pom.xml index 13296e7f3..fd48fd253 100644 --- a/postgresql/postgresql-side/postgresql-async-side/pom.xml +++ b/postgresql/postgresql-side/postgresql-async-side/pom.xml @@ -41,6 +41,7 @@ shade + false diff --git a/postgresql/postgresql-sink/pom.xml b/postgresql/postgresql-sink/pom.xml index d65368358..d1f983f7d 100644 --- a/postgresql/postgresql-sink/pom.xml +++ b/postgresql/postgresql-sink/pom.xml @@ -39,6 +39,7 @@ shade + false diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java index babff68f9..96cce9516 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java @@ -274,4 +274,7 @@ public void setDbType(String dbType) { public abstract RetractJDBCOutputFormat getOutputFormat(); + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } } From 0142a4ff0b8c7c0a7e1fbaf79402996b1e582f39 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 2 Jan 2020 17:27:26 +0800 Subject: [PATCH 002/523] rdb sink add update mode --- .../sql/sink/clickhouse/ClickhouseSink.java | 1 - .../dtstack/flink/sql/enums/EUpdateMode.java | 44 ++ .../dtstack/flink/sql/side/SideSqlExec.java | 3 +- .../flink/sql/sink/MetricOutputFormat.java | 15 +- .../flink/sql/table/AbsTableParser.java | 3 +- .../dtstack/flink/sql/table/TableInfo.java | 6 +- .../com/dtstack/flink/sql/sink/db/DbSink.java | 1 - .../sql/sink/impala/ImpalaOutputFormat.java | 1 - .../flink/sql/sink/impala/ImpalaSink.java | 1 - .../flink/sql/sink/mysql/MySQLDialect.java | 81 +++ .../flink/sql/sink/mysql/MysqlSink.java | 69 +-- .../flink/sql/sink/oracle/OracleSink.java | 2 - .../flink/sql/sink/polardb/PolardbSink.java | 1 - pom.xml | 40 +- .../sql/sink/postgresql/PostgresqlSink.java | 1 - .../sql/sink/rdb/JDBCTypeConvertUtils.java | 201 +++++++ .../dtstack/flink/sql/sink/rdb/RdbSink.java | 188 ++----- .../sql/sink/rdb/dialect/JDBCDialect.java | 140 +++++ .../sink/rdb/format/ExtendOutputFormat.java | 116 ---- .../rdb/format/JDBCUpsertOutputFormat.java | 300 +++++++++++ .../rdb/format/RetractJDBCOutputFormat.java | 497 ------------------ .../sql/sink/rdb/table/RdbSinkParser.java | 4 +- .../sql/sink/rdb/table/RdbTableInfo.java | 38 +- .../sql/sink/rdb/writer/AppendOnlyWriter.java | 131 +++++ .../flink/sql/sink/rdb/writer/JDBCWriter.java | 65 +++ .../sql/sink/rdb/writer/UpsertWriter.java | 332 ++++++++++++ .../sql/sink/redis/table/RedisSinkParser.java | 9 +- .../sql/sink/sqlserver/SqlserverSink.java | 2 - 28 files changed, 1419 insertions(+), 873 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/enums/EUpdateMode.java create mode 100644 mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java create mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java create mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java delete mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/ExtendOutputFormat.java create mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java delete mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java create mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java create mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/JDBCWriter.java create mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java index 66c1f26d5..4c91f6873 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java @@ -22,7 +22,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.RdbSink; -import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; import java.util.List; import java.util.Map; diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EUpdateMode.java b/core/src/main/java/com/dtstack/flink/sql/enums/EUpdateMode.java new file mode 100644 index 000000000..eec1a4c4a --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EUpdateMode.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.enums; + +/** + * restract stream数据处理模式 + * + * Reason: + * Date: 2019/1/2 + * Company: www.dtstack.com + * @author maqi + */ +public enum EUpdateMode { + // 不回撤数据，只下发增量数据 + APPEND(0), + // 先删除回撤数据，然后更新 + UPSERT(1); + + private int type; + + EUpdateMode(int type) { + this.type = type; + } + + public int getType() { + return this.type; + } +} diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 0fddcbc37..8a82df237 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -60,6 +60,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Timestamp; +import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -626,7 +627,7 @@ private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, private List convertPrimaryAlias(SideTableInfo sideTableInfo) { List res = Lists.newArrayList(); - sideTableInfo.getPrimaryKeys().forEach(field -> { + Arrays.stream(sideTableInfo.getPrimaryKeys()).forEach(field -> { res.add(sideTableInfo.getPhysicalFields().getOrDefault(field, field)); }); return res; diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/MetricOutputFormat.java b/core/src/main/java/com/dtstack/flink/sql/sink/MetricOutputFormat.java index f56f531e0..056a19dea 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/MetricOutputFormat.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/MetricOutputFormat.java @@ -24,22 +24,21 @@ import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.Meter; import org.apache.flink.metrics.MeterView; +import org.apache.flink.types.Row; /** * Created by sishu.yss on 2018/11/28. */ -public abstract class MetricOutputFormat extends RichOutputFormat{ +public abstract class MetricOutputFormat extends RichOutputFormat> { - protected transient Counter outRecords; + public transient Counter outRecords; + public transient Counter outDirtyRecords; + public transient Meter outRecordsRate; - protected transient Counter outDirtyRecords; - - protected transient Meter outRecordsRate; - - public void initMetric() { + public void initMetric() { outRecords = getRuntimeContext().getMetricGroup().counter(MetricConstant.DT_NUM_RECORDS_OUT); outDirtyRecords = getRuntimeContext().getMetricGroup().counter(MetricConstant.DT_NUM_DIRTY_RECORDS_OUT); outRecordsRate = getRuntimeContext().getMetricGroup().meter(MetricConstant.DT_NUM_RECORDS_OUT_RATE, new MeterView(outRecords, 20)); - } + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java index 5af06d087..64e730a7d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java @@ -115,8 +115,7 @@ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ public void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ String primaryFields = matcher.group(1).trim(); String[] splitArry = primaryFields.split(","); - List primaryKes = Lists.newArrayList(splitArry); - tableInfo.setPrimaryKeys(primaryKes); + tableInfo.setPrimaryKeys(splitArry); } /** diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java index 59e4fdd39..2559da2ec 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java @@ -59,7 +59,7 @@ public abstract class TableInfo implements Serializable { private final List fieldExtraInfoList = Lists.newArrayList(); - private List primaryKeys; + private String[] primaryKeys; private Integer parallelism = -1; @@ -85,11 +85,11 @@ public Class[] getFieldClasses() { return fieldClasses; } - public List getPrimaryKeys() { + public String[] getPrimaryKeys() { return primaryKeys; } - public void setPrimaryKeys(List primaryKeys) { + public void setPrimaryKeys(String[] primaryKeys) { this.primaryKeys = primaryKeys; } diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java index ccdb00eb9..028a156dc 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java @@ -1,7 +1,6 @@ package com.dtstack.flink.sql.sink.db; import com.dtstack.flink.sql.sink.rdb.RdbSink; -import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; import java.util.List; import java.util.Map; diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index 0916a6822..fae2ef257 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -20,7 +20,6 @@ package com.dtstack.flink.sql.sink.impala; import com.dtstack.flink.sql.sink.impala.table.ImpalaTableInfo; -import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index ac90df616..0ce3ad29a 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -21,7 +21,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.impala.table.ImpalaTableInfo; import com.dtstack.flink.sql.sink.rdb.RdbSink; -import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.commons.lang.StringUtils; import org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction; diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java new file mode 100644 index 000000000..f7fe449b1 --- /dev/null +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.mysql; + +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; + +import java.util.Arrays; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Date: 2019/12/31 + * Company: www.dtstack.com + * @author maqi + */ +public class MySQLDialect implements JDBCDialect { + private static final long serialVersionUID = 1L; + + @Override + public boolean canHandle(String url) { + return url.startsWith("jdbc:mysql:"); + } + + @Override + public Optional defaultDriverName() { + return Optional.of("com.mysql.jdbc.Driver"); + } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } + + /** + * 根据ALLReplace参数，选择使用replace语句还是ON DUPLICATE KEY UPDATE 语句 + * @param tableName + * @param fieldNames + * @param uniqueKeyFields + * @param allReplace + * @return + */ + @Override + public Optional getUpsertStatement(String tableName, String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + return allReplace ? buildReplaceIntoStatement(tableName, fieldNames) : buildDuplicateUpsertStatement(tableName, fieldNames); + } + + public Optional buildDuplicateUpsertStatement(String tableName, String[] fieldNames) { + String updateClause = Arrays.stream(fieldNames).map(f -> quoteIdentifier(f) + "=IFNULL(VALUES(" + quoteIdentifier(f) + ")," + quoteIdentifier(f) + ")") + .collect(Collectors.joining(", ")); + return Optional.of(getInsertIntoStatement(tableName, fieldNames) + + " ON DUPLICATE KEY UPDATE " + updateClause + ); + } + + public Optional buildReplaceIntoStatement(String tableName, String[] fieldNames) { + String columns = Arrays.stream(fieldNames) + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")); + String placeholders = Arrays.stream(fieldNames) + .map(f -> "?") + .collect(Collectors.joining(", ")); + return Optional.of("REPLACE INTO " + quoteIdentifier(tableName) + + "(" + columns + ")" + " VALUES (" + placeholders + ")"); + } +} diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java index bb780d876..c2b774a64 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java @@ -22,14 +22,9 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.RdbSink; -import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; -import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.sink.rdb.JDBCOptions; +import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; /** * Date: 2017/2/27 @@ -43,52 +38,24 @@ public class MysqlSink extends RdbSink implements IStreamSinkGener { private static final String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; public MysqlSink() { + super(new MySQLDialect()); } @Override - public RetractJDBCOutputFormat getOutputFormat() { - return new RetractJDBCOutputFormat(); + public JDBCUpsertOutputFormat getOutputFormat() { + JDBCOptions jdbcOptions = JDBCOptions.builder() + .setDBUrl(dbURL).setDialect(jdbcDialect) + .setUsername(userName).setPassword(password) + .setTableName(tableName).build(); + + return JDBCUpsertOutputFormat.builder() + .setOptions(jdbcOptions) + .setFieldNames(fieldNames) + .setFlushMaxSize(batchNum) + .setFlushIntervalMills(batchWaitInterval) + .setFieldTypes(sqlTypes) + .setKeyFields(primaryKeys) + .setAllReplace(allReplace) + .setUpdateMode(updateMode).build(); } - - @Override - public void buildSql(String scheam, String tableName, List fields) { - buildReplaceUpsertStatement(tableName, fields); - buildDuplicateUpsertStatement(tableName, fields); - } - - @Override - public String buildUpdateSql(String schema, String tableName, List fieldNames, Map> realIndexes, List fullField) { - return null; - } - - private void buildReplaceUpsertStatement(String tableName, List fields) { - this.sql = getUpsertIntoStatement("REPLACE ", tableName, fields); - } - - public void buildDuplicateUpsertStatement(String tableName, List fields) { - String updateClause = fields.stream().map(f -> quoteIdentifier(f) + "=IFNULL(VALUES(" + quoteIdentifier(f) + "),"+ quoteIdentifier(f) + ")") - .collect(Collectors.joining(", ")); - - this.sql = getUpsertIntoStatement("INSERT",tableName, fields) + - " ON DUPLICATE KEY UPDATE " + updateClause; - } - - public String getUpsertIntoStatement(String operator, String tableName, List fields) { - String columns = fields.stream() - .map(this::quoteIdentifier) - .collect(Collectors.joining(", ")); - - String placeholders = fields.stream() - .map(f -> "?") - .collect(Collectors.joining(", ")); - - return operator + " INTO " + quoteIdentifier(tableName) + - "(" + columns + ")" + " VALUES (" + placeholders + ")"; - } - - @Override - public String getDriverName() { - return MYSQL_DRIVER; - } - } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index e744ed97b..b6a68c4e5 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -20,10 +20,8 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.ExtendOutputFormat; -import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; -import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.Iterator; diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java index 01cdf4bad..6fd18d10b 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java @@ -2,7 +2,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.RdbSink; -import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; import java.util.List; import java.util.Map; diff --git a/pom.xml b/pom.xml index 8f134831a..e77bef17b 100644 --- a/pom.xml +++ b/pom.xml @@ -10,29 +10,29 @@ http://maven.apache.org core - kafka09 - kafka10 - kafka11 - kafka + + + + mysql - hbase - elasticsearch5 - mongo - redis5 - launcher + + + + + rdb - sqlserver - oracle - cassandra - kudu - postgresql + + + + + - serversocket - console - clickhouse - impala - db2 - polardb + + + + + + diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java index 663816ca0..ac6e2e4ea 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java @@ -23,7 +23,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.postgresql.table.PostgresqlTableInfo; import com.dtstack.flink.sql.sink.rdb.RdbSink; -import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.commons.lang3.StringUtils; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java new file mode 100644 index 000000000..28811d2fd --- /dev/null +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.rdb; + +import org.apache.flink.types.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.math.BigDecimal; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.List; + +/** + * Utils for jdbc connectors. + */ +public class JDBCTypeConvertUtils { + + private static final Logger LOG = LoggerFactory.getLogger(JDBCTypeConvertUtils.class); + + /** + * Adds a record to the prepared statement. + * + *

When this method is called, the output format is guaranteed to be opened. + * + *

WARNING: this may fail when no column types specified (because a best effort approach is attempted in order to + * insert a null value but it's not guaranteed that the JDBC driver handles PreparedStatement.setObject(pos, null)) + * + * @param upload The prepared statement. + * @param typesArray The jdbc types of the row. + * @param row The records to add to the output. + * @see PreparedStatement + */ + public static void setRecordToStatement(PreparedStatement upload, int[] typesArray, Row row) throws SQLException { + if (typesArray != null && typesArray.length > 0 && typesArray.length != row.getArity()) { + LOG.warn("Column SQL types array doesn't match arity of passed Row! Check the passed array..."); + } + if (typesArray == null) { + // no types provided + for (int index = 0; index < row.getArity(); index++) { + LOG.warn("Unknown column type for column {}. Best effort approach to set its value: {}.", index + 1, row.getField(index)); + upload.setObject(index + 1, row.getField(index)); + } + } else { + // types provided + for (int i = 0; i < row.getArity(); i++) { + setField(upload, typesArray[i], row.getField(i), i); + } + } + } + + public static void setField(PreparedStatement upload, int type, Object field, int index) throws SQLException { + if (field == null) { + upload.setNull(index + 1, type); + } else { + try { + // casting values as suggested by http://docs.oracle.com/javase/1.5.0/docs/guide/jdbc/getstart/mapping.html + switch (type) { + case java.sql.Types.NULL: + upload.setNull(index + 1, type); + break; + case java.sql.Types.BOOLEAN: + case java.sql.Types.BIT: + upload.setBoolean(index + 1, (boolean) field); + break; + case java.sql.Types.CHAR: + case java.sql.Types.NCHAR: + case java.sql.Types.VARCHAR: + case java.sql.Types.LONGVARCHAR: + case java.sql.Types.LONGNVARCHAR: + upload.setString(index + 1, (String) field); + break; + case java.sql.Types.TINYINT: + upload.setByte(index + 1, (byte) field); + break; + case java.sql.Types.SMALLINT: + upload.setShort(index + 1, (short) field); + break; + case java.sql.Types.INTEGER: + upload.setInt(index + 1, (int) field); + break; + case java.sql.Types.BIGINT: + upload.setLong(index + 1, (long) field); + break; + case java.sql.Types.REAL: + upload.setFloat(index + 1, (float) field); + break; + case java.sql.Types.FLOAT: + case java.sql.Types.DOUBLE: + upload.setDouble(index + 1, (double) field); + break; + case java.sql.Types.DECIMAL: + case java.sql.Types.NUMERIC: + upload.setBigDecimal(index + 1, (java.math.BigDecimal) field); + break; + case java.sql.Types.DATE: + upload.setDate(index + 1, (java.sql.Date) field); + break; + case java.sql.Types.TIME: + upload.setTime(index + 1, (java.sql.Time) field); + break; + case java.sql.Types.TIMESTAMP: + upload.setTimestamp(index + 1, (java.sql.Timestamp) field); + break; + case java.sql.Types.BINARY: + case java.sql.Types.VARBINARY: + case java.sql.Types.LONGVARBINARY: + upload.setBytes(index + 1, (byte[]) field); + break; + default: + upload.setObject(index + 1, field); + LOG.warn("Unmanaged sql type ({}) for column {}. Best effort approach to set its value: {}.", + type, index + 1, field); + // case java.sql.Types.SQLXML + // case java.sql.Types.ARRAY: + // case java.sql.Types.JAVA_OBJECT: + // case java.sql.Types.BLOB: + // case java.sql.Types.CLOB: + // case java.sql.Types.NCLOB: + // case java.sql.Types.DATALINK: + // case java.sql.Types.DISTINCT: + // case java.sql.Types.OTHER: + // case java.sql.Types.REF: + // case java.sql.Types.ROWID: + // case java.sql.Types.STRUC + } + } catch (ClassCastException e) { + // enrich the exception with detailed information. + String errorMessage = String.format( + "%s, field index: %s, field value: %s.", e.getMessage(), index, field); + ClassCastException enrichedException = new ClassCastException(errorMessage); + enrichedException.setStackTrace(e.getStackTrace()); + throw enrichedException; + } + } + } + + /** + * By now specified class type conversion. + * FIXME Follow-up has added a new type of time needs to be modified + * + * @param fieldTypeArray + */ + public static int[] buildSqlTypes(List fieldTypeArray) { + + int[] tmpFieldsType = new int[fieldTypeArray.size()]; + for (int i = 0; i < fieldTypeArray.size(); i++) { + String fieldType = fieldTypeArray.get(i).getName(); + if (fieldType.equals(Integer.class.getName())) { + tmpFieldsType[i] = Types.INTEGER; + }else if (fieldType.equals(Boolean.class.getName())) { + tmpFieldsType[i] = Types.BOOLEAN; + }else if (fieldType.equals(Long.class.getName())) { + tmpFieldsType[i] = Types.BIGINT; + } else if (fieldType.equals(Byte.class.getName())) { + tmpFieldsType[i] = Types.TINYINT; + } else if (fieldType.equals(Short.class.getName())) { + tmpFieldsType[i] = Types.SMALLINT; + } else if (fieldType.equals(String.class.getName())) { + tmpFieldsType[i] = Types.CHAR; + } else if (fieldType.equals(Byte.class.getName())) { + tmpFieldsType[i] = Types.BINARY; + } else if (fieldType.equals(Float.class.getName())) { + tmpFieldsType[i] = Types.FLOAT; + } else if (fieldType.equals(Double.class.getName())) { + tmpFieldsType[i] = Types.DOUBLE; + } else if (fieldType.equals(Timestamp.class.getName())) { + tmpFieldsType[i] = Types.TIMESTAMP; + } else if (fieldType.equals(BigDecimal.class.getName())) { + tmpFieldsType[i] = Types.DECIMAL; + } else if (fieldType.equals(Date.class.getName())) { + tmpFieldsType[i] = Types.DATE; + } else { + throw new RuntimeException("no support field type for sql. the input type:" + fieldType); + } + } + + return tmpFieldsType; + } + +} diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java index 96cce9516..8b3bb7e5c 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java @@ -18,20 +18,20 @@ package com.dtstack.flink.sql.sink.rdb; import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; +import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; import com.dtstack.flink.sql.table.TargetTableInfo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.DataStreamSink; import org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction; -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import java.io.Serializable; import java.math.BigDecimal; @@ -40,7 +40,6 @@ import java.sql.Types; import java.util.Arrays; import java.util.List; -import java.util.Map; /** * Reason: @@ -73,7 +72,7 @@ public abstract class RdbSink implements RetractStreamTableSink, Serializab protected String sql; - protected List primaryKeys; + protected String[] primaryKeys; protected String[] fieldNames; @@ -81,130 +80,46 @@ public abstract class RdbSink implements RetractStreamTableSink, Serializab private int parallelism = -1; - private String schema; - - public RichSinkFunction createJdbcSinkFunc() { - if (driverName == null || dbURL == null || userName == null - || password == null || sqlTypes == null || tableName == null) { - throw new RuntimeException("any of params in(driverName, dbURL, userName, password, type, tableName) " + - " must not be null. please check it!!!"); - } - RetractJDBCOutputFormat outputFormat = getOutputFormat(); - outputFormat.setDbURL(dbURL); - outputFormat.setDrivername(driverName); - outputFormat.setUsername(userName); - outputFormat.setPassword(password); - outputFormat.setInsertQuery(sql); - outputFormat.setBatchNum(batchNum); - outputFormat.setBatchWaitInterval(batchWaitInterval); - outputFormat.setTypesArray(sqlTypes); - outputFormat.setTableName(tableName); - outputFormat.setDbType(dbType); - outputFormat.setSchema(schema); - outputFormat.setDbSink(this); - - outputFormat.verifyField(); - OutputFormatSinkFunction outputFormatSinkFunc = new OutputFormatSinkFunction(outputFormat); - return outputFormatSinkFunc; - } - - - @Override - public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { - RdbTableInfo rdbTableInfo = (RdbTableInfo) targetTableInfo; - - String tmpDbURL = rdbTableInfo.getUrl(); - String tmpUserName = rdbTableInfo.getUserName(); - String tmpPassword = rdbTableInfo.getPassword(); - String tmpTableName = rdbTableInfo.getTableName(); - String tmpRegisterName = rdbTableInfo.getName(); + protected String schema; - Integer tmpSqlBatchSize = rdbTableInfo.getBatchSize(); - if (tmpSqlBatchSize != null) { - setBatchNum(tmpSqlBatchSize); - } + protected JDBCDialect jdbcDialect; - Long batchWaitInterval = rdbTableInfo.getBatchWaitInterval(); - if (batchWaitInterval != null) { - setBatchWaitInterval(batchWaitInterval); - } + protected boolean allReplace; - Integer tmpSinkParallelism = rdbTableInfo.getParallelism(); - if (tmpSinkParallelism != null) { - setParallelism(tmpSinkParallelism); - } + protected String updateMode; - List fields = Arrays.asList(rdbTableInfo.getFields()); - List fieldTypeArray = Arrays.asList(rdbTableInfo.getFieldClasses()); + public RdbSink(JDBCDialect jdbcDialect) { + this.jdbcDialect = jdbcDialect; + } - this.driverName = getDriverName(); - this.dbURL = tmpDbURL; - this.userName = tmpUserName; - this.password = tmpPassword; - this.tableName = tmpTableName; - this.registerTabName = tmpRegisterName; + @Override + public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { + RdbTableInfo rdbTableInfo = (RdbTableInfo) targetTableInfo; + this.batchNum = rdbTableInfo.getBatchSize() == null ? batchNum : rdbTableInfo.getBatchSize(); + this.batchWaitInterval = rdbTableInfo.getBatchWaitInterval() == null ? + batchWaitInterval : rdbTableInfo.getBatchWaitInterval(); + this.parallelism = rdbTableInfo.getParallelism() == null ? parallelism : rdbTableInfo.getParallelism(); + this.dbURL = rdbTableInfo.getUrl(); + this.userName = rdbTableInfo.getUserName(); + this.password = rdbTableInfo.getPassword(); + this.tableName = rdbTableInfo.getTableName(); + this.registerTabName = rdbTableInfo.getName(); this.primaryKeys = rdbTableInfo.getPrimaryKeys(); this.dbType = rdbTableInfo.getType(); this.schema = rdbTableInfo.getSchema(); - - buildSql(schema, tableName, fields); - buildSqlTypes(fieldTypeArray); + List fieldTypeArray = Arrays.asList(rdbTableInfo.getFieldClasses()); + this.sqlTypes = JDBCTypeConvertUtils.buildSqlTypes(fieldTypeArray); + this.allReplace = rdbTableInfo.isAllReplace(); + this.updateMode = rdbTableInfo.getUpdateMode(); return this; } - /** - * By now specified class type conversion. - * FIXME Follow-up has added a new type of time needs to be modified - * - * @param fieldTypeArray - */ - protected void buildSqlTypes(List fieldTypeArray) { - - int[] tmpFieldsType = new int[fieldTypeArray.size()]; - for (int i = 0; i < fieldTypeArray.size(); i++) { - String fieldType = fieldTypeArray.get(i).getName(); - if (fieldType.equals(Integer.class.getName())) { - tmpFieldsType[i] = Types.INTEGER; - }else if (fieldType.equals(Boolean.class.getName())) { - tmpFieldsType[i] = Types.BOOLEAN; - }else if (fieldType.equals(Long.class.getName())) { - tmpFieldsType[i] = Types.BIGINT; - } else if (fieldType.equals(Byte.class.getName())) { - tmpFieldsType[i] = Types.TINYINT; - } else if (fieldType.equals(Short.class.getName())) { - tmpFieldsType[i] = Types.SMALLINT; - } else if (fieldType.equals(String.class.getName())) { - tmpFieldsType[i] = Types.CHAR; - } else if (fieldType.equals(Byte.class.getName())) { - tmpFieldsType[i] = Types.BINARY; - } else if (fieldType.equals(Float.class.getName())) { - tmpFieldsType[i] = Types.FLOAT; - } else if (fieldType.equals(Double.class.getName())) { - tmpFieldsType[i] = Types.DOUBLE; - } else if (fieldType.equals(Timestamp.class.getName())) { - tmpFieldsType[i] = Types.TIMESTAMP; - } else if (fieldType.equals(BigDecimal.class.getName())) { - tmpFieldsType[i] = Types.DECIMAL; - } else if (fieldType.equals(Date.class.getName())) { - tmpFieldsType[i] = Types.DATE; - } else { - throw new RuntimeException("no support field type for sql. the input type:" + fieldType); - } - } - - this.sqlTypes = tmpFieldsType; - } - - @Override public void emitDataStream(DataStream> dataStream) { - RichSinkFunction richSinkFunction = createJdbcSinkFunc(); - DataStreamSink streamSink = dataStream.addSink(richSinkFunction); - streamSink.name(registerTabName); - if (parallelism > 0) { - streamSink.setParallelism(parallelism); - } + dataStream.addSink(new OutputFormatSinkFunction(getOutputFormat())) + .setParallelism(parallelism) + .name(registerTabName); } @Override @@ -214,15 +129,6 @@ public TableSink> configure(String[] fieldNames, TypeInform return this; } - - public void setBatchNum(int batchNum) { - this.batchNum = batchNum; - } - - public void setBatchWaitInterval(long batchWaitInterval) { - this.batchWaitInterval = batchWaitInterval; - } - @Override public TupleTypeInfo> getOutputType() { return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), getRecordType()); @@ -243,38 +149,6 @@ public TypeInformation[] getFieldTypes() { return fieldTypes; } + public abstract JDBCUpsertOutputFormat getOutputFormat(); - public void setParallelism(int parallelism) { - this.parallelism = parallelism; - } - - public void setDbType(String dbType) { - this.dbType = dbType; - } - - /** - * you need to implements this method in your own class. - * - * @param tableName - * @param fields - */ - public abstract void buildSql(String schema, String tableName, List fields); - - /** - * sqlserver and oracle maybe implement - * - * @param tableName - * @param fieldNames - * @param realIndexes - * @return - */ - public abstract String buildUpdateSql(String schema, String tableName, List fieldNames, Map> realIndexes, List fullField); - - public abstract String getDriverName(); - - public abstract RetractJDBCOutputFormat getOutputFormat(); - - public String quoteIdentifier(String identifier) { - return "`" + identifier + "`"; - } } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java new file mode 100644 index 000000000..111c4c8dd --- /dev/null +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.rdb.dialect; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Handle the SQL dialect of jdbc driver. + */ +public interface JDBCDialect extends Serializable { + + /** + * Check if this dialect instance can handle a certain jdbc url. + * @param url the jdbc url. + * @return True if the dialect can be applied on the given jdbc url. + */ + boolean canHandle(String url); + + /** + * @return the default driver class name, if user not configure the driver class name, + * then will use this one. + */ + default Optional defaultDriverName() { + return Optional.empty(); + } + + /** + * Quotes the identifier. This is used to put quotes around the identifier in case the column + * name is a reserved keyword, or in case it contains characters that require quotes (e.g. space). + * Default using double quotes {@code "} to quote. + */ + default String quoteIdentifier(String identifier) { + return "\"" + identifier + "\""; + } + + + /** + * Get dialect upsert statement, the database has its own upsert syntax, such as Mysql + * using DUPLICATE KEY UPDATE, and PostgresSQL using ON CONFLICT... DO UPDATE SET.. + * + * if allReplace is true, new data will replace previous values such as mysql replace into .. + * + * @return None if dialect does not support upsert statement, the writer will degrade to + * the use of select + update/insert, this performance is poor. + * + * @param tableName + * @param fieldNames + * @param uniqueKeyFields + * @param allReplace + * @return + */ + default Optional getUpsertStatement( + String tableName, String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + return Optional.empty(); + } + + /** + * Get row exists statement by condition fields. Default use SELECT. + */ + default String getRowExistsStatement(String tableName, String[] conditionFields) { + String fieldExpressions = Arrays.stream(conditionFields) + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(" AND ")); + return "SELECT 1 FROM " + quoteIdentifier(tableName) + " WHERE " + fieldExpressions; + } + + /** + * Get insert into statement. + */ + default String getInsertIntoStatement(String tableName, String[] fieldNames) { + String columns = Arrays.stream(fieldNames) + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")); + String placeholders = Arrays.stream(fieldNames) + .map(f -> "?") + .collect(Collectors.joining(", ")); + return "INSERT INTO " + quoteIdentifier(tableName) + + "(" + columns + ")" + " VALUES (" + placeholders + ")"; + } + + /** + * Get update one row statement by condition fields, default not use limit 1, + * because limit 1 is a sql dialect. + */ + default String getUpdateStatement(String tableName, String[] fieldNames, String[] conditionFields) { + String setClause = Arrays.stream(fieldNames) + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(", ")); + String conditionClause = Arrays.stream(conditionFields) + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(" AND ")); + return "UPDATE " + quoteIdentifier(tableName) + + " SET " + setClause + + " WHERE " + conditionClause; + } + + /** + * Get delete one row statement by condition fields, default not use limit 1, + * because limit 1 is a sql dialect. + */ + default String getDeleteStatement(String tableName, String[] conditionFields) { + String conditionClause = Arrays.stream(conditionFields) + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(" AND ")); + return "DELETE FROM " + quoteIdentifier(tableName) + " WHERE " + conditionClause; + } + + /** + * Get select fields statement by condition fields. Default use SELECT. + */ + default String getSelectFromStatement(String tableName, String[] selectFields, String[] conditionFields) { + String selectExpressions = Arrays.stream(selectFields) + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")); + String fieldExpressions = Arrays.stream(conditionFields) + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(" AND ")); + return "SELECT " + selectExpressions + " FROM " + + quoteIdentifier(tableName) + (conditionFields.length > 0 ? " WHERE " + fieldExpressions : ""); + } +} diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/ExtendOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/ExtendOutputFormat.java deleted file mode 100644 index 6265bbadd..000000000 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/ExtendOutputFormat.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.rdb.format; - -import com.dtstack.flink.sql.util.DtStringUtil; -import org.apache.commons.lang3.StringUtils; -import com.google.common.collect.Maps; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -/** - * Reason: - * Date: 2018/11/30 - * Company: www.dtstack.com - * - * @author maqi - */ -public class ExtendOutputFormat extends RetractJDBCOutputFormat { - - - @Override - public boolean isReplaceInsertQuery() throws SQLException { - fillRealIndexes(); - fillFullColumns(); - - if (!getRealIndexes().isEmpty()) { - for (List value : getRealIndexes().values()) { - for (String fieldName : getDbSink().getFieldNames()) { - if (containsIgnoreCase(value, fieldName)) { - return true; - } - } - } - } - return false; - } - - /** - * get db all index - * - * @throws SQLException - */ - public void fillRealIndexes() throws SQLException { - Map> map = Maps.newHashMap(); - - ResultSet rs = getDbConn().getMetaData().getIndexInfo(null, getSchema(), DtStringUtil.addQuoteForStr(getTableName()), true, false); - - while (rs.next()) { - String indexName = rs.getString("INDEX_NAME"); - if (StringUtils.isNotBlank(indexName) && !map.containsKey(indexName)) { - map.put(indexName, new ArrayList<>()); - } - String column_name = rs.getString("COLUMN_NAME"); - if (StringUtils.isNotBlank(column_name)) { - map.get(indexName).add(column_name); - } - } - - for (Map.Entry> entry : map.entrySet()) { - String k = entry.getKey(); - List v = entry.getValue(); - if (v != null && v.size() != 0 && v.get(0) != null) { - realIndexesAdd(k, v); - } - } - } - - - - /** - * get db all column name - * - * @throws SQLException - */ - public void fillFullColumns() throws SQLException { - // table name not quote - ResultSet rs = getDbConn().getMetaData().getColumns(null, getSchema(), getTableName(), null); - while (rs.next()) { - String columnName = rs.getString("COLUMN_NAME"); - if (StringUtils.isNotBlank(columnName)) { - fullFieldAdd(columnName); - } - } - } - - public boolean containsIgnoreCase(List l, String s) { - Iterator it = l.iterator(); - while (it.hasNext()) { - if (it.next().equalsIgnoreCase(s)) - return true; - } - return false; - } - -} diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java new file mode 100644 index 000000000..239d76f47 --- /dev/null +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.rdb.format; + + +import com.dtstack.flink.sql.enums.EUpdateMode; +import com.dtstack.flink.sql.factory.DTThreadFactory; +import com.dtstack.flink.sql.sink.rdb.JDBCOptions; +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import com.dtstack.flink.sql.sink.rdb.writer.AppendOnlyWriter; +import com.dtstack.flink.sql.sink.rdb.writer.JDBCWriter; +import com.dtstack.flink.sql.sink.rdb.writer.UpsertWriter; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.types.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * An upsert OutputFormat for JDBC. + */ +public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat> { + + private static final long serialVersionUID = 1L; + + private static final Logger LOG = LoggerFactory.getLogger(JDBCUpsertOutputFormat.class); + + static final int RECEIVEDATA_PRINT_FREQUENTY = 1000; + + private final String tableName; + private final JDBCDialect dialect; + private final String[] fieldNames; + private final String[] keyFields; + private final int[] fieldTypes; + + private final int flushMaxSize; + private final long flushIntervalMills; + private final boolean allReplace; + private final String updateMode; + + private transient JDBCWriter jdbcWriter; + private transient int batchCount = 0; + private transient volatile boolean closed = false; + + private transient ScheduledExecutorService scheduler; + private transient ScheduledFuture scheduledFuture; + + public JDBCUpsertOutputFormat( + JDBCOptions options, + String[] fieldNames, + String[] keyFields, + int[] fieldTypes, + int flushMaxSize, + long flushIntervalMills, + boolean allReplace, + String updateMode) { + super(options.getUsername(), options.getPassword(), options.getDriverName(), options.getDbURL()); + this.tableName = options.getTableName(); + this.dialect = options.getDialect(); + this.fieldNames = fieldNames; + this.keyFields = keyFields; + this.fieldTypes = fieldTypes; + this.flushMaxSize = flushMaxSize; + this.flushIntervalMills = flushIntervalMills; + this.allReplace = allReplace; + this.updateMode = updateMode; + } + + /** + * Connects to the target database and initializes the prepared statement. + * + * @param taskNumber The number of the parallel instance. + * @throws IOException Thrown, if the output could not be opened due to an + * I/O problem. + */ + @Override + public void open(int taskNumber, int numTasks) throws IOException { + try { + establishConnection(); + initMetric(); + + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.APPEND.name()) || keyFields == null || keyFields.length == 0) { + String insertSQL = dialect.getInsertIntoStatement(tableName, fieldNames); + LOG.info("execute insert sql： {}", insertSQL); + System.out.println("execute insert sql :" + insertSQL); + jdbcWriter = new AppendOnlyWriter(insertSQL, fieldTypes, this); + } else { + jdbcWriter = UpsertWriter.create( + dialect, tableName, fieldNames, fieldTypes, keyFields, + getRuntimeContext().getExecutionConfig().isObjectReuseEnabled(), allReplace, this); + } + jdbcWriter.open(connection); + } catch (SQLException sqe) { + throw new IllegalArgumentException("open() failed.", sqe); + } catch (ClassNotFoundException cnfe) { + throw new IllegalArgumentException("JDBC driver class not found.", cnfe); + } + + if (flushIntervalMills != 0) { + this.scheduler = new ScheduledThreadPoolExecutor(1, + new DTThreadFactory("jdbc-upsert-output-format")); + this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(() -> { + synchronized (JDBCUpsertOutputFormat.this) { + if (closed) { + return; + } + try { + flush(); + } catch (Exception e) { + throw new RuntimeException("Writing records to JDBC failed.", e); + } + } + }, flushIntervalMills, flushIntervalMills, TimeUnit.MILLISECONDS); + } + } + + @Override + public synchronized void writeRecord(Tuple2 tuple2) throws IOException { + checkConnectionOpen(); + try { + // Receive data + outRecords.inc(); + if (outRecords.getCount() % RECEIVEDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { + LOG.info("Receive data : {}", tuple2); + } + jdbcWriter.addRecord(tuple2); + batchCount++; + if (batchCount >= flushMaxSize) { + flush(); + } + } catch (Exception e) { + throw new RuntimeException("Writing records to JDBC failed.", e); + } + } + + private void checkConnectionOpen() { + try { + if (connection.isClosed()) { + LOG.info("db connection reconnect.."); + establishConnection(); + } + } catch (SQLException e) { + LOG.error("check connection open failed..", e); + } catch (ClassNotFoundException e) { + LOG.error("load jdbc class error when reconnect db..", e); + } + } + + public synchronized void flush() throws Exception { + jdbcWriter.executeBatch(connection); + batchCount = 0; + } + + /** + * Executes prepared statement and closes all resources of this instance. + * + * @throws IOException Thrown, if the input could not be closed properly. + */ + @Override + public synchronized void close() throws IOException { + if (closed) { + return; + } + closed = true; + if (this.scheduledFuture != null) { + scheduledFuture.cancel(false); + this.scheduler.shutdown(); + } + + if (batchCount > 0) { + try { + flush(); + } catch (Exception e) { + throw new RuntimeException("Writing records to JDBC failed.", e); + } + } + + try { + jdbcWriter.close(); + } catch (SQLException e) { + LOG.warn("Close JDBC writer failed.", e); + } + + closeDbConnection(); + } + + public static Builder builder() { + return new Builder(); + } + + /** + * Builder for a {@link JDBCUpsertOutputFormat}. + */ + public static class Builder { + private JDBCOptions options; + private String[] fieldNames; + private String[] keyFields; + private int[] fieldTypes; + private int flushMaxSize = DEFAULT_FLUSH_MAX_SIZE; + private long flushIntervalMills = DEFAULT_FLUSH_INTERVAL_MILLS; + private boolean allReplace = DEFAULT_ALLREPLACE_VALUE; + private String updateMode; + + /** + * required, jdbc options. + */ + public Builder setOptions(JDBCOptions options) { + this.options = options; + return this; + } + + /** + * required, field names of this jdbc sink. + */ + public Builder setFieldNames(String[] fieldNames) { + this.fieldNames = fieldNames; + return this; + } + + /** + * required, upsert unique keys. + */ + public Builder setKeyFields(String[] keyFields) { + this.keyFields = keyFields; + return this; + } + + /** + * required, field types of this jdbc sink. + */ + public Builder setFieldTypes(int[] fieldTypes) { + this.fieldTypes = fieldTypes; + return this; + } + + /** + * optional, flush max size (includes all append, upsert and delete records), + * over this number of records, will flush data. + */ + public Builder setFlushMaxSize(int flushMaxSize) { + this.flushMaxSize = flushMaxSize; + return this; + } + + /** + * optional, flush interval mills, over this time, asynchronous threads will flush data. + */ + public Builder setFlushIntervalMills(long flushIntervalMills) { + this.flushIntervalMills = flushIntervalMills; + return this; + } + + public Builder setAllReplace(boolean allReplace) { + this.allReplace = allReplace; + return this; + } + + public Builder setUpdateMode(String updateMode) { + this.updateMode = updateMode; + return this; + } + + /** + * Finalizes the configuration and checks validity. + * + * @return Configured JDBCUpsertOutputFormat + */ + public JDBCUpsertOutputFormat build() { + checkNotNull(options, "No options supplied."); + checkNotNull(fieldNames, "No fieldNames supplied."); + return new JDBCUpsertOutputFormat( + options, fieldNames, keyFields, fieldTypes, flushMaxSize, flushIntervalMills, allReplace, updateMode); + } + } +} diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java deleted file mode 100644 index 5eef60186..000000000 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java +++ /dev/null @@ -1,497 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.rdb.format; - -import com.dtstack.flink.sql.sink.rdb.RdbSink; -import com.dtstack.flink.sql.util.JDBCUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.configuration.Configuration; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import org.apache.flink.types.Row; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import com.dtstack.flink.sql.sink.MetricOutputFormat; -import sun.rmi.runtime.Log; - -/** - * OutputFormat to write tuples into a database. - * The OutputFormat has to be configured using the supplied OutputFormatBuilder. - * - */ -public class RetractJDBCOutputFormat extends MetricOutputFormat { - private static final long serialVersionUID = 1L; - - private static final Logger LOG = LoggerFactory.getLogger(RetractJDBCOutputFormat.class); - - private static int dirtyDataPrintFrequency = 1000; - - private static int receiveDataPrintFrequency = 1000; - - private String username; - private String password; - private String drivername; - private String dbURL; - private String tableName; - private String dbType; - private String schema; - private RdbSink dbSink; - // trigger preparedStatement execute batch interval - private long batchWaitInterval = 10000l; - // PreparedStatement execute batch num - private int batchNum = 100; - private String insertQuery; - public int[] typesArray; - - /** 存储用于批量写入的数据 */ - protected List rows = new ArrayList(); - - private Connection dbConn; - private PreparedStatement upload; - private transient ScheduledThreadPoolExecutor timerService; - - - //index field - private Map> realIndexes = Maps.newHashMap(); - //full field - private List fullField = Lists.newArrayList(); - - public RetractJDBCOutputFormat() { - } - - @Override - public void configure(Configuration parameters) { - } - - /** - * Connects to the target database and initializes the prepared statement. - * - * @param taskNumber The number of the parallel instance. - * @throws IOException Thrown, if the output could not be opened due to an - * I/O problem. - */ - @Override - public void open(int taskNumber, int numTasks) throws IOException { - try { - LOG.info("PreparedStatement execute batch num is {}", batchNum); - dbConn = establishConnection(); - initMetric(); - - if (existTabname()) { - if (isReplaceInsertQuery()) { - insertQuery = dbSink.buildUpdateSql(schema , tableName, Arrays.asList(dbSink.getFieldNames()), realIndexes, fullField); - } - upload = dbConn.prepareStatement(insertQuery); - } else { - throw new SQLException("Table " + tableName + " doesn't exist"); - } - - if (batchWaitInterval > 0 && batchNum > 1) { - LOG.info("open batch wait interval scheduled, interval is {} ms", batchWaitInterval); - - timerService = new ScheduledThreadPoolExecutor(1); - timerService.scheduleAtFixedRate(() -> { - submitExecuteBatch(); - }, 0, batchWaitInterval, TimeUnit.MILLISECONDS); - - } - - } catch (SQLException sqe) { - LOG.error("", sqe); - throw new IllegalArgumentException("open() failed.", sqe); - } catch (ClassNotFoundException cnfe) { - LOG.error("", cnfe); - throw new IllegalArgumentException("JDBC driver class not found.", cnfe); - } - } - - - public Connection establishConnection() throws SQLException, ClassNotFoundException, IOException { - Connection connection ; - JDBCUtils.forName(drivername, getClass().getClassLoader()); - if (username == null) { - connection = DriverManager.getConnection(dbURL); - } else { - connection = DriverManager.getConnection(dbURL, username, password); - } - connection.setAutoCommit(false); - return connection; - } - - /** - * Adds a record to the prepared statement. - *

- * When this method is called, the output format is guaranteed to be opened. - *

- *

- * WARNING: this may fail when no column types specified (because a best effort approach is attempted in order to - * insert a null value but it's not guaranteed that the JDBC driver handles PreparedStatement.setObject(pos, null)) - * - * @param tuple2 The records to add to the output. - * @throws IOException Thrown, if the records could not be added due to an I/O problem. - * @see PreparedStatement - */ - @Override - public void writeRecord(Tuple2 tuple2) { - - Tuple2 tupleTrans = tuple2; - Boolean retract = tupleTrans.getField(0); - Row row = tupleTrans.getField(1); - - if (typesArray != null && typesArray.length > 0 && typesArray.length != row.getArity()) { - LOG.warn("Column SQL types array doesn't match arity of passed Row! Check the passed array..."); - } - - if (retract) { - outRecords.inc(); - if (outRecords.getCount() % receiveDataPrintFrequency == 0) { - LOG.info("Receive data : {}", row); - } - insertWrite(row); - } else { - //do nothing - } - } - - - private void insertWrite(Row row) { - checkConnectionOpen(dbConn); - try { - if (batchNum == 1) { - writeSingleRecord(row); - } else { - updatePreparedStmt(row, upload); - rows.add(row); - upload.addBatch(); - if (rows.size() >= batchNum) { - submitExecuteBatch(); - } - } - } catch (SQLException e) { - LOG.error("", e); - } - - } - - private void writeSingleRecord(Row row) { - try { - updatePreparedStmt(row, upload); - upload.executeUpdate(); - dbConn.commit(); - } catch (SQLException e) { - outDirtyRecords.inc(); - if (outDirtyRecords.getCount() % dirtyDataPrintFrequency == 0 || LOG.isDebugEnabled()) { - LOG.error("record insert failed ..{}", row.toString()); - LOG.error("", e); - } - } - } - - private void updatePreparedStmt(Row row, PreparedStatement pstmt) throws SQLException { - if (typesArray == null) { - // no types provided - for (int index = 0; index < row.getArity(); index++) { - LOG.warn("Unknown column type for column %s. Best effort approach to set its value: %s.", index + 1, row.getField(index)); - pstmt.setObject(index + 1, row.getField(index)); - } - } else { - // types provided - for (int index = 0; index < row.getArity(); index++) { - - if (row.getField(index) == null) { - pstmt.setNull(index + 1, typesArray[index]); - } else { - // casting values as suggested by http://docs.oracle.com/javase/1.5.0/docs/guide/jdbc/getstart/mapping.html - switch (typesArray[index]) { - case java.sql.Types.NULL: - pstmt.setNull(index + 1, typesArray[index]); - break; - case java.sql.Types.BOOLEAN: - case java.sql.Types.BIT: - pstmt.setBoolean(index + 1, (boolean) row.getField(index)); - break; - case java.sql.Types.CHAR: - case java.sql.Types.NCHAR: - case java.sql.Types.VARCHAR: - case java.sql.Types.LONGVARCHAR: - case java.sql.Types.LONGNVARCHAR: - pstmt.setString(index + 1, (String) row.getField(index)); - break; - case java.sql.Types.TINYINT: - pstmt.setByte(index + 1, (byte) row.getField(index)); - break; - case java.sql.Types.SMALLINT: - pstmt.setShort(index + 1, (short) row.getField(index)); - break; - case java.sql.Types.INTEGER: - pstmt.setInt(index + 1, (int) row.getField(index)); - break; - case java.sql.Types.BIGINT: - pstmt.setLong(index + 1, (long) row.getField(index)); - break; - case java.sql.Types.REAL: - case java.sql.Types.FLOAT: - pstmt.setFloat(index + 1, (float) row.getField(index)); - break; - case java.sql.Types.DOUBLE: - pstmt.setDouble(index + 1, (double) row.getField(index)); - break; - case java.sql.Types.DECIMAL: - case java.sql.Types.NUMERIC: - pstmt.setBigDecimal(index + 1, (java.math.BigDecimal) row.getField(index)); - break; - case java.sql.Types.DATE: - pstmt.setDate(index + 1, (java.sql.Date) row.getField(index)); - break; - case java.sql.Types.TIME: - pstmt.setTime(index + 1, (java.sql.Time) row.getField(index)); - break; - case java.sql.Types.TIMESTAMP: - pstmt.setTimestamp(index + 1, (java.sql.Timestamp) row.getField(index)); - break; - case java.sql.Types.BINARY: - case java.sql.Types.VARBINARY: - case java.sql.Types.LONGVARBINARY: - pstmt.setBytes(index + 1, (byte[]) row.getField(index)); - break; - default: - pstmt.setObject(index + 1, row.getField(index)); - LOG.warn("Unmanaged sql type (%s) for column %s. Best effort approach to set its value: %s.", - typesArray[index], index + 1, row.getField(index)); - // case java.sql.Types.SQLXML - // case java.sql.Types.ARRAY: - // case java.sql.Types.JAVA_OBJECT: - // case java.sql.Types.BLOB: - // case java.sql.Types.CLOB: - // case java.sql.Types.NCLOB: - // case java.sql.Types.DATALINK: - // case java.sql.Types.DISTINCT: - // case java.sql.Types.OTHER: - // case java.sql.Types.REF: - // case java.sql.Types.ROWID: - // case java.sql.Types.STRUC - } - } - } - } - } - - - private synchronized void submitExecuteBatch() { - try { - this.upload.executeBatch(); - dbConn.commit(); - } catch (SQLException e) { - try { - dbConn.rollback(); - } catch (SQLException e1) { - LOG.error("rollback data error !", e); - } - - rows.forEach(this::writeSingleRecord); - } finally { - rows.clear(); - } - } - - private void checkConnectionOpen(Connection dbConn) { - try { - if (dbConn.isClosed()) { - LOG.info("db connection reconnect.."); - dbConn= establishConnection(); - upload = dbConn.prepareStatement(insertQuery); - this.dbConn = dbConn; - } - } catch (SQLException e) { - LOG.error("check connection open failed..", e); - } catch (ClassNotFoundException e) { - LOG.error("load jdbc class error when reconnect db..", e); - } catch (IOException e) { - LOG.error("kerberos authentication failed..", e); - } - } - - /** - * Executes prepared statement and closes all resources of this instance. - * - * @throws IOException Thrown, if the input could not be closed properly. - */ - @Override - public void close() throws IOException { - try { - if (upload != null) { - upload.executeBatch(); - upload.close(); - } - if (null != timerService) { - timerService.shutdown(); - LOG.info("batch wait interval scheduled service closed "); - } - } catch (SQLException se) { - LOG.info("Inputformat couldn't be closed - ", se); - } finally { - upload = null; - rows.clear(); - } - - try { - if (dbConn != null) { - dbConn.close(); - } - } catch (SQLException se) { - LOG.info("Inputformat couldn't be closed - ", se); - } finally { - dbConn = null; - } - } - - - public boolean isReplaceInsertQuery() throws SQLException { - return false; - } - - public void verifyField() { - if (StringUtils.isBlank(username)) { - LOG.warn("Username was not supplied separately."); - } - if (StringUtils.isBlank(password)) { - LOG.warn("Username was not supplied separately."); - } - if (StringUtils.isBlank(dbURL)) { - throw new IllegalArgumentException("No dababase URL supplied."); - } - if (StringUtils.isBlank(insertQuery)) { - throw new IllegalArgumentException("No insertQuery supplied"); - } - if (StringUtils.isBlank(drivername)) { - throw new IllegalArgumentException("No driver supplied"); - } - } - - - public void setUsername(String username) { - this.username = username; - } - - public String getSchema() { - if (StringUtils.isNotEmpty(schema)) { - return schema; - } - return null; - } - - public void setSchema(String schema) { - this.schema = schema; - } - - public boolean existTabname() throws SQLException { - return dbConn.getMetaData().getTables(null, getSchema(), tableName, null).next(); - } - - public void setPassword(String password) { - this.password = password; - } - - public void setDrivername(String drivername) { - this.drivername = drivername; - } - - public String getDrivername() { - return this.drivername; - } - - public void setDbURL(String dbURL) { - this.dbURL = dbURL; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public void setDbType(String dbType) { - this.dbType = dbType; - } - - public void setDbSink(RdbSink dbSink) { - this.dbSink = dbSink; - } - - public void setBatchNum(int batchNum) { - this.batchNum = batchNum; - } - - public void setInsertQuery(String insertQuery) { - this.insertQuery = insertQuery; - } - - public void setTypesArray(int[] typesArray) { - this.typesArray = typesArray; - } - - public String getDbType() { - return dbType; - } - - public RdbSink getDbSink() { - return dbSink; - } - - public Connection getDbConn() { - return dbConn; - } - - public String getTableName() { - return tableName; - } - - public void realIndexesAdd(String index, List fieldes) { - this.realIndexes.put(index, fieldes); - } - - public Map> getRealIndexes() { - return realIndexes; - } - - - public void setBatchWaitInterval(long batchWaitInterval) { - this.batchWaitInterval = batchWaitInterval; - } - - public List getFullField() { - return fullField; - } - - public void fullFieldAdd(String colName) { - this.fullField.add(colName); - } -} diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java index beb51ffaa..9bbff3b4d 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java @@ -17,6 +17,7 @@ */ package com.dtstack.flink.sql.sink.rdb.table; +import com.dtstack.flink.sql.constrant.ConfigConstrant; import com.dtstack.flink.sql.table.AbsTableParser; import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; @@ -47,7 +48,8 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map { + Preconditions.checkArgument(getFieldList().contains(pk), "primary key " + pk + " not found in sink table field"); + }); + return true; } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java new file mode 100644 index 000000000..f9373c102 --- /dev/null +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.rdb.writer; + +import com.dtstack.flink.sql.sink.MetricOutputFormat; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.types.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import static com.dtstack.flink.sql.sink.rdb.JDBCTypeConvertUtils.setRecordToStatement; + +/** + * Just append record to jdbc, can not receive retract/delete message. + */ +public class AppendOnlyWriter implements JDBCWriter { + + private static final long serialVersionUID = 1L; + + private static final Logger LOG = LoggerFactory.getLogger(AppendOnlyWriter.class); + + private final String insertSQL; + private final int[] fieldTypes; + + private transient PreparedStatement statement; + private transient List rows; + // only use metric + private transient MetricOutputFormat metricOutputFormat; + + public AppendOnlyWriter(String insertSQL, int[] fieldTypes, MetricOutputFormat metricOutputFormat) { + this.insertSQL = insertSQL; + this.fieldTypes = fieldTypes; + this.metricOutputFormat = metricOutputFormat; + } + + @Override + public void open(Connection connection) throws SQLException { + this.rows = new ArrayList(); + this.statement = connection.prepareStatement(insertSQL); + } + + /** + * Append mode retract/delete message will not execute + * @param record + * @throws SQLException + */ + @Override + public void addRecord(Tuple2 record) throws SQLException { + if (!record.f0) { + return; + } + rows.add(record.f1); + } + + @Override + public void executeBatch(Connection connection) throws SQLException { + try { + rows.forEach(row -> { + try { + setRecordToStatement(statement, fieldTypes, row); + statement.addBatch(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + }); + statement.executeBatch(); + connection.commit(); + rows.clear(); + } catch (Exception e) { + LOG.debug("AppendOnlyWriter executeBatch error ", e); + cleanBatchWhenError(); + executeUpdate(connection); + } + } + + + @Override + public void executeUpdate(Connection connection) { + rows.forEach(row -> { + try { + setRecordToStatement(statement, fieldTypes, row); + statement.executeUpdate(); + connection.commit(); + } catch (SQLException e) { + metricOutputFormat.outDirtyRecords.inc(); + if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { + LOG.error("record insert failed ,this row is {}", row.toString()); + LOG.error("", e); + } + } + }); + rows.clear(); + } + + + @Override + public void cleanBatchWhenError() throws SQLException { + statement.clearBatch(); + } + + @Override + public void close() throws SQLException { + if (statement != null) { + statement.close(); + statement = null; + } + } + +} diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/JDBCWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/JDBCWriter.java new file mode 100644 index 000000000..efac8b3b8 --- /dev/null +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/JDBCWriter.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.rdb.writer; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.types.Row; + +import java.io.Serializable; +import java.sql.Connection; +import java.sql.SQLException; + +/** + * JDBCWriter used to execute statements (e.g. INSERT, UPSERT, DELETE). + */ +public interface JDBCWriter extends Serializable { + + public static final int DIRTYDATA_PRINT_FREQUENTY = 1000; + + /** + * Open the writer by JDBC Connection. It can create Statement from Connection. + */ + void open(Connection connection) throws SQLException; + + /** + * Add record to writer, the writer may cache the data. + */ + void addRecord(Tuple2 record) throws SQLException; + + /** + * Submits a batch of commands to the database for execution. + */ + void executeBatch(Connection connection) throws SQLException; + + /** + * clean batch cache when addBatch error + * @throws SQLException + */ + void cleanBatchWhenError() throws SQLException; + + /** + * Submits a single of commands to the database for execution. + */ + void executeUpdate(Connection connection) throws SQLException; + + /** + * Close JDBC related statements and other classes. + */ + void close() throws SQLException; +} diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java new file mode 100644 index 000000000..8ce2148d7 --- /dev/null +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java @@ -0,0 +1,332 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.rdb.writer; + +import com.dtstack.flink.sql.sink.MetricOutputFormat; +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.types.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +import static com.dtstack.flink.sql.sink.rdb.JDBCTypeConvertUtils.setRecordToStatement; +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * Upsert writer to deal with upsert, delete message.dd + */ +public abstract class UpsertWriter implements JDBCWriter { + + private static final long serialVersionUID = 1L; + private static final Logger LOG = LoggerFactory.getLogger(UpsertWriter.class); + + + public static UpsertWriter create( + JDBCDialect dialect, + String tableName, + String[] fieldNames, + int[] fieldTypes, + String[] keyFields, + boolean objectReuse, + boolean allReplace, + MetricOutputFormat metricOutputFormat) { + + checkNotNull(keyFields); + + List nameList = Arrays.asList(fieldNames); + int[] pkFields = Arrays.stream(keyFields).mapToInt(nameList::indexOf).toArray(); + int[] pkTypes = fieldTypes == null ? null : + Arrays.stream(pkFields).map(f -> fieldTypes[f]).toArray(); + + String deleteSQL = dialect.getDeleteStatement(tableName, keyFields); + LOG.info("deleteSQL is :{}", deleteSQL); + System.out.println("deleteSQL is :" + deleteSQL); + + Optional upsertSQL = dialect.getUpsertStatement(tableName, fieldNames, keyFields, allReplace); + LOG.info("execute UpsertStatement: {}", upsertSQL.orElse("use UsingInsertUpdateStatement")); + System.out.println("execute UpsertStatement: " + upsertSQL.orElse("use UsingInsertUpdateStatement")); + + return upsertSQL.map((Function) sql -> + new UpsertWriterUsingUpsertStatement( + fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, sql, metricOutputFormat)) + .orElseGet(() -> + new UpsertWriterUsingInsertUpdateStatement( + fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, + // 表存在检查 + dialect.getRowExistsStatement(tableName, keyFields), + dialect.getInsertIntoStatement(tableName, fieldNames), + dialect.getUpdateStatement(tableName, fieldNames, keyFields), + metricOutputFormat)); + } + + final int[] fieldTypes; + final int[] pkTypes; + private final int[] pkFields; + private final String deleteSQL; + private final boolean objectReuse; + + private transient Map> keyToRows; + private transient PreparedStatement deleteStatement; + // only use metric + private transient MetricOutputFormat metricOutputFormat; + + private UpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSQL, boolean objectReuse, MetricOutputFormat metricOutputFormat) { + this.fieldTypes = fieldTypes; + this.pkFields = pkFields; + this.pkTypes = pkTypes; + this.deleteSQL = deleteSQL; + this.objectReuse = objectReuse; + this.metricOutputFormat = metricOutputFormat; + } + + @Override + public void open(Connection connection) throws SQLException { + this.keyToRows = new HashMap<>(); + this.deleteStatement = connection.prepareStatement(deleteSQL); + } + + @Override + public void addRecord(Tuple2 record) throws SQLException { + // we don't need perform a deep copy, because jdbc field are immutable object. + Tuple2 tuple2 = objectReuse ? new Tuple2<>(record.f0, Row.copy(record.f1)) : record; + // add records to buffer + keyToRows.put(getPrimaryKey(tuple2.f1), tuple2); + } + + @Override + public void executeBatch(Connection connection) throws SQLException { + try { + if (keyToRows.size() > 0) { + for (Map.Entry> entry : keyToRows.entrySet()) { + Row pk = entry.getKey(); + Tuple2 tuple = entry.getValue(); + if (tuple.f0) { + processOneRowInBatch(pk, tuple.f1); + } else { + setRecordToStatement(deleteStatement, pkTypes, pk); + deleteStatement.addBatch(); + } + } + internalExecuteBatch(); + deleteStatement.executeBatch(); + connection.commit(); + keyToRows.clear(); + } + } catch (Exception e) { + // 清理批处理中的正确字段，防止重复写入 + connection.rollback(); + cleanBatchWhenError(); + executeUpdate(connection); + } + } + + @Override + public void executeUpdate(Connection connection) { + if (keyToRows.size() > 0) { + for (Map.Entry> entry : keyToRows.entrySet()) { + try { + Row pk = entry.getKey(); + Tuple2 tuple = entry.getValue(); + if (tuple.f0) { + processOneRowInBatch(pk, tuple.f1); + internalExecuteBatch(); + } else { + setRecordToStatement(deleteStatement, pkTypes, pk); + deleteStatement.executeUpdate(); + } + connection.commit(); + } catch (Exception e) { + metricOutputFormat.outDirtyRecords.inc(); + if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { + LOG.error("record insert failed ,this row is {}", entry.getValue()); + LOG.error("", e); + } + } + } + keyToRows.clear(); + } + } + + abstract void processOneRowInBatch(Row pk, Row row) throws SQLException; + + abstract void internalExecuteBatch() throws SQLException; + + @Override + public void close() throws SQLException { + if (deleteStatement != null) { + deleteStatement.close(); + deleteStatement = null; + } + } + + private Row getPrimaryKey(Row row) { + Row pks = new Row(pkFields.length); + for (int i = 0; i < pkFields.length; i++) { + pks.setField(i, row.getField(pkFields[i])); + } + return pks; + } + + // ---------------------------------------------------------------------------------------- + + private static final class UpsertWriterUsingUpsertStatement extends UpsertWriter { + + private static final long serialVersionUID = 1L; + private final String upsertSQL; + + private transient PreparedStatement upsertStatement; + + private UpsertWriterUsingUpsertStatement( + int[] fieldTypes, + int[] pkFields, + int[] pkTypes, + boolean objectReuse, + String deleteSQL, + String upsertSQL, + MetricOutputFormat metricOutputFormat) { + super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); + this.upsertSQL = upsertSQL; + } + + @Override + public void open(Connection connection) throws SQLException { + super.open(connection); + upsertStatement = connection.prepareStatement(upsertSQL); + } + + @Override + void processOneRowInBatch(Row pk, Row row) throws SQLException { + setRecordToStatement(upsertStatement, fieldTypes, row); + upsertStatement.addBatch(); + } + + @Override + public void cleanBatchWhenError() throws SQLException { + upsertStatement.clearBatch(); + upsertStatement.clearParameters(); + } + + @Override + void internalExecuteBatch() throws SQLException { + upsertStatement.executeBatch(); + } + + @Override + public void close() throws SQLException { + super.close(); + if (upsertStatement != null) { + upsertStatement.close(); + upsertStatement = null; + } + } + } + + private static final class UpsertWriterUsingInsertUpdateStatement extends UpsertWriter { + + private static final long serialVersionUID = 1L; + private final String existSQL; + private final String insertSQL; + private final String updateSQL; + + private transient PreparedStatement existStatement; + private transient PreparedStatement insertStatement; + private transient PreparedStatement updateStatement; + + private UpsertWriterUsingInsertUpdateStatement( + int[] fieldTypes, + int[] pkFields, + int[] pkTypes, + boolean objectReuse, + String deleteSQL, + String existSQL, + String insertSQL, + String updateSQL, + MetricOutputFormat metricOutputFormat) { + super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); + this.existSQL = existSQL; + this.insertSQL = insertSQL; + this.updateSQL = updateSQL; + } + + @Override + public void open(Connection connection) throws SQLException { + super.open(connection); + existStatement = connection.prepareStatement(existSQL); + insertStatement = connection.prepareStatement(insertSQL); + updateStatement = connection.prepareStatement(updateSQL); + } + + @Override + void processOneRowInBatch(Row pk, Row row) throws SQLException { + setRecordToStatement(existStatement, pkTypes, pk); + ResultSet resultSet = existStatement.executeQuery(); + boolean exist = resultSet.next(); + resultSet.close(); + if (exist) { + // do update + setRecordToStatement(updateStatement, fieldTypes, row); + updateStatement.addBatch(); + } else { + // do insert + setRecordToStatement(insertStatement, fieldTypes, row); + insertStatement.addBatch(); + } + } + + @Override + public void cleanBatchWhenError() throws SQLException { + updateStatement.clearBatch(); + insertStatement.clearBatch(); + } + + @Override + void internalExecuteBatch() throws SQLException { + updateStatement.executeBatch(); + insertStatement.executeBatch(); + } + + @Override + public void close() throws SQLException { + super.close(); + if (existStatement != null) { + existStatement.close(); + existStatement = null; + } + if (insertStatement != null) { + insertStatement.close(); + insertStatement = null; + } + if (updateStatement != null) { + updateStatement.close(); + updateStatement = null; + } + } + } +} diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 2fcd20dd8..a0a84b8de 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -47,14 +47,11 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map primaryKeysList = null; + String[] primaryKeysArray = new String[]{}; if (!StringUtils.isEmpty(primaryKeysStr)) { - String[] primaryKeysArray = primaryKeysStr.split(","); - primaryKeysList = new ArrayList(Arrays.asList(primaryKeysArray)); - } else { - primaryKeysList = new ArrayList<>(); + primaryKeysArray = primaryKeysStr.split(","); } - redisTableInfo.setPrimaryKeys(primaryKeysList); + redisTableInfo.setPrimaryKeys(primaryKeysArray); return redisTableInfo; } diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java index 2341bca66..82db4d318 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java @@ -20,8 +20,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.ExtendOutputFormat; -import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; -import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import java.util.*; From ebe21fbbe109d8aab275951fbecf284c7d27ea36 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 2 Jan 2020 17:28:16 +0800 Subject: [PATCH 003/523] add class --- .../flink/sql/sink/rdb/JDBCOptions.java | 179 ++++++++++++++++++ .../rdb/format/AbstractJDBCOutputFormat.java | 88 +++++++++ 2 files changed, 267 insertions(+) create mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java create mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java new file mode 100644 index 000000000..77d9f19d0 --- /dev/null +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.rdb; + + +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; + +import java.util.Objects; +import java.util.Optional; + +import static org.apache.flink.util.Preconditions.checkNotNull; + + +public class JDBCOptions { + + private String dbURL; + private String tableName; + private String driverName; + private String username; + private String password; + private String schema; + private JDBCDialect dialect; + + private JDBCOptions(String dbURL, String tableName, String driverName, String username, + String password, String schema, JDBCDialect dialect) { + this.dbURL = dbURL; + this.tableName = tableName; + this.driverName = driverName; + this.username = username; + this.password = password; + this.schema = schema; + this.dialect = dialect; + } + + public String getDbURL() { + return dbURL; + } + + public String getTableName() { + return tableName; + } + + public String getDriverName() { + return driverName; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public JDBCDialect getDialect() { + return dialect; + } + + public String getSchema() { + return schema; + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public boolean equals(Object o) { + if (o instanceof JDBCOptions) { + JDBCOptions options = (JDBCOptions) o; + return Objects.equals(dbURL, options.dbURL) && + Objects.equals(tableName, options.tableName) && + Objects.equals(driverName, options.driverName) && + Objects.equals(username, options.username) && + Objects.equals(password, options.password) && + Objects.equals(schema, options.schema) && + Objects.equals(dialect.getClass().getName(), options.dialect.getClass().getName()); + } else { + return false; + } + } + + /** + * Builder of {@link JDBCOptions}. + */ + public static class Builder { + private String dbURL; + private String tableName; + private String driverName; + private String username; + private String password; + private String scheam; + private JDBCDialect dialect; + + /** + * required, table name. + */ + public Builder setTableName(String tableName) { + this.tableName = tableName; + return this; + } + + /** + * optional, user name. + */ + public Builder setUsername(String username) { + this.username = username; + return this; + } + + /** + * optional, password. + */ + public Builder setPassword(String password) { + this.password = password; + return this; + } + + /** + * optional, driver name, dialect has a default driver name, + * See {@link JDBCDialect#defaultDriverName}. + */ + public Builder setDriverName(String driverName) { + this.driverName = driverName; + return this; + } + + /** + * optional, schema info + */ + public Builder setScheam(String scheam) { + this.scheam = scheam; + return this; + } + + /** + * required, JDBC DB url. + */ + public Builder setDBUrl(String dbURL) { + this.dbURL = dbURL; + return this; + } + + public Builder setDialect(JDBCDialect dialect) { + this.dialect = dialect; + return this; + } + + public JDBCOptions build() { + checkNotNull(dbURL, "No dbURL supplied."); + checkNotNull(tableName, "No tableName supplied."); + + if (this.driverName == null) { + Optional optional = dialect.defaultDriverName(); + this.driverName = optional.orElseGet(() -> { + throw new NullPointerException("No driverName supplied."); + }); + } + + return new JDBCOptions(dbURL, tableName, driverName, username, password, scheam, dialect); + } + } +} diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java new file mode 100644 index 000000000..c66fa9e72 --- /dev/null +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.rdb.format; + +import com.dtstack.flink.sql.sink.MetricOutputFormat; +import com.dtstack.flink.sql.util.JDBCUtils; +import org.apache.flink.api.common.io.RichOutputFormat; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.types.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +/** + * OutputFormat to write Rows into a JDBC database. + * + * @see Row + * @see DriverManager + */ +public abstract class AbstractJDBCOutputFormat extends MetricOutputFormat { + + private static final long serialVersionUID = 1L; + static final int DEFAULT_FLUSH_MAX_SIZE = 100; + static final long DEFAULT_FLUSH_INTERVAL_MILLS = 10000L; + static final boolean DEFAULT_ALLREPLACE_VALUE = false; + + private static final Logger LOG = LoggerFactory.getLogger(AbstractJDBCOutputFormat.class); + + private final String username; + private final String password; + private final String drivername; + protected final String dbURL; + + protected transient Connection connection; + + public AbstractJDBCOutputFormat(String username, String password, String drivername, String dbURL) { + this.username = username; + this.password = password; + this.drivername = drivername; + this.dbURL = dbURL; + } + + @Override + public void configure(Configuration parameters) { + } + + protected void establishConnection() throws SQLException, ClassNotFoundException { + JDBCUtils.forName(drivername, getClass().getClassLoader()); + if (username == null) { + connection = DriverManager.getConnection(dbURL); + } else { + connection = DriverManager.getConnection(dbURL, username, password); + } + connection.setAutoCommit(false); + } + + protected void closeDbConnection() throws IOException { + if (connection != null) { + try { + connection.close(); + } catch (SQLException se) { + LOG.warn("JDBC connection could not be closed: " + se.getMessage()); + } finally { + connection = null; + } + } + } +} From e902d317dd288be181c4f52d6d87e70edbc58723 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 3 Jan 2020 15:04:41 +0800 Subject: [PATCH 004/523] oracle upsert mode --- .../flink/sql/sink/mysql/MySQLDialect.java | 2 +- .../flink/sql/sink/mysql/MysqlSink.java | 2 - .../flink/sql/sink/oracle/OracleDialect.java | 102 +++++ .../flink/sql/sink/oracle/OracleSink.java | 396 +++++++++--------- .../sql/sink/rdb/dialect/JDBCDialect.java | 7 +- .../rdb/format/JDBCUpsertOutputFormat.java | 4 +- .../sql/sink/rdb/table/RdbTableInfo.java | 2 +- .../sql/sink/rdb/writer/UpsertWriter.java | 3 +- 8 files changed, 312 insertions(+), 206 deletions(-) create mode 100644 oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java index f7fe449b1..bc02ca403 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java @@ -56,7 +56,7 @@ public String quoteIdentifier(String identifier) { * @return */ @Override - public Optional getUpsertStatement(String tableName, String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + public Optional getUpsertStatement(String schema, String tableName, String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { return allReplace ? buildReplaceIntoStatement(tableName, fieldNames) : buildDuplicateUpsertStatement(tableName, fieldNames); } diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java index c2b774a64..269b7b258 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java @@ -35,8 +35,6 @@ public class MysqlSink extends RdbSink implements IStreamSinkGener { - private static final String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; - public MysqlSink() { super(new MySQLDialect()); } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java new file mode 100644 index 000000000..d51e33056 --- /dev/null +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.oracle; + +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import com.dtstack.flink.sql.util.DtStringUtil; +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Date: 2020/1/3 + * Company: www.dtstack.com + * @author maqi + */ +public class OracleDialect implements JDBCDialect { + + @Override + public boolean canHandle(String url) { + return url.startsWith("jdbc:oracle:"); + } + + @Override + public Optional defaultDriverName() { + return Optional.of("oracle.jdbc.driver.OracleDriver"); + } + + @Override + public Optional getUpsertStatement(String schema, String tableName, String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + tableName = DtStringUtil.getTableFullPath(schema, tableName); + StringBuilder sb = new StringBuilder(); + sb.append("MERGE INTO " + tableName + " T1 USING " + + "(" + buildDualQueryStatement(fieldNames) + ") T2 ON (" + + buildConnectionConditions(uniqueKeyFields) + ") "); + + String updateSql = buildUpdateConnection(fieldNames, uniqueKeyFields, allReplace); + + if (StringUtils.isNotEmpty(updateSql)) { + sb.append(" WHEN MATCHED THEN UPDATE SET "); + sb.append(updateSql); + } + + sb.append(" WHEN NOT MATCHED THEN " + + "INSERT (" + Arrays.stream(fieldNames).map(this::quoteIdentifier).collect(Collectors.joining(",")) + ") VALUES (" + + Arrays.stream(fieldNames).map(col -> "T2." + quoteIdentifier(col)).collect(Collectors.joining(",")) + ")"); + + return Optional.of(sb.toString()); + } + + /** + * build T1."A"=T2."A" or T1."A"=nvl(T2."A",T1."A") + * @param fieldNames + * @param uniqueKeyFields + * @param allReplace + * @return + */ + private String buildUpdateConnection(String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + List uniqueKeyList = Arrays.asList(uniqueKeyFields); + return Arrays.stream(fieldNames).filter(col -> !uniqueKeyList.contains(col)).map(col -> { + return allReplace ? quoteIdentifier("T1") + "." + quoteIdentifier(col) + " = " + quoteIdentifier("T2") + "." + quoteIdentifier(col) : + quoteIdentifier("T1") + "." + quoteIdentifier(col) + " =nvl(" + quoteIdentifier("T2") + "." + quoteIdentifier(col) + "," + + quoteIdentifier("T1") + "." + quoteIdentifier(col) + ")"; + }).collect(Collectors.joining(",")); + } + + + private String buildConnectionConditions(String[] uniqueKeyFields) { + return Arrays.stream(uniqueKeyFields).map(col -> "T1." + quoteIdentifier(col) + "=T2." + quoteIdentifier(col)).collect(Collectors.joining(",")); + } + + /** + * build select sql , such as (SELECT ? "A",? "B" FROM DUAL) + * + * @param column destination column + * @return + */ + public String buildDualQueryStatement(String[] column) { + StringBuilder sb = new StringBuilder("SELECT "); + String collect = Arrays.stream(column).map(col -> " ? " + quoteIdentifier(col)).collect(Collectors.joining(", ")); + sb.append(collect).append(" FROM DUAL"); + return sb.toString(); + } +} diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index b6a68c4e5..ce25f9660 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -18,16 +18,10 @@ package com.dtstack.flink.sql.sink.oracle; import com.dtstack.flink.sql.sink.IStreamSinkGener; +import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.RdbSink; -import com.dtstack.flink.sql.sink.rdb.format.ExtendOutputFormat; -import com.dtstack.flink.sql.util.DtStringUtil; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** * Reason: @@ -37,198 +31,212 @@ * @author maqi */ public class OracleSink extends RdbSink implements IStreamSinkGener { - private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; - - @Override - public String getDriverName() { - return ORACLE_DRIVER; - } - - @Override - public RetractJDBCOutputFormat getOutputFormat() { - return new ExtendOutputFormat(); - } - - @Override - public void buildSql(String scheam, String tableName, List fields) { - buildInsertSql(scheam, tableName, fields); - } - - private void buildInsertSql(String scheam, String tableName, List fields) { - tableName = DtStringUtil.getTableFullPath(scheam, tableName); - String columns = fields.stream() - .map(this::quoteIdentifier) - .collect(Collectors.joining(", ")); - String placeholders = fields.stream() - .map(f -> "?") - .collect(Collectors.joining(", ")); - this.sql = "INSERT INTO " + tableName + "(" + columns + ")" + " VALUES (" + placeholders + ")"; + public OracleSink() { + super(new OracleDialect()); } - /** - * use MERGE INTO build oracle replace into sql - * @param tableName - * @param fieldNames create table contained column columns - * @param realIndexes - * @param fullField real columns , query from db - * @return - */ @Override - public String buildUpdateSql(String scheam, String tableName, List fieldNames, Map> realIndexes, List fullField) { - tableName = DtStringUtil.getTableFullPath(scheam, tableName); - - StringBuilder sb = new StringBuilder(); - - sb.append("MERGE INTO " + tableName + " T1 USING " - + "(" + makeValues(fieldNames) + ") T2 ON (" - + updateKeySql(realIndexes) + ") "); - - - String updateSql1 = buildUpdateSqlForAllValue(fieldNames, fullField, "T1", "T2", keyColList(realIndexes)); - String updateSql = buildUpdateSqlForNotnullValue(fieldNames, fullField, "T1", "T2", keyColList(realIndexes)); - - if (StringUtils.isNotEmpty(updateSql)) { - sb.append(" WHEN MATCHED THEN UPDATE SET "); - sb.append(updateSql); - } - - sb.append(" WHEN NOT MATCHED THEN " - + "INSERT (" + quoteColumns(fieldNames) + ") VALUES (" - + quoteColumns(fieldNames, "T2") + ")"); - - return sb.toString(); + public JDBCUpsertOutputFormat getOutputFormat() { + JDBCOptions jdbcOptions = JDBCOptions.builder() + .setDBUrl(dbURL).setDialect(jdbcDialect) + .setUsername(userName).setPassword(password) + .setTableName(tableName).setScheam(schema).build(); + + return JDBCUpsertOutputFormat.builder() + .setOptions(jdbcOptions) + .setFieldNames(fieldNames) + .setFlushMaxSize(batchNum) + .setFlushIntervalMills(batchWaitInterval) + .setFieldTypes(sqlTypes) + .setKeyFields(primaryKeys) + .setAllReplace(allReplace) + .setUpdateMode(updateMode).build(); } - public String quoteColumns(List column) { - return quoteColumns(column, null); - } + // @Override +// public RetractJDBCOutputFormat getOutputFormat() { +// return new ExtendOutputFormat(); +// } +// +// @Override +// public void buildSql(String scheam, String tableName, List fields) { +// buildInsertSql(scheam, tableName, fields); +// } +// +// private void buildInsertSql(String scheam, String tableName, List fields) { +// tableName = DtStringUtil.getTableFullPath(scheam, tableName); +// String columns = fields.stream() +// .map(this::quoteIdentifier) +// .collect(Collectors.joining(", ")); +// +// String placeholders = fields.stream() +// .map(f -> "?") +// .collect(Collectors.joining(", ")); +// this.sql = "INSERT INTO " + tableName + "(" + columns + ")" + " VALUES (" + placeholders + ")"; +// } +// +// /** +// * use MERGE INTO build oracle replace into sql +// * @param tableName +// * @param fieldNames create table contained column columns +// * @param realIndexes +// * @param fullField real columns , query from db +// * @return +// */ +// @Override +// public String buildUpdateSql(String scheam, String tableName, List fieldNames, Map> realIndexes, List fullField) { +// tableName = DtStringUtil.getTableFullPath(scheam, tableName); +// +// StringBuilder sb = new StringBuilder(); +// +// sb.append("MERGE INTO " + tableName + " T1 USING " +// + "(" + makeValues(fieldNames) + ") T2 ON (" +// + updateKeySql(realIndexes) + ") "); +// +// +// String updateSql1 = buildUpdateSqlForAllValue(fieldNames, fullField, "T1", "T2", keyColList(realIndexes)); +// String updateSql = buildUpdateSqlForNotnullValue(fieldNames, fullField, "T1", "T2", keyColList(realIndexes)); +// +// if (StringUtils.isNotEmpty(updateSql)) { +// sb.append(" WHEN MATCHED THEN UPDATE SET "); +// sb.append(updateSql); +// } +// +// sb.append(" WHEN NOT MATCHED THEN " +// + "INSERT (" + quoteColumns(fieldNames) + ") VALUES (" +// + quoteColumns(fieldNames, "T2") + ")"); +// +// return sb.toString(); +// } +// +// +// public String quoteColumns(List column) { +// return quoteColumns(column, null); +// } +// +// public String quoteColumns(List column, String table) { +// String prefix = StringUtils.isBlank(table) ? "" : DtStringUtil.addQuoteForStr(table) + "."; +// List list = new ArrayList<>(); +// for (String col : column) { +// list.add(prefix + DtStringUtil.addQuoteForStr(col)); +// } +// return StringUtils.join(list, ","); +// } +// +// /** +// * extract all distinct index column +// * @param realIndexes +// * @return +// */ +// protected List keyColList(Map> realIndexes) { +// List keyCols = new ArrayList<>(); +// for (Map.Entry> entry : realIndexes.entrySet()) { +// List list = entry.getValue(); +// for (String col : list) { +// if (!containsIgnoreCase(keyCols,col)) { +// keyCols.add(col); +// } +// } +// } +// return keyCols; +// } +// +// /** +// * build update sql , such as UPDATE SET "T1".A="T2".A +// * @param updateColumn create table contained column columns +// * @param fullColumn real columns , query from db +// * @param leftTable alias +// * @param rightTable alias +// * @param indexCols index column +// * @return +// */ +// public String buildUpdateSqlForAllValue(List updateColumn, List fullColumn, String leftTable, String rightTable, List indexCols) { +// String prefixLeft = StringUtils.isBlank(leftTable) ? "" : DtStringUtil.addQuoteForStr(leftTable) + "."; +// String prefixRight = StringUtils.isBlank(rightTable) ? "" : DtStringUtil.addQuoteForStr(rightTable) + "."; +// +// String sql = fullColumn.stream().filter(col -> { +// return !(indexCols == null || indexCols.size() == 0 || containsIgnoreCase(indexCols, col)); +// }).map(col -> { +// String leftCol = prefixLeft + DtStringUtil.addQuoteForStr(col); +// String rightCol = prefixRight + DtStringUtil.addQuoteForStr(col); +// +// if (containsIgnoreCase(updateColumn, col)) { +// return (leftCol + "=" + rightCol); +// } else { +// return (leftCol + "=null"); +// } +// }).collect(Collectors.joining(",")); +// +// return sql; +// } +// +// public String buildUpdateSqlForNotnullValue(List updateColumn, List fullColumn, String leftTable, String rightTable, List indexCols) { +// String prefixLeft = StringUtils.isBlank(leftTable) ? "" : DtStringUtil.addQuoteForStr(leftTable) + "."; +// String prefixRight = StringUtils.isBlank(rightTable) ? "" : DtStringUtil.addQuoteForStr(rightTable) + "."; +// +// String sql = fullColumn.stream().filter(col -> { +// return !(indexCols == null || indexCols.size() == 0 || containsIgnoreCase(indexCols, col)); +// }).map(col -> { +// String leftCol = prefixLeft + DtStringUtil.addQuoteForStr(col); +// String rightCol = prefixRight + DtStringUtil.addQuoteForStr(col); +// +// if (containsIgnoreCase(updateColumn, col)) { +// return leftCol + "= nvl(" + rightCol + "," + leftCol + ")"; +// } +// return ""; +// }).collect(Collectors.joining(",")); +// +// return sql; +// } +// +// +// +// +// /** +// * build connect sql by index column, such as T1."A"=T2."A" +// * @param updateKey +// * @return +// */ +// public String updateKeySql(Map> updateKey) { +// List exprList = new ArrayList<>(); +// for (Map.Entry> entry : updateKey.entrySet()) { +// List colList = new ArrayList<>(); +// for (String col : entry.getValue()) { +// colList.add("T1." + DtStringUtil.addQuoteForStr(col) + "=T2." + DtStringUtil.addQuoteForStr(col)); +// } +// exprList.add(StringUtils.join(colList, " AND ")); +// } +// return StringUtils.join(exprList, " OR "); +// } +// +// /** +// * build select sql , such as (SELECT ? "A",? "B" FROM DUAL) +// * +// * @param column destination column +// * @return +// */ +// public String makeValues(List column) { +// StringBuilder sb = new StringBuilder("SELECT "); +// for (int i = 0; i < column.size(); ++i) { +// if (i != 0) { +// sb.append(","); +// } +// sb.append("? " + DtStringUtil.addQuoteForStr(column.get(i))); +// } +// sb.append(" FROM DUAL"); +// return sb.toString(); +// } +// +// public boolean containsIgnoreCase(List l, String s) { +// Iterator it = l.iterator(); +// while (it.hasNext()) { +// if (it.next().equalsIgnoreCase(s)) +// return true; +// } +// return false; +// } - public String quoteColumns(List column, String table) { - String prefix = StringUtils.isBlank(table) ? "" : DtStringUtil.addQuoteForStr(table) + "."; - List list = new ArrayList<>(); - for (String col : column) { - list.add(prefix + DtStringUtil.addQuoteForStr(col)); - } - return StringUtils.join(list, ","); - } - - /** - * extract all distinct index column - * @param realIndexes - * @return - */ - protected List keyColList(Map> realIndexes) { - List keyCols = new ArrayList<>(); - for (Map.Entry> entry : realIndexes.entrySet()) { - List list = entry.getValue(); - for (String col : list) { - if (!containsIgnoreCase(keyCols,col)) { - keyCols.add(col); - } - } - } - return keyCols; - } - - /** - * build update sql , such as UPDATE SET "T1".A="T2".A - * @param updateColumn create table contained column columns - * @param fullColumn real columns , query from db - * @param leftTable alias - * @param rightTable alias - * @param indexCols index column - * @return - */ - public String buildUpdateSqlForAllValue(List updateColumn, List fullColumn, String leftTable, String rightTable, List indexCols) { - String prefixLeft = StringUtils.isBlank(leftTable) ? "" : DtStringUtil.addQuoteForStr(leftTable) + "."; - String prefixRight = StringUtils.isBlank(rightTable) ? "" : DtStringUtil.addQuoteForStr(rightTable) + "."; - - String sql = fullColumn.stream().filter(col -> { - return !(indexCols == null || indexCols.size() == 0 || containsIgnoreCase(indexCols, col)); - }).map(col -> { - String leftCol = prefixLeft + DtStringUtil.addQuoteForStr(col); - String rightCol = prefixRight + DtStringUtil.addQuoteForStr(col); - - if (containsIgnoreCase(updateColumn, col)) { - return (leftCol + "=" + rightCol); - } else { - return (leftCol + "=null"); - } - }).collect(Collectors.joining(",")); - - return sql; - } - - public String buildUpdateSqlForNotnullValue(List updateColumn, List fullColumn, String leftTable, String rightTable, List indexCols) { - String prefixLeft = StringUtils.isBlank(leftTable) ? "" : DtStringUtil.addQuoteForStr(leftTable) + "."; - String prefixRight = StringUtils.isBlank(rightTable) ? "" : DtStringUtil.addQuoteForStr(rightTable) + "."; - - String sql = fullColumn.stream().filter(col -> { - return !(indexCols == null || indexCols.size() == 0 || containsIgnoreCase(indexCols, col)); - }).map(col -> { - String leftCol = prefixLeft + DtStringUtil.addQuoteForStr(col); - String rightCol = prefixRight + DtStringUtil.addQuoteForStr(col); - - if (containsIgnoreCase(updateColumn, col)) { - return leftCol + "= nvl(" + rightCol + "," + leftCol + ")"; - } - return ""; - }).collect(Collectors.joining(",")); - - return sql; - } - - - - - /** - * build connect sql by index column, such as T1."A"=T2."A" - * @param updateKey - * @return - */ - public String updateKeySql(Map> updateKey) { - List exprList = new ArrayList<>(); - for (Map.Entry> entry : updateKey.entrySet()) { - List colList = new ArrayList<>(); - for (String col : entry.getValue()) { - colList.add("T1." + DtStringUtil.addQuoteForStr(col) + "=T2." + DtStringUtil.addQuoteForStr(col)); - } - exprList.add(StringUtils.join(colList, " AND ")); - } - return StringUtils.join(exprList, " OR "); - } - - /** - * build select sql , such as (SELECT ? "A",? "B" FROM DUAL) - * - * @param column destination column - * @return - */ - public String makeValues(List column) { - StringBuilder sb = new StringBuilder("SELECT "); - for (int i = 0; i < column.size(); ++i) { - if (i != 0) { - sb.append(","); - } - sb.append("? " + DtStringUtil.addQuoteForStr(column.get(i))); - } - sb.append(" FROM DUAL"); - return sb.toString(); - } - - public boolean containsIgnoreCase(List l, String s) { - Iterator it = l.iterator(); - while (it.hasNext()) { - if (it.next().equalsIgnoreCase(s)) - return true; - } - return false; - } - - public String quoteIdentifier(String identifier) { - return "\"" + identifier + "\""; - } } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java index 111c4c8dd..d635a7a06 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java @@ -62,14 +62,9 @@ default String quoteIdentifier(String identifier) { * @return None if dialect does not support upsert statement, the writer will degrade to * the use of select + update/insert, this performance is poor. * - * @param tableName - * @param fieldNames - * @param uniqueKeyFields - * @param allReplace - * @return */ default Optional getUpsertStatement( - String tableName, String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + String schema, String tableName, String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { return Optional.empty(); } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index 239d76f47..b2c51d18d 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -52,6 +52,7 @@ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat 0, "updateMode mode primary is required"); } Arrays.stream(getPrimaryKeys()).forEach(pk -> { diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java index 8ce2148d7..20c82c910 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java @@ -50,6 +50,7 @@ public abstract class UpsertWriter implements JDBCWriter { public static UpsertWriter create( JDBCDialect dialect, + String schema, String tableName, String[] fieldNames, int[] fieldTypes, @@ -69,7 +70,7 @@ public static UpsertWriter create( LOG.info("deleteSQL is :{}", deleteSQL); System.out.println("deleteSQL is :" + deleteSQL); - Optional upsertSQL = dialect.getUpsertStatement(tableName, fieldNames, keyFields, allReplace); + Optional upsertSQL = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); LOG.info("execute UpsertStatement: {}", upsertSQL.orElse("use UsingInsertUpdateStatement")); System.out.println("execute UpsertStatement: " + upsertSQL.orElse("use UsingInsertUpdateStatement")); From d66acefdb7c1bd16a69347379d72dfce1af1b8b9 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 3 Jan 2020 20:50:32 +0800 Subject: [PATCH 005/523] pg sink --- docs/postgresqlSink.md | 4 +- .../flink/sql/sink/mysql/MySQLDialect.java | 2 +- .../flink/sql/sink/oracle/OracleSink.java | 187 ------------------ pom.xml | 6 +- .../sink/postgresql/PostgresqlDialect.java | 62 ++++++ .../sql/sink/postgresql/PostgresqlSink.java | 93 ++------- .../table/PostgresqlSinkParser.java | 24 +-- .../postgresql/table/PostgresqlTableInfo.java | 76 ------- .../sql/sink/rdb/dialect/JDBCDialect.java | 16 +- .../rdb/format/JDBCUpsertOutputFormat.java | 2 +- .../sql/sink/rdb/table/RdbTableInfo.java | 9 +- .../sql/sink/rdb/writer/AppendOnlyWriter.java | 15 +- .../sql/sink/rdb/writer/UpsertWriter.java | 11 +- 13 files changed, 122 insertions(+), 385 deletions(-) create mode 100644 postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java delete mode 100644 postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlTableInfo.java diff --git a/docs/postgresqlSink.md b/docs/postgresqlSink.md index da09c4c34..7fa7cab2f 100644 --- a/docs/postgresqlSink.md +++ b/docs/postgresqlSink.md @@ -16,7 +16,7 @@ CREATE TABLE tableName( ``` ## 2.支持版本 - postgresql-8.2+ + postgresql-9.5+ ## 3.表结构定义 @@ -36,8 +36,6 @@ CREATE TABLE tableName( | password | postgresql连接密码|是|| | tableName | postgresqll表名称|是|| | parallelism | 并行度设置|否|1| -| isUpsert | 使用upsert模式插入数据(版本9.5之后才支持upsert) |否|false -| keyField | 设置更新主键字段名（isupsert为true时为必填项）|否| ## 5.样例： ``` diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java index bc02ca403..9f53cd328 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java @@ -63,7 +63,7 @@ public Optional getUpsertStatement(String schema, String tableName, Stri public Optional buildDuplicateUpsertStatement(String tableName, String[] fieldNames) { String updateClause = Arrays.stream(fieldNames).map(f -> quoteIdentifier(f) + "=IFNULL(VALUES(" + quoteIdentifier(f) + ")," + quoteIdentifier(f) + ")") .collect(Collectors.joining(", ")); - return Optional.of(getInsertIntoStatement(tableName, fieldNames) + + return Optional.of(getInsertIntoStatement("", tableName, fieldNames) + " ON DUPLICATE KEY UPDATE " + updateClause ); } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index ce25f9660..176c46a23 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -20,7 +20,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.RdbSink; -import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** @@ -53,190 +52,4 @@ public JDBCUpsertOutputFormat getOutputFormat() { .setAllReplace(allReplace) .setUpdateMode(updateMode).build(); } - - - // @Override -// public RetractJDBCOutputFormat getOutputFormat() { -// return new ExtendOutputFormat(); -// } -// -// @Override -// public void buildSql(String scheam, String tableName, List fields) { -// buildInsertSql(scheam, tableName, fields); -// } -// -// private void buildInsertSql(String scheam, String tableName, List fields) { -// tableName = DtStringUtil.getTableFullPath(scheam, tableName); -// String columns = fields.stream() -// .map(this::quoteIdentifier) -// .collect(Collectors.joining(", ")); -// -// String placeholders = fields.stream() -// .map(f -> "?") -// .collect(Collectors.joining(", ")); -// this.sql = "INSERT INTO " + tableName + "(" + columns + ")" + " VALUES (" + placeholders + ")"; -// } -// -// /** -// * use MERGE INTO build oracle replace into sql -// * @param tableName -// * @param fieldNames create table contained column columns -// * @param realIndexes -// * @param fullField real columns , query from db -// * @return -// */ -// @Override -// public String buildUpdateSql(String scheam, String tableName, List fieldNames, Map> realIndexes, List fullField) { -// tableName = DtStringUtil.getTableFullPath(scheam, tableName); -// -// StringBuilder sb = new StringBuilder(); -// -// sb.append("MERGE INTO " + tableName + " T1 USING " -// + "(" + makeValues(fieldNames) + ") T2 ON (" -// + updateKeySql(realIndexes) + ") "); -// -// -// String updateSql1 = buildUpdateSqlForAllValue(fieldNames, fullField, "T1", "T2", keyColList(realIndexes)); -// String updateSql = buildUpdateSqlForNotnullValue(fieldNames, fullField, "T1", "T2", keyColList(realIndexes)); -// -// if (StringUtils.isNotEmpty(updateSql)) { -// sb.append(" WHEN MATCHED THEN UPDATE SET "); -// sb.append(updateSql); -// } -// -// sb.append(" WHEN NOT MATCHED THEN " -// + "INSERT (" + quoteColumns(fieldNames) + ") VALUES (" -// + quoteColumns(fieldNames, "T2") + ")"); -// -// return sb.toString(); -// } -// -// -// public String quoteColumns(List column) { -// return quoteColumns(column, null); -// } -// -// public String quoteColumns(List column, String table) { -// String prefix = StringUtils.isBlank(table) ? "" : DtStringUtil.addQuoteForStr(table) + "."; -// List list = new ArrayList<>(); -// for (String col : column) { -// list.add(prefix + DtStringUtil.addQuoteForStr(col)); -// } -// return StringUtils.join(list, ","); -// } -// -// /** -// * extract all distinct index column -// * @param realIndexes -// * @return -// */ -// protected List keyColList(Map> realIndexes) { -// List keyCols = new ArrayList<>(); -// for (Map.Entry> entry : realIndexes.entrySet()) { -// List list = entry.getValue(); -// for (String col : list) { -// if (!containsIgnoreCase(keyCols,col)) { -// keyCols.add(col); -// } -// } -// } -// return keyCols; -// } -// -// /** -// * build update sql , such as UPDATE SET "T1".A="T2".A -// * @param updateColumn create table contained column columns -// * @param fullColumn real columns , query from db -// * @param leftTable alias -// * @param rightTable alias -// * @param indexCols index column -// * @return -// */ -// public String buildUpdateSqlForAllValue(List updateColumn, List fullColumn, String leftTable, String rightTable, List indexCols) { -// String prefixLeft = StringUtils.isBlank(leftTable) ? "" : DtStringUtil.addQuoteForStr(leftTable) + "."; -// String prefixRight = StringUtils.isBlank(rightTable) ? "" : DtStringUtil.addQuoteForStr(rightTable) + "."; -// -// String sql = fullColumn.stream().filter(col -> { -// return !(indexCols == null || indexCols.size() == 0 || containsIgnoreCase(indexCols, col)); -// }).map(col -> { -// String leftCol = prefixLeft + DtStringUtil.addQuoteForStr(col); -// String rightCol = prefixRight + DtStringUtil.addQuoteForStr(col); -// -// if (containsIgnoreCase(updateColumn, col)) { -// return (leftCol + "=" + rightCol); -// } else { -// return (leftCol + "=null"); -// } -// }).collect(Collectors.joining(",")); -// -// return sql; -// } -// -// public String buildUpdateSqlForNotnullValue(List updateColumn, List fullColumn, String leftTable, String rightTable, List indexCols) { -// String prefixLeft = StringUtils.isBlank(leftTable) ? "" : DtStringUtil.addQuoteForStr(leftTable) + "."; -// String prefixRight = StringUtils.isBlank(rightTable) ? "" : DtStringUtil.addQuoteForStr(rightTable) + "."; -// -// String sql = fullColumn.stream().filter(col -> { -// return !(indexCols == null || indexCols.size() == 0 || containsIgnoreCase(indexCols, col)); -// }).map(col -> { -// String leftCol = prefixLeft + DtStringUtil.addQuoteForStr(col); -// String rightCol = prefixRight + DtStringUtil.addQuoteForStr(col); -// -// if (containsIgnoreCase(updateColumn, col)) { -// return leftCol + "= nvl(" + rightCol + "," + leftCol + ")"; -// } -// return ""; -// }).collect(Collectors.joining(",")); -// -// return sql; -// } -// -// -// -// -// /** -// * build connect sql by index column, such as T1."A"=T2."A" -// * @param updateKey -// * @return -// */ -// public String updateKeySql(Map> updateKey) { -// List exprList = new ArrayList<>(); -// for (Map.Entry> entry : updateKey.entrySet()) { -// List colList = new ArrayList<>(); -// for (String col : entry.getValue()) { -// colList.add("T1." + DtStringUtil.addQuoteForStr(col) + "=T2." + DtStringUtil.addQuoteForStr(col)); -// } -// exprList.add(StringUtils.join(colList, " AND ")); -// } -// return StringUtils.join(exprList, " OR "); -// } -// -// /** -// * build select sql , such as (SELECT ? "A",? "B" FROM DUAL) -// * -// * @param column destination column -// * @return -// */ -// public String makeValues(List column) { -// StringBuilder sb = new StringBuilder("SELECT "); -// for (int i = 0; i < column.size(); ++i) { -// if (i != 0) { -// sb.append(","); -// } -// sb.append("? " + DtStringUtil.addQuoteForStr(column.get(i))); -// } -// sb.append(" FROM DUAL"); -// return sb.toString(); -// } -// -// public boolean containsIgnoreCase(List l, String s) { -// Iterator it = l.iterator(); -// while (it.hasNext()) { -// if (it.next().equalsIgnoreCase(s)) -// return true; -// } -// return false; -// } - - } diff --git a/pom.xml b/pom.xml index e77bef17b..4cb63b0a1 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ - mysql + @@ -22,10 +22,10 @@ rdb - + oracle - + postgresql diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java new file mode 100644 index 000000000..212bb2df7 --- /dev/null +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.postgresql; + + +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import com.dtstack.flink.sql.util.DtStringUtil; + +import java.util.Arrays; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Date: 2020/1/3 + * Company: www.dtstack.com + * @author maqi + */ +public class PostgresqlDialect implements JDBCDialect { + @Override + public boolean canHandle(String url) { + return url.startsWith("jdbc:postgresql:"); + } + + @Override + public Optional defaultDriverName() { + return Optional.of("org.postgresql.Driver"); + } + + @Override + public Optional getUpsertStatement(String schema, String tableName, String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + String uniqueColumns = Arrays.stream(uniqueKeyFields) + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")); + + String updateClause = Arrays.stream(fieldNames) + .map(f -> quoteIdentifier(f) + "=EXCLUDED." + quoteIdentifier(f)) + .collect(Collectors.joining(", ")); + + return Optional.of(getInsertIntoStatement(schema, tableName, fieldNames) + + " ON CONFLICT (" + uniqueColumns + ")" + + " DO UPDATE SET " + updateClause + ); + + } + +} diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java index ac6e2e4ea..65379ed19 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java @@ -21,90 +21,33 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.postgresql.table.PostgresqlTableInfo; +import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.RdbSink; -import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.commons.lang3.StringUtils; - -import java.util.List; -import java.util.Map; +import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** - * Date: 2019-08-22 - * Company: mmg * - * @author tcm */ - public class PostgresqlSink extends RdbSink implements IStreamSinkGener { - - private static final String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - - private boolean isUpsert; - - private String keyField; - public PostgresqlSink() { + super(new PostgresqlDialect()); } @Override - public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { - PostgresqlTableInfo pgTableInfo = (PostgresqlTableInfo) targetTableInfo; - this.isUpsert = pgTableInfo.isUpsert(); - this.keyField = pgTableInfo.getKeyField(); - super.genStreamSink(targetTableInfo); - return this; - } - - @Override - public RetractJDBCOutputFormat getOutputFormat() { - return new RetractJDBCOutputFormat(); - } - - @Override - public void buildSql(String scheam, String tableName, List fields) { - buildInsertSql(tableName, fields); + public JDBCUpsertOutputFormat getOutputFormat() { + JDBCOptions jdbcOptions = JDBCOptions.builder() + .setDBUrl(dbURL).setDialect(jdbcDialect) + .setUsername(userName).setPassword(password) + .setTableName(tableName).setScheam(schema).build(); + + return JDBCUpsertOutputFormat.builder() + .setOptions(jdbcOptions) + .setFieldNames(fieldNames) + .setFlushMaxSize(batchNum) + .setFlushIntervalMills(batchWaitInterval) + .setFieldTypes(sqlTypes) + .setKeyFields(primaryKeys) + .setAllReplace(allReplace) + .setUpdateMode(updateMode).build(); } - - @Override - public String buildUpdateSql(String schema, String tableName, List fieldNames, Map> realIndexes, List fullField) { - return null; - } - - private void buildInsertSql(String tableName, List fields) { - StringBuffer sqlBuffer = new StringBuffer(); - - sqlBuffer.append("insert into ".concat(tableName) - .concat(" (") - .concat(StringUtils.join(fields, ",")) - .concat(") ") - ); - sqlBuffer.append("values ("); - StringBuffer upsertFields = new StringBuffer(); - for (String fieldName : fields) { - sqlBuffer.append("?,"); - if (this.isUpsert) { - if (fieldName.equals(this.keyField)) { - continue; - } - upsertFields.append(String.format("%s=excluded.%s,", fieldName, fieldName)); - } - } - sqlBuffer.deleteCharAt(sqlBuffer.length() - 1); - sqlBuffer.append(")"); - - if (this.isUpsert) { - upsertFields.deleteCharAt(upsertFields.length() - 1); - sqlBuffer.append(" ON conflict(".concat(keyField).concat(")")); - sqlBuffer.append(" DO UPDATE SET "); - sqlBuffer.append(upsertFields); - } - this.sql = sqlBuffer.toString(); - } - - @Override - public String getDriverName() { - return POSTGRESQL_DRIVER; - } - } diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java index f773b5a5c..5943b5e52 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java @@ -21,8 +21,6 @@ import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.util.MathUtil; -import org.apache.commons.lang3.StringUtils; import java.util.Map; @@ -38,26 +36,8 @@ public class PostgresqlSinkParser extends RdbSinkParser { @Override public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - PostgresqlTableInfo pgTableInfo = new PostgresqlTableInfo(); - pgTableInfo.setName(tableName); - parseFieldsInfo(fieldsInfo, pgTableInfo); - - pgTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(PostgresqlTableInfo.PARALLELISM_KEY.toLowerCase()))); - pgTableInfo.setUrl(MathUtil.getString(props.get(PostgresqlTableInfo.URL_KEY.toLowerCase()))); - pgTableInfo.setTableName(MathUtil.getString(props.get(PostgresqlTableInfo.TABLE_NAME_KEY.toLowerCase()))); - pgTableInfo.setUserName(MathUtil.getString(props.get(PostgresqlTableInfo.USER_NAME_KEY.toLowerCase()))); - pgTableInfo.setPassword(MathUtil.getString(props.get(PostgresqlTableInfo.PASSWORD_KEY.toLowerCase()))); - pgTableInfo.setBatchSize(MathUtil.getIntegerVal(props.get(PostgresqlTableInfo.BATCH_SIZE_KEY.toLowerCase()))); - pgTableInfo.setBatchWaitInterval(MathUtil.getLongVal(props.get(PostgresqlTableInfo.BATCH_WAIT_INTERVAL_KEY.toLowerCase()))); - pgTableInfo.setBufferSize(MathUtil.getString(props.get(PostgresqlTableInfo.BUFFER_SIZE_KEY.toLowerCase()))); - pgTableInfo.setFlushIntervalMs(MathUtil.getString(props.get(PostgresqlTableInfo.FLUSH_INTERVALMS_KEY.toLowerCase()))); - - pgTableInfo.setKeyField(MathUtil.getString(props.get(PostgresqlTableInfo.TABLE_KEY_FIELD.toLowerCase()))); - - String isUpsertStr = (String) props.get(PostgresqlTableInfo.TABLE_IS_UPSERT.toLowerCase()); - pgTableInfo.setUpsert(!StringUtils.isEmpty(isUpsertStr) && isUpsertStr.equals("true") ? true : false); - - pgTableInfo.check(); + TableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + pgTableInfo.setType(CURR_TYPE); return pgTableInfo; } } diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlTableInfo.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlTableInfo.java deleted file mode 100644 index 78df0de14..000000000 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlTableInfo.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.sink.postgresql.table; - -import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; -import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions; - -/** - * Date: 2019-08-22 - * Company: mmg - * - * @author tcm - */ - -public class PostgresqlTableInfo extends RdbTableInfo { - - public static final String TABLE_IS_UPSERT = "isUpsert"; - - public static final String TABLE_KEY_FIELD = "keyField"; - - private static final String CURR_TYPE = "postgresql"; - - private boolean isUpsert; - - private String keyField; - - - public PostgresqlTableInfo() { - setType(CURR_TYPE); - } - - public boolean isUpsert() { - return isUpsert; - } - - public void setUpsert(boolean upsert) { - isUpsert = upsert; - } - - public String getKeyField() { - return keyField; - } - - public void setKeyField(String keyField) { - this.keyField = keyField; - } - - @Override - public boolean check() { - Preconditions.checkNotNull(getUrl(), "postgresql field of URL is required"); - Preconditions.checkNotNull(getTableName(), "postgresql field of tableName is required"); - Preconditions.checkNotNull(getUserName(), "postgresql field of userName is required"); - Preconditions.checkNotNull(getPassword(), "postgresql field of password is required"); - if (isUpsert()) { - Preconditions.checkNotNull(getKeyField(), "postgresql field of keyField is required"); - } - return true; - } -} diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java index d635a7a06..2655eb82b 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java @@ -18,6 +18,8 @@ package com.dtstack.flink.sql.sink.rdb.dialect; +import org.apache.commons.lang3.StringUtils; + import java.io.Serializable; import java.util.Arrays; import java.util.Optional; @@ -81,14 +83,15 @@ default String getRowExistsStatement(String tableName, String[] conditionFields) /** * Get insert into statement. */ - default String getInsertIntoStatement(String tableName, String[] fieldNames) { + default String getInsertIntoStatement(String schema, String tableName, String[] fieldNames) { + String schemaInfo = StringUtils.isEmpty(schema) ? "" : quoteIdentifier(schema) + "."; String columns = Arrays.stream(fieldNames) .map(this::quoteIdentifier) .collect(Collectors.joining(", ")); String placeholders = Arrays.stream(fieldNames) .map(f -> "?") .collect(Collectors.joining(", ")); - return "INSERT INTO " + quoteIdentifier(tableName) + + return "INSERT INTO " + schemaInfo + quoteIdentifier(tableName) + "(" + columns + ")" + " VALUES (" + placeholders + ")"; } @@ -108,15 +111,12 @@ default String getUpdateStatement(String tableName, String[] fieldNames, String[ " WHERE " + conditionClause; } - /** - * Get delete one row statement by condition fields, default not use limit 1, - * because limit 1 is a sql dialect. - */ - default String getDeleteStatement(String tableName, String[] conditionFields) { + default String getDeleteStatement(String schema, String tableName, String[] conditionFields) { + String schemaInfo = StringUtils.isEmpty(schema) ? "" : quoteIdentifier(schema) + "."; String conditionClause = Arrays.stream(conditionFields) .map(f -> quoteIdentifier(f) + "=?") .collect(Collectors.joining(" AND ")); - return "DELETE FROM " + quoteIdentifier(tableName) + " WHERE " + conditionClause; + return "DELETE FROM " + schemaInfo + quoteIdentifier(tableName) + " WHERE " + conditionClause; } /** diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index b2c51d18d..4639356bb 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -107,7 +107,7 @@ public void open(int taskNumber, int numTasks) throws IOException { initMetric(); if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.APPEND.name()) || keyFields == null || keyFields.length == 0) { - String insertSQL = dialect.getInsertIntoStatement(tableName, fieldNames); + String insertSQL = dialect.getInsertIntoStatement(schema, tableName, fieldNames); LOG.info("execute insert sql： {}", insertSQL); System.out.println("execute insert sql :" + insertSQL); jdbcWriter = new AppendOnlyWriter(insertSQL, fieldTypes, this); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java index 0ea90bc0e..158c309eb 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java @@ -176,9 +176,12 @@ public boolean check() { Preconditions.checkArgument(null != getPrimaryKeys() && getPrimaryKeys().length > 0, "updateMode mode primary is required"); } - Arrays.stream(getPrimaryKeys()).forEach(pk -> { - Preconditions.checkArgument(getFieldList().contains(pk), "primary key " + pk + " not found in sink table field"); - }); + if (null != getPrimaryKeys()) { + Arrays.stream(getPrimaryKeys()).forEach(pk -> { + Preconditions.checkArgument(getFieldList().contains(pk), "primary key " + pk + " not found in sink table field"); + }); + } + return true; } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index f9373c102..48c5f3768 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -71,7 +71,7 @@ public void addRecord(Tuple2 record) throws SQLException { if (!record.f0) { return; } - rows.add(record.f1); + rows.add(Row.copy(record.f1)); } @Override @@ -90,6 +90,9 @@ public void executeBatch(Connection connection) throws SQLException { rows.clear(); } catch (Exception e) { LOG.debug("AppendOnlyWriter executeBatch error ", e); + connection.rollback(); + connection.commit(); + System.out.println(e); cleanBatchWhenError(); executeUpdate(connection); } @@ -97,13 +100,19 @@ public void executeBatch(Connection connection) throws SQLException { @Override - public void executeUpdate(Connection connection) { + public void executeUpdate(Connection connection) { rows.forEach(row -> { try { setRecordToStatement(statement, fieldTypes, row); statement.executeUpdate(); connection.commit(); - } catch (SQLException e) { + } catch (Exception e) { + try { + connection.rollback(); + connection.commit(); + } catch (SQLException e1) { + throw new RuntimeException(e1); + } metricOutputFormat.outDirtyRecords.inc(); if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed ,this row is {}", row.toString()); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java index 20c82c910..db852fc16 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java @@ -66,7 +66,7 @@ public static UpsertWriter create( int[] pkTypes = fieldTypes == null ? null : Arrays.stream(pkFields).map(f -> fieldTypes[f]).toArray(); - String deleteSQL = dialect.getDeleteStatement(tableName, keyFields); + String deleteSQL = dialect.getDeleteStatement(schema, tableName, keyFields); LOG.info("deleteSQL is :{}", deleteSQL); System.out.println("deleteSQL is :" + deleteSQL); @@ -82,7 +82,7 @@ public static UpsertWriter create( fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, // 表存在检查 dialect.getRowExistsStatement(tableName, keyFields), - dialect.getInsertIntoStatement(tableName, fieldNames), + dialect.getInsertIntoStatement(schema, tableName, fieldNames), dialect.getUpdateStatement(tableName, fieldNames, keyFields), metricOutputFormat)); } @@ -143,13 +143,14 @@ public void executeBatch(Connection connection) throws SQLException { } catch (Exception e) { // 清理批处理中的正确字段，防止重复写入 connection.rollback(); + connection.commit(); cleanBatchWhenError(); executeUpdate(connection); } } @Override - public void executeUpdate(Connection connection) { + public void executeUpdate(Connection connection) throws SQLException { if (keyToRows.size() > 0) { for (Map.Entry> entry : keyToRows.entrySet()) { try { @@ -164,11 +165,15 @@ public void executeUpdate(Connection connection) { } connection.commit(); } catch (Exception e) { + // deal pg error: current transaction is aborted, commands ignored until end of transaction block + connection.rollback(); + connection.commit(); metricOutputFormat.outDirtyRecords.inc(); if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed ,this row is {}", entry.getValue()); LOG.error("", e); } + System.out.println(e.getCause()); } } keyToRows.clear(); From 17e5831962d3dfa56ffb5c93be123c4fa8d86c85 Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Tue, 7 Jan 2020 14:24:17 +0800 Subject: [PATCH 006/523] =?UTF-8?q?=E5=9B=9E=E6=BB=9A=E6=B3=A8=E9=87=8A?= =?UTF-8?q?=E6=8E=89=E7=9A=84pom?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/pom.xml b/pom.xml index 4cb63b0a1..8f134831a 100644 --- a/pom.xml +++ b/pom.xml @@ -10,29 +10,29 @@ http://maven.apache.org core - - - - - - - - - - + kafka09 + kafka10 + kafka11 + kafka + mysql + hbase + elasticsearch5 + mongo + redis5 + launcher rdb - + sqlserver oracle - - + cassandra + kudu postgresql - - - - - - + serversocket + console + clickhouse + impala + db2 + polardb From 07bbecc9e0f48ac88985c112a0f7918c1365effc Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Tue, 7 Jan 2020 16:12:23 +0800 Subject: [PATCH 007/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9table=20primarykey=20?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E5=8F=98=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/table/AbsTableParser.java | 3 ++- .../com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java | 4 ++-- .../com/dtstack/flink/sql/side/hbase/HbaseAsyncSideInfo.java | 4 ++-- .../com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java | 4 ++-- .../main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java index 64e730a7d..6ee15876f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java @@ -24,6 +24,7 @@ import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Map; @@ -114,7 +115,7 @@ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ public void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ String primaryFields = matcher.group(1).trim(); - String[] splitArry = primaryFields.split(","); + String[] splitArry = StringUtils.split(primaryFields, ","); tableInfo.setPrimaryKeys(splitArry); } diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java index ea51f46e4..2c95d1e4a 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java @@ -42,11 +42,11 @@ public HbaseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List[] fieldTypes; - protected List primaryKeys; + protected String[] primaryKeys; protected int timeout; @@ -243,7 +243,7 @@ public RedisOutputFormatBuilder setFieldTypes(TypeInformation[] fieldTypes){ return this; } - public RedisOutputFormatBuilder setPrimaryKeys(List primaryKeys){ + public RedisOutputFormatBuilder setPrimaryKeys(String[] primaryKeys){ redisOutputFormat.primaryKeys = primaryKeys; return this; } diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java index d2e28c01f..4ebe867ff 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java @@ -48,7 +48,7 @@ public class RedisSink implements RetractStreamTableSink, IStreamSinkGener< protected String password; - protected List primaryKeys; + protected String[] primaryKeys; protected int timeout; From b6491d682da0bfd3d79fd7b50d73e8ff6e8d47ea Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 14 Jan 2020 15:36:48 +0800 Subject: [PATCH 008/523] function field check --- .../java/com/dtstack/flink/sql/table/AbsTableParser.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java index 5af06d087..2405ad7c6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java @@ -22,6 +22,7 @@ import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.DtStringUtil; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -44,6 +45,7 @@ public abstract class AbsTableParser { private static Pattern primaryKeyPattern = Pattern.compile("(?i)PRIMARY\\s+KEY\\s*\\((.*)\\)"); private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(\\w+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); + private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); private Map patternMap = Maps.newHashMap(); @@ -126,9 +128,12 @@ public void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ */ protected void dealNestField(Matcher matcher, TableInfo tableInfo) { String physicalField = matcher.group(1); + Preconditions.checkArgument(!physicalFieldFunPattern.matcher(physicalField).find(), + "No need to add data types when using functions, The correct way is : strLen(name) as nameSize, "); + String fieldType = matcher.group(3); String mappingField = matcher.group(4); - Class fieldClass= dbTypeConvertToJavaType(fieldType); + Class fieldClass = dbTypeConvertToJavaType(fieldType); boolean notNull = matcher.group(5) != null; TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); fieldExtraInfo.setNotNull(notNull); From 977a20bd20a60965b8c6cb37d1a4dcf8ff8db212 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 15 Jan 2020 20:59:06 +0800 Subject: [PATCH 009/523] sqlserver sink --- .../sink/clickhouse/ClickhouseDialect.java | 46 +++++ .../sql/sink/clickhouse/ClickhouseSink.java | 55 ++---- .../dtstack/flink/sql/side/SideSqlExec.java | 2 +- .../flink/sql/sink/MetricOutputFormat.java | 2 +- .../flink/sql/table/AbsTableParser.java | 3 +- .../dtstack/flink/sql/table/TableInfo.java | 6 +- .../sql/sink/polardb/PolardbDialect.java | 80 ++++++++ .../flink/sql/sink/polardb/PolardbSink.java | 58 ++---- .../sink/polardb/table/PolardbSinkParser.java | 6 +- pom.xml | 40 ++-- .../dtstack/flink/sql/sink/rdb/RdbSink.java | 2 +- .../rdb/format/AbstractJDBCOutputFormat.java | 2 +- .../rdb/format/JDBCUpsertOutputFormat.java | 5 +- .../sql/sink/rdb/table/RdbTableInfo.java | 4 +- .../sql/sink/rdb/writer/UpsertWriter.java | 4 +- .../sql/sink/redis/RedisOutputFormat.java | 2 +- .../sql/sink/sqlserver/SqlserverDialect.java | 102 ++++++++++ .../sql/sink/sqlserver/SqlserverSink.java | 175 ++---------------- 18 files changed, 325 insertions(+), 269 deletions(-) create mode 100644 clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java create mode 100644 polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbDialect.java create mode 100644 sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverDialect.java diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java new file mode 100644 index 000000000..29bf54798 --- /dev/null +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.clickhouse; + +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; + +import java.util.Optional; + +/** + * Date: 2020/1/15 + * Company: www.dtstack.com + * @author maqi + */ +public class ClickhouseDialect implements JDBCDialect { + + @Override + public boolean canHandle(String url) { + return url.startsWith("jdbc:clickhouse:"); + } + + @Override + public Optional defaultDriverName() { + return Optional.of("ru.yandex.clickhouse.ClickHouseDriver"); + } + + @Override + public String getUpdateStatement(String tableName, String[] fieldNames, String[] conditionFields) { + throw new RuntimeException("Clickhouse does not support update sql, please remove primary key or use append mode"); + } +} diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java index 4c91f6873..6e7a3eef4 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java @@ -21,7 +21,9 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; +import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import java.util.List; import java.util.Map; @@ -32,46 +34,25 @@ public class ClickhouseSink extends RdbSink implements IStreamSinkGener private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; public ClickhouseSink() { + super(new ClickhouseDialect()); } @Override - public RetractJDBCOutputFormat getOutputFormat() { - return new RetractJDBCOutputFormat(); - } - - @Override - public void buildSql(String scheam, String tableName, List fields) { - buildInsertSql(tableName, fields); - } - - @Override - public String buildUpdateSql(String schema, String tableName, List fieldNames, Map> realIndexes, List fullField) { - return null; - } - - private void buildInsertSql(String tableName, List fields) { - String sqlTmp = "insert into " + tableName + " (${fields}) values (${placeholder})"; - String fieldsStr = ""; - String placeholder = ""; - - for (String fieldName : fields) { - fieldsStr += ",`" + fieldName + "`"; - placeholder += ",?"; - } - - fieldsStr = fieldsStr.replaceFirst(",", ""); - placeholder = placeholder.replaceFirst(",", ""); - - sqlTmp = sqlTmp.replace("${fields}", fieldsStr).replace("${placeholder}", placeholder); - this.sql = sqlTmp; - System.out.println("---insert sql----"); - System.out.println(sql); - } - - - @Override - public String getDriverName() { - return CLICKHOUSE_DRIVER; + public JDBCUpsertOutputFormat getOutputFormat() { + JDBCOptions jdbcOptions = JDBCOptions.builder() + .setDBUrl(dbURL).setDialect(jdbcDialect) + .setUsername(userName).setPassword(password) + .setTableName(tableName).build(); + + return JDBCUpsertOutputFormat.builder() + .setOptions(jdbcOptions) + .setFieldNames(fieldNames) + .setFlushMaxSize(batchNum) + .setFlushIntervalMills(batchWaitInterval) + .setFieldTypes(sqlTypes) + .setKeyFields(primaryKeys) + .setAllReplace(allReplace) + .setUpdateMode(updateMode).build(); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 8a82df237..687b4e4d4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -627,7 +627,7 @@ private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, private List convertPrimaryAlias(SideTableInfo sideTableInfo) { List res = Lists.newArrayList(); - Arrays.stream(sideTableInfo.getPrimaryKeys()).forEach(field -> { + sideTableInfo.getPrimaryKeys().forEach(field -> { res.add(sideTableInfo.getPhysicalFields().getOrDefault(field, field)); }); return res; diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/MetricOutputFormat.java b/core/src/main/java/com/dtstack/flink/sql/sink/MetricOutputFormat.java index 056a19dea..f052b9702 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/MetricOutputFormat.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/MetricOutputFormat.java @@ -29,7 +29,7 @@ /** * Created by sishu.yss on 2018/11/28. */ -public abstract class MetricOutputFormat extends RichOutputFormat> { +public abstract class MetricOutputFormat extends RichOutputFormat { public transient Counter outRecords; public transient Counter outDirtyRecords; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java index 64e730a7d..5af06d087 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java @@ -115,7 +115,8 @@ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ public void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ String primaryFields = matcher.group(1).trim(); String[] splitArry = primaryFields.split(","); - tableInfo.setPrimaryKeys(splitArry); + List primaryKes = Lists.newArrayList(splitArry); + tableInfo.setPrimaryKeys(primaryKes); } /** diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java index 2559da2ec..59e4fdd39 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java @@ -59,7 +59,7 @@ public abstract class TableInfo implements Serializable { private final List fieldExtraInfoList = Lists.newArrayList(); - private String[] primaryKeys; + private List primaryKeys; private Integer parallelism = -1; @@ -85,11 +85,11 @@ public Class[] getFieldClasses() { return fieldClasses; } - public String[] getPrimaryKeys() { + public List getPrimaryKeys() { return primaryKeys; } - public void setPrimaryKeys(String[] primaryKeys) { + public void setPrimaryKeys(List primaryKeys) { this.primaryKeys = primaryKeys; } diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbDialect.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbDialect.java new file mode 100644 index 000000000..e609de36c --- /dev/null +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbDialect.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.polardb; + +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; + +import java.util.Arrays; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Date: 2020/1/15 + * Company: www.dtstack.com + * @author maqi + */ +public class PolardbDialect implements JDBCDialect { + + @Override + public boolean canHandle(String url) { + return url.startsWith("jdbc:mysql:"); + } + + @Override + public Optional defaultDriverName() { + return Optional.of("com.mysql.cj.jdbc.Driver"); + } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } + + /** + * 根据ALLReplace参数，选择使用replace语句还是ON DUPLICATE KEY UPDATE 语句 + * @param tableName + * @param fieldNames + * @param uniqueKeyFields + * @param allReplace + * @return + */ + @Override + public Optional getUpsertStatement(String schema, String tableName, String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + return allReplace ? buildReplaceIntoStatement(tableName, fieldNames) : buildDuplicateUpsertStatement(tableName, fieldNames); + } + + public Optional buildDuplicateUpsertStatement(String tableName, String[] fieldNames) { + String updateClause = Arrays.stream(fieldNames).map(f -> quoteIdentifier(f) + "=IFNULL(VALUES(" + quoteIdentifier(f) + ")," + quoteIdentifier(f) + ")") + .collect(Collectors.joining(", ")); + return Optional.of(getInsertIntoStatement("", tableName, fieldNames) + + " ON DUPLICATE KEY UPDATE " + updateClause + ); + } + + public Optional buildReplaceIntoStatement(String tableName, String[] fieldNames) { + String columns = Arrays.stream(fieldNames) + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")); + String placeholders = Arrays.stream(fieldNames) + .map(f -> "?") + .collect(Collectors.joining(", ")); + return Optional.of("REPLACE INTO " + quoteIdentifier(tableName) + + "(" + columns + ")" + " VALUES (" + placeholders + ")"); + } +} diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java index 6fd18d10b..e8d695b8e 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java @@ -1,53 +1,33 @@ package com.dtstack.flink.sql.sink.polardb; -import com.dtstack.flink.sql.sink.IStreamSinkGener; +import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import java.util.List; -import java.util.Map; -public class PolardbSink extends RdbSink implements IStreamSinkGener { +public class PolardbSink extends RdbSink { private static final String POLARDB_DRIVER = "com.mysql.cj.jdbc.Driver"; public PolardbSink() { + super(new PolardbDialect()); } @Override - public RetractJDBCOutputFormat getOutputFormat() { - return new RetractJDBCOutputFormat(); - } - - @Override - public void buildSql(String scheam, String tableName, List fields) { - buildInsertSql(tableName, fields); - } - - @Override - public String buildUpdateSql(String schema, String tableName, List fieldNames, Map> realIndexes, List fullField) { - return null; - } - - private void buildInsertSql(String tableName, List fields) { - String sqlTmp = "replace into " + tableName + " (${fields}) values (${placeholder})"; - String fieldsStr = ""; - String placeholder = ""; - - for (String fieldName : fields) { - fieldsStr += ",`" + fieldName + "`"; - placeholder += ",?"; - } - - fieldsStr = fieldsStr.replaceFirst(",", ""); - placeholder = placeholder.replaceFirst(",", ""); - - sqlTmp = sqlTmp.replace("${fields}", fieldsStr).replace("${placeholder}", placeholder); - this.sql = sqlTmp; - } - - - @Override - public String getDriverName() { - return POLARDB_DRIVER; + public JDBCUpsertOutputFormat getOutputFormat() { + JDBCOptions jdbcOptions = JDBCOptions.builder() + .setDBUrl(dbURL).setDialect(jdbcDialect) + .setUsername(userName).setPassword(password) + .setTableName(tableName).build(); + + return JDBCUpsertOutputFormat.builder() + .setOptions(jdbcOptions) + .setFieldNames(fieldNames) + .setFlushMaxSize(batchNum) + .setFlushIntervalMills(batchWaitInterval) + .setFieldTypes(sqlTypes) + .setKeyFields(primaryKeys) + .setAllReplace(allReplace) + .setUpdateMode(updateMode).build(); } } diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java index bece33a42..68eead938 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java @@ -10,8 +10,8 @@ public class PolardbSinkParser extends RdbSinkParser { @Override public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); - mysqlTableInfo.setType(CURR_TYPE); - return mysqlTableInfo; + TableInfo polardbTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + polardbTableInfo.setType(CURR_TYPE); + return polardbTableInfo; } } diff --git a/pom.xml b/pom.xml index 4cb63b0a1..fb1661c02 100644 --- a/pom.xml +++ b/pom.xml @@ -10,29 +10,29 @@ http://maven.apache.org core - - - - - - - - - - + kafka09 + kafka10 + kafka11 + kafka + mysql + hbase + elasticsearch5 + mongo + redis5 + launcher rdb - + sqlserver oracle - - + cassandra + kudu postgresql - - - - - - - + kafka08 + serversocket + console + clickhouse + impala + db2 + polardb diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java index 8b3bb7e5c..c65696903 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java @@ -72,7 +72,7 @@ public abstract class RdbSink implements RetractStreamTableSink, Serializab protected String sql; - protected String[] primaryKeys; + protected List primaryKeys; protected String[] fieldNames; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java index c66fa9e72..0574becaf 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java @@ -37,7 +37,7 @@ * @see Row * @see DriverManager */ -public abstract class AbstractJDBCOutputFormat extends MetricOutputFormat { +public abstract class AbstractJDBCOutputFormat extends MetricOutputFormat { private static final long serialVersionUID = 1L; static final int DEFAULT_FLUSH_MAX_SIZE = 100; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index 4639356bb..22267738a 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.sql.SQLException; +import java.util.List; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -247,8 +248,8 @@ public Builder setFieldNames(String[] fieldNames) { /** * required, upsert unique keys. */ - public Builder setKeyFields(String[] keyFields) { - this.keyFields = keyFields; + public Builder setKeyFields(List keyFields) { + this.keyFields = keyFields == null ? null : keyFields.toArray(new String[keyFields.size()]); return this; } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java index 158c309eb..d5ad4eab6 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java @@ -173,11 +173,11 @@ public boolean check() { Preconditions.checkNotNull(password, "rdb field of password is required"); if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { - Preconditions.checkArgument(null != getPrimaryKeys() && getPrimaryKeys().length > 0, "updateMode mode primary is required"); + Preconditions.checkArgument(null != getPrimaryKeys() && getPrimaryKeys().size() > 0, "updateMode mode primary is required"); } if (null != getPrimaryKeys()) { - Arrays.stream(getPrimaryKeys()).forEach(pk -> { + getPrimaryKeys().forEach(pk -> { Preconditions.checkArgument(getFieldList().contains(pk), "primary key " + pk + " not found in sink table field"); }); } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java index db852fc16..43367be6e 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java @@ -165,15 +165,15 @@ public void executeUpdate(Connection connection) throws SQLException { } connection.commit(); } catch (Exception e) { + System.out.println(e.getCause()); // deal pg error: current transaction is aborted, commands ignored until end of transaction block connection.rollback(); connection.commit(); - metricOutputFormat.outDirtyRecords.inc(); if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed ,this row is {}", entry.getValue()); LOG.error("", e); } - System.out.println(e.getCause()); + metricOutputFormat.outDirtyRecords.inc(); } } keyToRows.clear(); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index bb3963edb..22cc4ccdb 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -31,7 +31,7 @@ import java.io.IOException; import java.util.*; -public class RedisOutputFormat extends MetricOutputFormat { +public class RedisOutputFormat extends MetricOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(RedisOutputFormat.class); private String url; diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverDialect.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverDialect.java new file mode 100644 index 000000000..f94f71cfc --- /dev/null +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverDialect.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.sqlserver; + +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import com.dtstack.flink.sql.util.DtStringUtil; +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Date: 2020/1/15 + * Company: www.dtstack.com + * @author maqi + */ +public class SqlserverDialect implements JDBCDialect { + + @Override + public boolean canHandle(String url) { + return url.startsWith("jdbc:jtds:"); + } + + @Override + public Optional defaultDriverName() { + return Optional.of("net.sourceforge.jtds.jdbc.Driver"); + } + + @Override + public Optional getUpsertStatement(String schema, String tableName, String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + tableName = DtStringUtil.getTableFullPath(schema, tableName); + StringBuilder sb = new StringBuilder(); + sb.append("MERGE INTO " + tableName + " T1 USING " + + "(" + buildDualQueryStatement(fieldNames) + ") T2 ON (" + + buildConnectionConditions(uniqueKeyFields) + ") "); + + String updateSql = buildUpdateConnection(fieldNames, uniqueKeyFields, allReplace); + + if (StringUtils.isNotEmpty(updateSql)) { + sb.append(" WHEN MATCHED THEN UPDATE SET "); + sb.append(updateSql); + } + + sb.append(" WHEN NOT MATCHED THEN " + + "INSERT (" + Arrays.stream(fieldNames).map(this::quoteIdentifier).collect(Collectors.joining(",")) + ") VALUES (" + + Arrays.stream(fieldNames).map(col -> "T2." + quoteIdentifier(col)).collect(Collectors.joining(",")) + ")"); + sb.append(";"); + return Optional.of(sb.toString()); + } + + /** + * build T1."A"=T2."A" or T1."A"=nvl(T2."A",T1."A") + * @param fieldNames + * @param uniqueKeyFields + * @param allReplace + * @return + */ + private String buildUpdateConnection(String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + List uniqueKeyList = Arrays.asList(uniqueKeyFields); + return Arrays.stream(fieldNames).filter(col -> !uniqueKeyList.contains(col)).map(col -> { + return allReplace ? quoteIdentifier("T1") + "." + quoteIdentifier(col) + " = " + quoteIdentifier("T2") + "." + quoteIdentifier(col) : + quoteIdentifier("T1") + "." + quoteIdentifier(col) + " =ISNULL(" + quoteIdentifier("T2") + "." + quoteIdentifier(col) + "," + + quoteIdentifier("T1") + "." + quoteIdentifier(col) + ")"; + }).collect(Collectors.joining(",")); + } + + + private String buildConnectionConditions(String[] uniqueKeyFields) { + return Arrays.stream(uniqueKeyFields).map(col -> "T1." + quoteIdentifier(col) + "=T2." + quoteIdentifier(col)).collect(Collectors.joining(",")); + } + + /** + * build select sql , such as (SELECT ? "A",? "B" FROM DUAL) + * + * @param column destination column + * @return + */ + public String buildDualQueryStatement(String[] column) { + StringBuilder sb = new StringBuilder("SELECT "); + String collect = Arrays.stream(column).map(col -> " ? " + quoteIdentifier(col)).collect(Collectors.joining(", ")); + sb.append(collect); + return sb.toString(); + } +} diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java index 82db4d318..5cb382f17 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java @@ -18,11 +18,9 @@ package com.dtstack.flink.sql.sink.sqlserver; import com.dtstack.flink.sql.sink.IStreamSinkGener; +import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.RdbSink; -import com.dtstack.flink.sql.sink.rdb.format.ExtendOutputFormat; -import org.apache.commons.lang3.StringUtils; - -import java.util.*; +import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** * Reason: @@ -31,160 +29,27 @@ * * @author maqi */ -public class SqlserverSink extends RdbSink implements IStreamSinkGener { - private static final String SQLSERVER_DRIVER = "net.sourceforge.jtds.jdbc.Driver"; - - @Override - public String getDriverName() { - return SQLSERVER_DRIVER; - } - - @Override - public RetractJDBCOutputFormat getOutputFormat() { - return new ExtendOutputFormat(); - } - - @Override - public void buildSql(String scheam, String tableName, List fields) { - buildInsertSql(tableName, fields); - } - - private void buildInsertSql(String tableName, List fields) { - String sqlTmp = "insert into " + tableName + " (${fields}) values (${placeholder})"; - String fieldsStr = StringUtils.join(fields, ","); - String placeholder = ""; +public class SqlserverSink extends RdbSink implements IStreamSinkGener { - for (String fieldName : fields) { - placeholder += ",?"; - } - placeholder = placeholder.replaceFirst(",", ""); - sqlTmp = sqlTmp.replace("${fields}", fieldsStr).replace("${placeholder}", placeholder); - this.sql = sqlTmp; + public SqlserverSink() { + super(new SqlserverDialect()); } @Override - public String buildUpdateSql(String scheam, String tableName, List fieldNames, Map> realIndexes, List fullField) { - StringBuilder sb = new StringBuilder(); - - sb.append("MERGE INTO " + tableName + " T1 USING " - + "(" + makeValues(fieldNames) + ") T2 ON (" - + updateKeySql(realIndexes) + ") "); - - - String updateSql = getUpdateSql(fieldNames, fullField, "T1", "T2", keyColList(realIndexes)); - - if (StringUtils.isNotEmpty(updateSql)) { - sb.append(" WHEN MATCHED THEN UPDATE SET "); - sb.append(updateSql); - } - - sb.append(" WHEN NOT MATCHED THEN " - + "INSERT (" + quoteColumns(fieldNames) + ") VALUES (" - + quoteColumns(fieldNames, "T2") + ")"); - - return sb.toString(); - } - - - public String quoteColumns(List column) { - return quoteColumns(column, null); - } - - public String quoteColumns(List column, String table) { - String prefix = StringUtils.isBlank(table) ? "" : quoteTable(table) + "."; - List list = new ArrayList<>(); - for (String col : column) { - list.add(prefix + quoteColumn(col)); - } - return StringUtils.join(list, ","); + public JDBCUpsertOutputFormat getOutputFormat() { + JDBCOptions jdbcOptions = JDBCOptions.builder() + .setDBUrl(dbURL).setDialect(jdbcDialect) + .setUsername(userName).setPassword(password) + .setTableName(tableName).setScheam(schema).build(); + + return JDBCUpsertOutputFormat.builder() + .setOptions(jdbcOptions) + .setFieldNames(fieldNames) + .setFlushMaxSize(batchNum) + .setFlushIntervalMills(batchWaitInterval) + .setFieldTypes(sqlTypes) + .setKeyFields(primaryKeys) + .setAllReplace(allReplace) + .setUpdateMode(updateMode).build(); } - - protected List keyColList(Map> updateKey) { - List keyCols = new ArrayList<>(); - for (Map.Entry> entry : updateKey.entrySet()) { - List list = entry.getValue(); - for (String col : list) { - if (!containsIgnoreCase(keyCols,col)) { - keyCols.add(col); - } - } - } - return keyCols; - } - - public String getUpdateSql(List column, List fullColumn, String leftTable, String rightTable, List keyCols) { - String prefixLeft = StringUtils.isBlank(leftTable) ? "" : quoteTable(leftTable) + "."; - String prefixRight = StringUtils.isBlank(rightTable) ? "" : quoteTable(rightTable) + "."; - List list = new ArrayList<>(); - for (String col : fullColumn) { - if (keyCols == null || keyCols.size() == 0) { - continue; - } - if (fullColumn == null || containsIgnoreCase(column,col)) { - list.add(prefixLeft + col + "=" + prefixRight + col); - } else { - list.add(prefixLeft + col + "=null"); - } - } - return StringUtils.join(list, ","); - } - - public String quoteTable(String table) { - String[] parts = table.split("\\."); - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < parts.length; ++i) { - if (i != 0) { - sb.append("."); - } - sb.append(getStartQuote() + parts[i] + getEndQuote()); - } - return sb.toString(); - } - - - public String updateKeySql(Map> updateKey) { - List exprList = new ArrayList<>(); - for (Map.Entry> entry : updateKey.entrySet()) { - List colList = new ArrayList<>(); - for (String col : entry.getValue()) { - colList.add("T1." + quoteColumn(col) + "=T2." + quoteColumn(col)); - } - exprList.add(StringUtils.join(colList, " AND ")); - } - return StringUtils.join(exprList, " OR "); - } - - - public String makeValues(List column) { - StringBuilder sb = new StringBuilder("SELECT "); - for (int i = 0; i < column.size(); ++i) { - if (i != 0) { - sb.append(","); - } - sb.append("? " + quoteColumn(column.get(i))); - } - return sb.toString(); - } - - public boolean containsIgnoreCase(List l, String s) { - Iterator it = l.iterator(); - while (it.hasNext()) { - if (it.next().equalsIgnoreCase(s)) - return true; - } - return false; - } - public String quoteColumn(String column) { - return getStartQuote() + column + getEndQuote(); - } - - public String getStartQuote() { - return "\""; - } - - public String getEndQuote() { - return "\""; - } - - } From 8b7ca0bb2d703f88ebd5edf3d7f9a52ebe00da7e Mon Sep 17 00:00:00 2001 From: maqi Date: Sun, 19 Jan 2020 17:17:41 +0800 Subject: [PATCH 010/523] impala sink --- .../sink/cassandra/CassandraOutputFormat.java | 2 +- .../sql/sink/console/ConsoleOutputFormat.java | 2 +- .../dtstack/flink/sql/sink/db/DbDialect.java | 46 +++++ .../com/dtstack/flink/sql/sink/db/DbSink.java | 53 ++---- .../sql/sink/hbase/HbaseOutputFormat.java | 2 +- .../flink/sql/sink/impala/ImpalaDialect.java | 84 +++++++++ .../sql/sink/impala/ImpalaOutputFormat.java | 119 ------------- .../flink/sql/sink/impala/ImpalaSink.java | 164 ++++++++---------- .../sink/impala/table/ImpalaSinkParser.java | 4 +- .../sink/impala/table/ImpalaTableInfo.java | 10 +- .../flink/sql/sink/kudu/KuduOutputFormat.java | 2 +- .../sql/sink/mongo/MongoOutputFormat.java | 2 +- .../flink/sql/sink/mysql/MySQLDialect.java | 2 +- .../sql/sink/polardb/PolardbDialect.java | 2 +- pom.xml | 1 - .../sink/postgresql/PostgresqlDialect.java | 2 +- .../sql/sink/rdb/dialect/JDBCDialect.java | 2 +- .../rdb/format/AbstractJDBCOutputFormat.java | 12 +- .../rdb/format/JDBCUpsertOutputFormat.java | 38 ++-- .../sql/sink/rdb/writer/AppendOnlyWriter.java | 1 - .../sql/sink/rdb/writer/UpsertWriter.java | 4 +- .../sql/sink/redis/table/RedisSinkParser.java | 10 +- .../sql/sink/redis/table/RedisTableInfo.java | 2 + 23 files changed, 285 insertions(+), 281 deletions(-) create mode 100644 db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbDialect.java create mode 100644 impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java delete mode 100644 impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java index 242cba3ee..70fb6b12e 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java @@ -69,7 +69,7 @@ * @see Tuple * @see DriverManager */ -public class CassandraOutputFormat extends MetricOutputFormat { +public class CassandraOutputFormat extends MetricOutputFormat { private static final long serialVersionUID = -7994311331389155692L; private static final Logger LOG = LoggerFactory.getLogger(CassandraOutputFormat.class); diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java index 7658e9979..28666f315 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java @@ -37,7 +37,7 @@ * * @author xuqianjin */ -public class ConsoleOutputFormat extends MetricOutputFormat { +public class ConsoleOutputFormat extends MetricOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(ConsoleOutputFormat.class); diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbDialect.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbDialect.java new file mode 100644 index 000000000..9c7985d8f --- /dev/null +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbDialect.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.db; + +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; + +import java.util.Optional; + +/** + * Date: 2020/1/19 + * Company: www.dtstack.com + * @author maqi + */ +public class DbDialect implements JDBCDialect { + @Override + public boolean canHandle(String url) { + return url.startsWith("jdbc:db2:"); + } + + @Override + public Optional defaultDriverName() { + return Optional.of("com.ibm.db2.jcc.DB2Driver"); + } + + @Override + public String quoteIdentifier(String identifier) { + return identifier; + } + +} diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java index 028a156dc..8e6fb4bd7 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java @@ -1,6 +1,8 @@ package com.dtstack.flink.sql.sink.db; +import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import java.util.List; import java.util.Map; @@ -10,42 +12,23 @@ public class DbSink extends RdbSink { private static final String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; public DbSink() { + super(new DbDialect()); } - - @Override - public void buildSql(String schema, String tableName, List fields) { - buildInsertSql(tableName, fields); - } - - private void buildInsertSql(String tableName, List fields) { - String sqlTmp = "insert into " + tableName + " (${fields}) values (${placeholder})"; - String fieldsStr = ""; - String placeholder = ""; - - for (String fieldName : fields) { - fieldsStr += "," + fieldName; - placeholder += ",?"; - } - - fieldsStr = fieldsStr.replaceFirst(",", ""); - placeholder = placeholder.replaceFirst(",", ""); - - sqlTmp = sqlTmp.replace("${fields}", fieldsStr).replace("${placeholder}", placeholder); - this.sql = sqlTmp; - } - - @Override - public String buildUpdateSql(String schema, String tableName, List fieldNames, Map> realIndexes, List fullField) { - return null; - } - - @Override - public String getDriverName() { - return DB2_DRIVER; - } - @Override - public RetractJDBCOutputFormat getOutputFormat() { - return new RetractJDBCOutputFormat(); + public JDBCUpsertOutputFormat getOutputFormat() { + JDBCOptions jdbcOptions = JDBCOptions.builder() + .setDBUrl(dbURL).setDialect(jdbcDialect) + .setUsername(userName).setPassword(password) + .setTableName(tableName).build(); + + return JDBCUpsertOutputFormat.builder() + .setOptions(jdbcOptions) + .setFieldNames(fieldNames) + .setFlushMaxSize(batchNum) + .setFlushIntervalMills(batchWaitInterval) + .setFieldTypes(sqlTypes) + .setKeyFields(primaryKeys) + .setAllReplace(allReplace) + .setUpdateMode(updateMode).build(); } } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index b893892c0..0185ad068 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -45,7 +45,7 @@ * author: jingzhen@dtstack.com * date: 2017-6-29 */ -public class HbaseOutputFormat extends MetricOutputFormat { +public class HbaseOutputFormat extends MetricOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(HbaseOutputFormat.class); diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java new file mode 100644 index 000000000..d8523eeb7 --- /dev/null +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.impala; + +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Date: 2020/1/17 + * Company: www.dtstack.com + * @author maqi + */ +public class ImpalaDialect implements JDBCDialect { + private static final long serialVersionUID = 1L; + + private static final String IMPALA_PARTITION_KEYWORD = " partition"; + + @Override + public boolean canHandle(String url) { + return url.startsWith("jdbc:impala:"); + } + + @Override + public Optional defaultDriverName() { + return Optional.of("com.cloudera.impala.jdbc41.Driver"); + } + + @Override + public String quoteIdentifier(String identifier) { + return identifier; + } + + @Override + public String getUpdateStatement(String tableName, String[] fieldNames, String[] conditionFields) { + throw new RuntimeException("impala does not support update sql, please remove primary key or use append mode"); + } + + @Override + public String getInsertIntoStatement(String schema, String tableName, String[] fieldNames, String[] partitionFields) { + + String schemaInfo = StringUtils.isEmpty(schema) ? "" : quoteIdentifier(schema) + "."; + + List partitionFieldsList = Arrays.asList(partitionFields); + + String columns = Arrays.stream(fieldNames) + .filter(f -> !partitionFieldsList.contains(f)) + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")); + + String placeholders = Arrays.stream(fieldNames) + .map(f -> "?") + .collect(Collectors.joining(", ")); + + String partitionFieldStr = Arrays.stream(partitionFields) + .map(field -> field.replaceAll("\"", "'")) + .collect(Collectors.joining(", ")); + + String partitionStatement = StringUtils.isEmpty(partitionFieldStr) ? "" : IMPALA_PARTITION_KEYWORD + "(" + partitionFieldStr + ")"; + + return "INSERT INTO " + schemaInfo + quoteIdentifier(tableName) + + "(" + columns + ")" + partitionStatement + " VALUES (" + placeholders + ")"; + } +} diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java deleted file mode 100644 index fae2ef257..000000000 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.sink.impala; - -import com.dtstack.flink.sql.sink.impala.table.ImpalaTableInfo; -import com.dtstack.flink.sql.util.JDBCUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.UserGroupInformation; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -/** - * Reason: - * Date: 2019/11/13 - * - * @author xiuzhu - */ - -public class ImpalaOutputFormat extends RetractJDBCOutputFormat { - - private ImpalaTableInfo impalaTableInfo; - - @Override - public Connection establishConnection() throws SQLException, ClassNotFoundException, IOException { - Connection connection ; - String newUrl = ""; - String drivername = getDrivername(); - Integer authMech = impalaTableInfo.getAuthMech(); - - JDBCUtils.forName(drivername, getClass().getClassLoader()); - - StringBuffer urlBuffer = new StringBuffer(impalaTableInfo.getUrl()); - if (authMech == 0) { - newUrl = urlBuffer.toString(); - } else if (authMech == 1) { - String keyTabFilePath = impalaTableInfo.getKeyTabFilePath(); - String krb5FilePath = impalaTableInfo.getKrb5FilePath(); - String principal = impalaTableInfo.getPrincipal(); - String krbRealm = impalaTableInfo.getKrbRealm(); - String krbHostFQDN = impalaTableInfo.getKrbHostFQDN(); - String krbServiceName = impalaTableInfo.getKrbServiceName(); - urlBuffer.append(";" - .concat("AuthMech=1;") - .concat("KrbRealm=").concat(krbRealm).concat(";") - .concat("KrbHostFQDN=").concat(krbHostFQDN).concat(";") - .concat("KrbServiceName=").concat(krbServiceName).concat(";") - ); - newUrl = urlBuffer.toString(); - - System.setProperty("java.security.krb5.conf", krb5FilePath); - Configuration configuration = new Configuration(); - configuration.set("hadoop.security.authentication" , "Kerberos"); - UserGroupInformation.setConfiguration(configuration); - UserGroupInformation.loginUserFromKeytab(principal, keyTabFilePath); - - } else if (authMech == 2) { - String userName = impalaTableInfo.getUserName(); - urlBuffer.append(";" - .concat("AuthMech=3;") - .concat("UID=").concat(userName).concat(";") - .concat("PWD=;") - .concat("UseSasl=0") - ); - newUrl = urlBuffer.toString(); - } else if (authMech == 3) { - String userName = impalaTableInfo.getUserName(); - String password = impalaTableInfo.getPassword(); - urlBuffer.append(";" - .concat("AuthMech=3;") - .concat("UID=").concat(userName).concat(";") - .concat("PWD=").concat(password) - ); - newUrl = urlBuffer.toString(); - } else { - throw new IllegalArgumentException("The value of authMech is illegal, Please select 0, 1, 2, 3"); - } - - connection = DriverManager.getConnection(newUrl); - connection.setAutoCommit(false); - return connection; - } - - @Override - public void verifyField() { - super.verifyField(); - if (impalaTableInfo == null) { - throw new IllegalArgumentException("No impalaTableInfo supplied."); - } - - } - - public ImpalaTableInfo getImpalaTableInfo() { - return impalaTableInfo; - } - - public void setImpalaTableInfo(ImpalaTableInfo impalaTableInfo) { - this.impalaTableInfo = impalaTableInfo; - } -} diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index 0ce3ad29a..d60f302e1 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -20,12 +20,21 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.impala.table.ImpalaTableInfo; +import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.commons.lang.StringUtils; import org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import java.io.IOException; +import java.sql.DriverManager; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -40,108 +49,89 @@ public class ImpalaSink extends RdbSink implements IStreamSinkGener { - private static final String IMPALA_DRIVER = "com.cloudera.impala.jdbc41.Driver"; - private ImpalaTableInfo impalaTableInfo; public ImpalaSink() { + super(new ImpalaDialect()); } @Override - public RichSinkFunction createJdbcSinkFunc() { - ImpalaOutputFormat outputFormat = (ImpalaOutputFormat) getOutputFormat(); - outputFormat.setDbURL(dbURL); - outputFormat.setDrivername(driverName); - outputFormat.setUsername(userName); - outputFormat.setPassword(password); - outputFormat.setInsertQuery(sql); - outputFormat.setBatchNum(batchNum); - outputFormat.setBatchWaitInterval(batchWaitInterval); - outputFormat.setTypesArray(sqlTypes); - outputFormat.setTableName(tableName); - outputFormat.setDbType(dbType); - outputFormat.setSchema(impalaTableInfo.getSchema()); - outputFormat.setDbSink(this); - outputFormat.setImpalaTableInfo(impalaTableInfo); - - outputFormat.verifyField(); - OutputFormatSinkFunction outputFormatSinkFunc = new OutputFormatSinkFunction(outputFormat); - return outputFormatSinkFunc; + public JDBCUpsertOutputFormat getOutputFormat() { + JDBCOptions jdbcOptions = JDBCOptions.builder() + .setDBUrl(getImpalaJdbcUrl()).setDialect(jdbcDialect) + .setUsername(userName).setPassword(password) + .setTableName(tableName).build(); + + return JDBCUpsertOutputFormat.builder() + .setOptions(jdbcOptions) + .setFieldNames(fieldNames) + .setFlushMaxSize(batchNum) + .setFlushIntervalMills(batchWaitInterval) + .setFieldTypes(sqlTypes) + .setKeyFields(primaryKeys) + .setPartitionFields(impalaTableInfo.getPartitionFields()) + .setAllReplace(allReplace) + .setUpdateMode(updateMode).build(); } - @Override - public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { - ImpalaTableInfo impalaTableInfo = (ImpalaTableInfo) targetTableInfo; - this.impalaTableInfo = impalaTableInfo; - super.genStreamSink(targetTableInfo); - return this; - } - @Override - public void buildSql(String schema, String tableName, List fields) { - buildInsertSql(tableName, fields); - } - - public void buildInsertSql(String tableName, List fields) { - - String sqlTmp = "insert into " + tableName + " (${fields}) " + "${partition}"+ " values (${placeholder})"; - int fieldsSize = fields.size(); - boolean enablePartition = impalaTableInfo.isEnablePartition(); - if (enablePartition) { - String partitionFieldsStr = impalaTableInfo.getPartitionFields(); - partitionFieldsStr = !StringUtils.isEmpty(partitionFieldsStr) ? partitionFieldsStr.replaceAll("\"", "'") : partitionFieldsStr; - - List partitionFields = Arrays.asList(partitionFieldsStr.split(",")); - List newFields = new ArrayList<>(); - for (String field : fields) { - if (partitionFields.contains(field)){ - continue; - } - newFields.add(field); + public String getImpalaJdbcUrl() { + Integer authMech = impalaTableInfo.getAuthMech(); + String newUrl = dbURL; + StringBuffer urlBuffer = new StringBuffer(dbURL); + if (authMech == 0) { + return newUrl; + } else if (authMech == 1) { + String keyTabFilePath = impalaTableInfo.getKeyTabFilePath(); + String krb5FilePath = impalaTableInfo.getKrb5FilePath(); + String principal = impalaTableInfo.getPrincipal(); + String krbRealm = impalaTableInfo.getKrbRealm(); + String krbHostFQDN = impalaTableInfo.getKrbHostFQDN(); + String krbServiceName = impalaTableInfo.getKrbServiceName(); + urlBuffer.append(";" + .concat("AuthMech=1;") + .concat("KrbRealm=").concat(krbRealm).concat(";") + .concat("KrbHostFQDN=").concat(krbHostFQDN).concat(";") + .concat("KrbServiceName=").concat(krbServiceName).concat(";") + ); + newUrl = urlBuffer.toString(); + + System.setProperty("java.security.krb5.conf", krb5FilePath); + Configuration configuration = new Configuration(); + configuration.set("hadoop.security.authentication", "Kerberos"); + UserGroupInformation.setConfiguration(configuration); + try { + UserGroupInformation.loginUserFromKeytab(principal, keyTabFilePath); + } catch (IOException e) { + throw new RuntimeException("loginUserFromKeytab error ..", e); } - fields = newFields; - String partition = String.format("partition(%s)", partitionFieldsStr); - sqlTmp = sqlTmp.replace("${partition}", partition); - } else { - sqlTmp = sqlTmp.replace("${partition}", ""); - } - String fieldsStr = ""; - String placeholder = ""; - for (int i= 0; i < fieldsSize; i++) { - placeholder += ",?"; - } - for (String fieldName : fields) { - fieldsStr += "," + fieldName; + } else if (authMech == 2) { + urlBuffer.append(";" + .concat("AuthMech=3;") + .concat("UID=").concat(userName).concat(";") + .concat("PWD=;") + .concat("UseSasl=0") + ); + newUrl = urlBuffer.toString(); + } else if (authMech == 3) { + urlBuffer.append(";" + .concat("AuthMech=3;") + .concat("UID=").concat(userName).concat(";") + .concat("PWD=").concat(password) + ); + newUrl = urlBuffer.toString(); + } else { + throw new IllegalArgumentException("The value of authMech is illegal, Please select 0, 1, 2, 3"); } - fieldsStr = fieldsStr.replaceFirst(",", ""); - placeholder = placeholder.replaceFirst(",", ""); - sqlTmp = sqlTmp.replace("${fields}", fieldsStr).replace("${placeholder}", placeholder); - - this.sql = sqlTmp; - } - - @Override - public String buildUpdateSql(String schema, String tableName, List fieldNames, Map> realIndexes, List fullField) { - return null; - } - - @Override - public String getDriverName() { - return IMPALA_DRIVER; + return newUrl; } @Override - public RetractJDBCOutputFormat getOutputFormat() { - return new ImpalaOutputFormat(); - } - - - public ImpalaTableInfo getImpalaTableInfo() { - return impalaTableInfo; + public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { + super.genStreamSink(targetTableInfo); + this.impalaTableInfo = (ImpalaTableInfo) targetTableInfo; + return this; } - public void setImpalaTableInfo(ImpalaTableInfo impalaTableInfo) { - this.impalaTableInfo = impalaTableInfo; - } } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java index c39246e46..4921f5e51 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java @@ -70,8 +70,8 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map 0, "impala field of partitionFields is required"); } return true; diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index 2eea34b8f..43c9f98ca 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -32,7 +32,7 @@ import java.sql.Timestamp; import java.util.Date; -public class KuduOutputFormat extends MetricOutputFormat { +public class KuduOutputFormat extends MetricOutputFormat { private static final long serialVersionUID = 1L; diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index f37f5f379..be6462c3c 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -47,7 +47,7 @@ * * @author xuqianjin */ -public class MongoOutputFormat extends MetricOutputFormat { +public class MongoOutputFormat extends MetricOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(MongoOutputFormat.class); private String address; diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java index 9f53cd328..9ce6833a8 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MySQLDialect.java @@ -63,7 +63,7 @@ public Optional getUpsertStatement(String schema, String tableName, Stri public Optional buildDuplicateUpsertStatement(String tableName, String[] fieldNames) { String updateClause = Arrays.stream(fieldNames).map(f -> quoteIdentifier(f) + "=IFNULL(VALUES(" + quoteIdentifier(f) + ")," + quoteIdentifier(f) + ")") .collect(Collectors.joining(", ")); - return Optional.of(getInsertIntoStatement("", tableName, fieldNames) + + return Optional.of(getInsertIntoStatement("", tableName, fieldNames, null) + " ON DUPLICATE KEY UPDATE " + updateClause ); } diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbDialect.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbDialect.java index e609de36c..ec4b190b5 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbDialect.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbDialect.java @@ -62,7 +62,7 @@ public Optional getUpsertStatement(String schema, String tableName, Stri public Optional buildDuplicateUpsertStatement(String tableName, String[] fieldNames) { String updateClause = Arrays.stream(fieldNames).map(f -> quoteIdentifier(f) + "=IFNULL(VALUES(" + quoteIdentifier(f) + ")," + quoteIdentifier(f) + ")") .collect(Collectors.joining(", ")); - return Optional.of(getInsertIntoStatement("", tableName, fieldNames) + + return Optional.of(getInsertIntoStatement("", tableName, fieldNames, null) + " ON DUPLICATE KEY UPDATE " + updateClause ); } diff --git a/pom.xml b/pom.xml index fb1661c02..a7d345185 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,6 @@ cassandra kudu postgresql - kafka08 serversocket console clickhouse diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java index 212bb2df7..58e92c143 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java @@ -52,7 +52,7 @@ public Optional getUpsertStatement(String schema, String tableName, Stri .map(f -> quoteIdentifier(f) + "=EXCLUDED." + quoteIdentifier(f)) .collect(Collectors.joining(", ")); - return Optional.of(getInsertIntoStatement(schema, tableName, fieldNames) + + return Optional.of(getInsertIntoStatement(schema, tableName, fieldNames, null) + " ON CONFLICT (" + uniqueColumns + ")" + " DO UPDATE SET " + updateClause ); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java index 2655eb82b..48108246a 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/dialect/JDBCDialect.java @@ -83,7 +83,7 @@ default String getRowExistsStatement(String tableName, String[] conditionFields) /** * Get insert into statement. */ - default String getInsertIntoStatement(String schema, String tableName, String[] fieldNames) { + default String getInsertIntoStatement(String schema, String tableName, String[] fieldNames, String[] partitionFields) { String schemaInfo = StringUtils.isEmpty(schema) ? "" : quoteIdentifier(schema) + "."; String columns = Arrays.stream(fieldNames) .map(this::quoteIdentifier) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java index 0574becaf..81494390b 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java @@ -40,14 +40,14 @@ public abstract class AbstractJDBCOutputFormat extends MetricOutputFormat { private static final long serialVersionUID = 1L; - static final int DEFAULT_FLUSH_MAX_SIZE = 100; - static final long DEFAULT_FLUSH_INTERVAL_MILLS = 10000L; - static final boolean DEFAULT_ALLREPLACE_VALUE = false; + public static final int DEFAULT_FLUSH_MAX_SIZE = 100; + public static final long DEFAULT_FLUSH_INTERVAL_MILLS = 10000L; + public static final boolean DEFAULT_ALLREPLACE_VALUE = false; private static final Logger LOG = LoggerFactory.getLogger(AbstractJDBCOutputFormat.class); - private final String username; - private final String password; + protected final String username; + protected final String password; private final String drivername; protected final String dbURL; @@ -64,7 +64,7 @@ public AbstractJDBCOutputFormat(String username, String password, String drivern public void configure(Configuration parameters) { } - protected void establishConnection() throws SQLException, ClassNotFoundException { + protected void establishConnection() throws SQLException, ClassNotFoundException, IOException { JDBCUtils.forName(drivername, getClass().getClassLoader()); if (username == null) { connection = DriverManager.getConnection(dbURL); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index 22267738a..9cb44b194 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -58,6 +58,7 @@ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat new UpsertWriterUsingInsertUpdateStatement( fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, - // 表存在检查 dialect.getRowExistsStatement(tableName, keyFields), - dialect.getInsertIntoStatement(schema, tableName, fieldNames), + dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields), dialect.getUpdateStatement(tableName, fieldNames, keyFields), metricOutputFormat)); } diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index a0a84b8de..49861bb9a 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -21,10 +21,12 @@ import com.dtstack.flink.sql.table.AbsTableParser; import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; +import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Map; public class RedisSinkParser extends AbsTableParser { @@ -46,12 +48,12 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map primaryKeysList = Lists.newArrayList(); if (!StringUtils.isEmpty(primaryKeysStr)) { - primaryKeysArray = primaryKeysStr.split(","); + primaryKeysList = Arrays.asList(primaryKeysStr.split(",")); } - redisTableInfo.setPrimaryKeys(primaryKeysArray); + redisTableInfo.setPrimaryKeys(primaryKeysList); return redisTableInfo; } diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java index 82def4115..60d6dd12f 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java @@ -45,6 +45,8 @@ public class RedisTableInfo extends TargetTableInfo { public static final String MASTER_NAME = "masterName"; + public static final String PRIMARY_KEYS_NAME = "primarykeys"; + public RedisTableInfo(){ setType(CURR_TYPE); } From ec341edaa897d5f6931c6af4a5c18ca9279b7c26 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 3 Feb 2020 17:54:46 +0800 Subject: [PATCH 011/523] hbase upsert mode --- .../sql/sink/hbase/HbaseOutputFormat.java | 108 +++++++++++++----- .../flink/sql/sink/hbase/HbaseSink.java | 14 ++- .../sql/sink/hbase/table/HbaseSinkParser.java | 5 + .../sql/sink/hbase/table/HbaseTableInfo.java | 10 ++ 4 files changed, 101 insertions(+), 36 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 0185ad068..6ba35d78f 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -16,10 +16,10 @@ * limitations under the License. */ - package com.dtstack.flink.sql.sink.hbase; +import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.sink.MetricOutputFormat; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; @@ -31,10 +31,13 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.IOException; import java.text.SimpleDateFormat; import java.util.List; @@ -54,8 +57,9 @@ public class HbaseOutputFormat extends MetricOutputFormat { private String[] rowkey; private String tableName; private String[] columnNames; + private String updateMode; private String[] columnTypes; - private Map columnNameFamily; + private Map columnNameFamily; private String[] families; private String[] qualifiers; @@ -75,7 +79,7 @@ public void configure(Configuration parameters) { LOG.warn("---configure---"); conf = HBaseConfiguration.create(); conf.set("hbase.zookeeper.quorum", host); - if(zkParent != null && !"".equals(zkParent)){ + if (zkParent != null && !"".equals(zkParent)) { conf.set("zookeeper.znode.parent", zkParent); } LOG.warn("---configure end ---"); @@ -91,38 +95,23 @@ public void open(int taskNumber, int numTasks) throws IOException { } @Override - public void writeRecord(Tuple2 tuple2) { - + public void writeRecord(Tuple2 tuple2) { Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); - if(!retract){ - //FIXME 暂时不处理hbase删除操作--->hbase要求有key,所有认为都是可以执行update查找 - return; + if (!retract && StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + dealDelete(tupleTrans); + } else { + dealInsert(tupleTrans); } + } + protected void dealInsert(Tuple2 tupleTrans) { Row record = tupleTrans.getField(1); - List rowKeyValues = getRowKeyValues(record); - // all rowkey not null - if (rowKeyValues.size() != rowkey.length ) { - LOG.error("row key value must not null,record is ..", record); - outDirtyRecords.inc(); + Put put = getPutByRow(record); + if (put == null) { return; } - String key = StringUtils.join(rowKeyValues, "-"); - Put put = new Put(key.getBytes()); - for(int i = 0; i < record.getArity(); ++i) { - Object fieldVal = record.getField(i); - if (fieldVal == null) { - continue; - } - byte[] val = fieldVal.toString().getBytes(); - byte[] cf = families[i].getBytes(); - byte[] qualifier = qualifiers[i].getBytes(); - - put.addColumn(cf, qualifier, val); - } - try { table.put(put); } catch (IOException e) { @@ -137,14 +126,65 @@ public void writeRecord(Tuple2 tuple2) { LOG.info(record.toString()); } outRecords.inc(); + } + protected void dealDelete(Tuple2 tupleTrans) { + Row record = tupleTrans.getField(1); + String rowKey = buildRowKey(record); + if (!StringUtils.isEmpty(rowKey)) { + Delete delete = new Delete(Bytes.toBytes(rowKey)); + try { + table.delete(delete); + } catch (IOException e) { + outDirtyRecords.inc(); + if (outDirtyRecords.getCount() % dirtyDataPrintFrequency == 0 || LOG.isDebugEnabled()) { + LOG.error("record insert failed ..", record.toString()); + LOG.error("", e); + } + } + if (outRecords.getCount() % rowLenth == 0) { + LOG.info(record.toString()); + } + outRecords.inc(); + } + } + + private Put getPutByRow(Row record) { + String rowKey = buildRowKey(record); + if (StringUtils.isEmpty(rowKey)) { + return null; + } + Put put = new Put(rowKey.getBytes()); + for (int i = 0; i < record.getArity(); ++i) { + Object fieldVal = record.getField(i); + if (fieldVal == null) { + continue; + } + byte[] val = fieldVal.toString().getBytes(); + byte[] cf = families[i].getBytes(); + byte[] qualifier = qualifiers[i].getBytes(); + + put.addColumn(cf, qualifier, val); + } + return put; + } + + private String buildRowKey(Row record) { + List rowKeyValues = getRowKeyValues(record); + // all rowkey not null + if (rowKeyValues.size() != rowkey.length) { + LOG.error("row key value must not null,record is ..", record); + outDirtyRecords.inc(); + return ""; + } + return StringUtils.join(rowKeyValues, "-"); } private List getRowKeyValues(Row record) { List rowKeyValues = Lists.newArrayList(); for (int i = 0; i < rowkey.length; ++i) { String colName = rowkey[i]; - int rowKeyIndex = 0; //rowkey index + int rowKeyIndex = 0; for (; rowKeyIndex < columnNames.length; ++rowKeyIndex) { if (columnNames[rowKeyIndex].equals(colName)) { break; @@ -168,13 +208,14 @@ private List getRowKeyValues(Row record) { @Override public void close() throws IOException { - if(conn != null) { + if (conn != null) { conn.close(); conn = null; } } - private HbaseOutputFormat() {} + private HbaseOutputFormat() { + } public static HbaseOutputFormatBuilder buildHbaseOutputFormat() { return new HbaseOutputFormatBuilder(); @@ -193,7 +234,7 @@ public HbaseOutputFormatBuilder setHost(String host) { return this; } - public HbaseOutputFormatBuilder setZkParent(String parent){ + public HbaseOutputFormatBuilder setZkParent(String parent) { format.zkParent = parent; return this; } @@ -209,6 +250,11 @@ public HbaseOutputFormatBuilder setRowkey(String[] rowkey) { return this; } + public HbaseOutputFormatBuilder setUpdateMode(String updateMode) { + format.updateMode = updateMode; + return this; + } + public HbaseOutputFormatBuilder setColumnNames(String[] columnNames) { format.columnNames = columnNames; return this; diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index 0dd3c801b..447b02921 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -51,6 +51,7 @@ public class HbaseSink implements RetractStreamTableSink, IStreamSinkGener< protected String port; protected String parent; protected String tableName; + protected String updateMode; protected String[] rowkey; public HbaseSink() { @@ -66,17 +67,20 @@ public HbaseSink genStreamSink(TargetTableInfo targetTableInfo) { this.tableName = hbaseTableInfo.getTableName(); this.rowkey = hbaseTableInfo.getRowkey(); this.columnNameFamily = hbaseTableInfo.getColumnNameFamily(); + this.updateMode = hbaseTableInfo.getUpdateMode(); return this; } @Override public void emitDataStream(DataStream> dataStream) { HbaseOutputFormat.HbaseOutputFormatBuilder builder = HbaseOutputFormat.buildHbaseOutputFormat(); - builder.setHost(this.zookeeperQuorum).setZkParent(this.parent).setTable(this.tableName); - - builder.setRowkey(rowkey); - builder.setColumnNames(fieldNames); - builder.setColumnNameFamily(columnNameFamily); + builder.setHost(this.zookeeperQuorum) + .setZkParent(this.parent) + .setTable(this.tableName) + .setRowkey(rowkey) + .setUpdateMode(updateMode) + .setColumnNames(fieldNames) + .setColumnNameFamily(columnNameFamily); HbaseOutputFormat outputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(outputFormat); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index 1afdbd481..f1ae72b6b 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -21,6 +21,7 @@ package com.dtstack.flink.sql.sink.hbase.table; +import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.table.AbsTableParser; import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.DtStringUtil; @@ -50,6 +51,8 @@ public class HbaseSinkParser extends AbsTableParser { public static final String TABLE_NAME_KEY = "tableName"; + public static final String UPDATE_KEY = "updateMode"; + @Override protected boolean fieldNameNeedsUpperCase() { return false; @@ -66,6 +69,8 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Tue, 4 Feb 2020 11:56:00 +0800 Subject: [PATCH 012/523] fix conflict --- .../sql/sink/cassandra/CassandraOutputFormat.java | 2 +- .../flink/sql/sink/console/ConsoleOutputFormat.java | 2 +- .../flink/sql/outputformat/DtRichOutputFormat.java | 6 +++--- .../sink/rdb/format/AbstractJDBCOutputFormat.java | 5 ++--- .../sql/sink/rdb/format/RetractJDBCOutputFormat.java | 0 .../flink/sql/sink/rdb/writer/AppendOnlyWriter.java | 6 +++--- .../flink/sql/sink/rdb/writer/UpsertWriter.java | 12 ++++++------ 7 files changed, 16 insertions(+), 17 deletions(-) delete mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java index 004124522..59d849426 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java @@ -69,7 +69,7 @@ * @see Tuple * @see DriverManager */ -public class CassandraOutputFormat extends DtRichOutputFormat { +public class CassandraOutputFormat extends DtRichOutputFormat { private static final long serialVersionUID = -7994311331389155692L; private static final Logger LOG = LoggerFactory.getLogger(CassandraOutputFormat.class); diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java index b3e5f655a..3fb23b2b4 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java @@ -37,7 +37,7 @@ * * @author xuqianjin */ -public class ConsoleOutputFormat extends DtRichOutputFormat { +public class ConsoleOutputFormat extends DtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(ConsoleOutputFormat.class); diff --git a/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java b/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java index 342e8b5e8..1fc40c13b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java +++ b/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java @@ -29,9 +29,9 @@ */ public abstract class DtRichOutputFormat extends RichOutputFormat{ - protected transient Counter outRecords; - protected transient Counter outDirtyRecords; - protected transient Meter outRecordsRate; + public transient Counter outRecords; + public transient Counter outDirtyRecords; + public transient Meter outRecordsRate; protected static int ROW_PRINT_FREQUENCY = 1000; protected static int DIRTY_PRINT_FREQUENCY = 1000; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java index 81494390b..73f17d8fd 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java @@ -18,9 +18,8 @@ package com.dtstack.flink.sql.sink.rdb.format; -import com.dtstack.flink.sql.sink.MetricOutputFormat; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import com.dtstack.flink.sql.util.JDBCUtils; -import org.apache.flink.api.common.io.RichOutputFormat; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -37,7 +36,7 @@ * @see Row * @see DriverManager */ -public abstract class AbstractJDBCOutputFormat extends MetricOutputFormat { +public abstract class AbstractJDBCOutputFormat extends DtRichOutputFormat { private static final long serialVersionUID = 1L; public static final int DEFAULT_FLUSH_MAX_SIZE = 100; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java deleted file mode 100644 index e69de29bb..000000000 diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 741d0a3e2..59eee0858 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.writer; -import com.dtstack.flink.sql.sink.MetricOutputFormat; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -47,9 +47,9 @@ public class AppendOnlyWriter implements JDBCWriter { private transient PreparedStatement statement; private transient List rows; // only use metric - private transient MetricOutputFormat metricOutputFormat; + private transient DtRichOutputFormat metricOutputFormat; - public AppendOnlyWriter(String insertSQL, int[] fieldTypes, MetricOutputFormat metricOutputFormat) { + public AppendOnlyWriter(String insertSQL, int[] fieldTypes, DtRichOutputFormat metricOutputFormat) { this.insertSQL = insertSQL; this.fieldTypes = fieldTypes; this.metricOutputFormat = metricOutputFormat; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java index 2e27439f0..c7074de59 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.writer; -import com.dtstack.flink.sql.sink.MetricOutputFormat; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; @@ -58,7 +58,7 @@ public static UpsertWriter create( String[] partitionFields, boolean objectReuse, boolean allReplace, - MetricOutputFormat metricOutputFormat) { + DtRichOutputFormat metricOutputFormat) { checkNotNull(keyFields); @@ -96,9 +96,9 @@ public static UpsertWriter create( private transient Map> keyToRows; private transient PreparedStatement deleteStatement; // only use metric - private transient MetricOutputFormat metricOutputFormat; + private transient DtRichOutputFormat metricOutputFormat; - private UpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSQL, boolean objectReuse, MetricOutputFormat metricOutputFormat) { + private UpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSQL, boolean objectReuse, DtRichOutputFormat metricOutputFormat) { this.fieldTypes = fieldTypes; this.pkFields = pkFields; this.pkTypes = pkTypes; @@ -216,7 +216,7 @@ private UpsertWriterUsingUpsertStatement( boolean objectReuse, String deleteSQL, String upsertSQL, - MetricOutputFormat metricOutputFormat) { + DtRichOutputFormat metricOutputFormat) { super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); this.upsertSQL = upsertSQL; } @@ -274,7 +274,7 @@ private UpsertWriterUsingInsertUpdateStatement( String existSQL, String insertSQL, String updateSQL, - MetricOutputFormat metricOutputFormat) { + DtRichOutputFormat metricOutputFormat) { super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); this.existSQL = existSQL; this.insertSQL = insertSQL; From d3eba0dfa8c102861b9eb33a538f8fdec01194a4 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 11 Feb 2020 15:47:09 +0800 Subject: [PATCH 013/523] async use crow --- .../side/cassandra/CassandraAsyncReqRow.java | 25 +++++------ .../sql/sink/console/ConsoleOutputFormat.java | 1 + .../dtstack/flink/sql/side/AsyncReqRow.java | 16 ++++---- .../dtstack/flink/sql/side/SideSqlExec.java | 41 ++++++++++++++----- .../sql/side/hbase/HbaseAsyncReqRow.java | 37 +++++++++-------- .../rowkeydealer/AbsRowKeyModeDealer.java | 11 ++--- .../PreRowKeyModeDealerDealer.java | 13 +++--- .../rowkeydealer/RowKeyEqualModeDealer.java | 7 ++-- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 41 ++++++++++--------- .../sql/side/mongo/MongoAsyncReqRow.java | 21 +++++----- .../sql/side/rdb/async/RdbAsyncReqRow.java | 29 ++++++------- .../sql/side/redis/RedisAsyncReqRow.java | 33 +++++++-------- 12 files changed, 152 insertions(+), 123 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index ab266c7b4..a60116e34 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -47,6 +47,7 @@ import org.apache.flink.configuration.Configuration; import com.google.common.collect.Lists; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -160,17 +161,17 @@ private void connCassandraDB(CassandraSideTableInfo tableInfo) { } @Override - public void asyncInvoke(Row input, ResultFuture resultFuture) throws Exception { - Row inputRow = Row.copy(input); + public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + CRow inputCopy = new CRow(input.row(), input.change()); JsonArray inputParams = new JsonArray(); StringBuffer stringBuffer = new StringBuffer(); String sqlWhere = " where "; for (int i = 0; i < sideInfo.getEqualFieldList().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = inputRow.getField(conValIndex); + Object equalObj = inputCopy.row().getField(conValIndex); if (equalObj == null) { - dealMissKey(inputRow, resultFuture); + dealMissKey(inputCopy, resultFuture); return; } inputParams.add(equalObj); @@ -194,13 +195,13 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except if (val != null) { if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(inputRow, resultFuture); + dealMissKey(inputCopy, resultFuture); return; } else if (ECacheContentType.MultiLine == val.getType()) { - List rowList = Lists.newArrayList(); + List rowList = Lists.newArrayList(); for (Object jsonArray : (List) val.getContent()) { - Row row = fillData(inputRow, jsonArray); - rowList.add(row); + Row row = fillData(inputCopy.row(), jsonArray); + rowList.add(new CRow(row, inputCopy.change())); } resultFuture.complete(rowList); } else { @@ -238,20 +239,20 @@ public void onSuccess(List rows) { cluster.closeAsync(); if (rows.size() > 0) { List cacheContent = Lists.newArrayList(); - List rowList = Lists.newArrayList(); + List rowList = Lists.newArrayList(); for (com.datastax.driver.core.Row line : rows) { - Row row = fillData(inputRow, line); + Row row = fillData(inputCopy.row(), line); if (openCache()) { cacheContent.add(line); } - rowList.add(row); + rowList.add(new CRow(row,inputCopy.change())); } resultFuture.complete(rowList); if (openCache()) { putCache(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); } } else { - dealMissKey(inputRow, resultFuture); + dealMissKey(inputCopy, resultFuture); if (openCache()) { putCache(key, CacheMissVal.getMissKeyObj()); } diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java index b3e5f655a..57e925075 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java @@ -56,6 +56,7 @@ public void open(int taskNumber, int numTasks) throws IOException { @Override public void writeRecord(Tuple2 tuple2) throws IOException { + System.out.println("received oriainal data:" + tuple2); Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); if (!retract) { diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index f0b79ee22..0415102ab 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -31,11 +31,11 @@ import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collection; import java.util.Collections; import java.util.concurrent.TimeoutException; @@ -47,7 +47,7 @@ * @author xuchao */ -public abstract class AsyncReqRow extends RichAsyncFunction implements ISideReqRow { +public abstract class AsyncReqRow extends RichAsyncFunction implements ISideReqRow { private static final Logger LOG = LoggerFactory.getLogger(AsyncReqRow.class); private static final long serialVersionUID = 2098635244857937717L; @@ -97,12 +97,12 @@ protected boolean openCache(){ return sideInfo.getSideCache() != null; } - protected void dealMissKey(Row input, ResultFuture resultFuture){ + protected void dealMissKey(CRow input, ResultFuture resultFuture){ if(sideInfo.getJoinType() == JoinType.LEFT){ //Reserved left table data try { - Row row = fillData(input, null); - resultFuture.complete(Collections.singleton(row)); + Row row = fillData(input.row(), null); + resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { dealFillDataError(resultFuture, e, input); } @@ -118,8 +118,8 @@ protected void dealCacheData(String key, CacheObj missKeyObj) { } @Override - public void timeout(Row input, ResultFuture resultFuture) throws Exception { - StreamRecordQueueEntry future = (StreamRecordQueueEntry)resultFuture; + public void timeout(CRow input, ResultFuture resultFuture) throws Exception { + StreamRecordQueueEntry future = (StreamRecordQueueEntry)resultFuture; try { if (null == future.get()) { resultFuture.completeExceptionally(new TimeoutException("Async function call has timed out.")); @@ -130,7 +130,7 @@ public void timeout(Row input, ResultFuture resultFuture) throws Exception } - protected void dealFillDataError(ResultFuture resultFuture, Exception e, Object sourceData) { + protected void dealFillDataError(ResultFuture resultFuture, Exception e, Object sourceData) { LOG.debug("source data {} join side table error ", sourceData); LOG.debug("async buid row error..{}", e); parseErrorRecords.inc(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 0fddcbc37..390b24f8f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -54,12 +54,17 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.table.api.StreamQueryConfig; import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.runtime.CRowKeySelector; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Timestamp; +import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -773,21 +778,24 @@ private void joinFun(Object pollObj, Map localTableCache, RowTypeInfo typeInfo = new RowTypeInfo(targetTable.getSchema().getTypes(), targetTable.getSchema().getColumnNames()); - DataStream adaptStream = tableEnv.toRetractStream(targetTable, org.apache.flink.types.Row.class) - .map((Tuple2 f0) -> { return f0.f1; }) - .returns(Row.class); + DataStream adaptStream = tableEnv.toRetractStream(targetTable, org.apache.flink.types.Row.class) + .map((Tuple2 tp2) -> { + return new CRow(tp2.f1, tp2.f0); + }).returns(CRow.class); //join side table before keyby ===> Reducing the size of each dimension table cache of async - if(sideTableInfo.isPartitionedJoin()){ - RowTypeInfo leftTableOutType = buildLeftTableOutType(leftTypeInfo); - adaptStream.getTransformation().setOutputType(leftTableOutType); + if (sideTableInfo.isPartitionedJoin()) { + // RowTypeInfo leftTableOutType = buildLeftTableOutType(leftTypeInfo); + // adaptStream.getTransformation().setOutputType(leftTableOutType); + List leftJoinColList = getConditionFields(joinInfo.getCondition(), joinInfo.getLeftTableAlias(), sideTableInfo); - String[] leftJoinColArr = leftJoinColList.toArray(new String[leftJoinColList.size()]); - adaptStream = adaptStream.keyBy(leftJoinColArr); + List fieldNames = Arrays.asList(targetTable.getSchema().getFieldNames()); + int[] keyIndex = leftJoinColList.stream().mapToInt(fieldNames::indexOf).toArray(); + adaptStream = adaptStream.keyBy(new CRowKeySelector(keyIndex, projectedTypeInfo(keyIndex, targetTable.getSchema()))); } - DataStream dsOut = null; + DataStream dsOut = null; if(ECacheType.ALL.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ dsOut = SideWithAllCacheOperator.getSideJoinDataStream(adaptStream, sideTableInfo.getType(), localSqlPluginPath, typeInfo, joinInfo, sideJoinFieldInfo, sideTableInfo); }else{ @@ -796,7 +804,10 @@ private void joinFun(Object pollObj, Map localTableCache, HashBasedTable mappingTable = HashBasedTable.create(); RowTypeInfo sideOutTypeInfo = buildOutRowTypeInfo(sideJoinFieldInfo, mappingTable); - dsOut.getTransformation().setOutputType(sideOutTypeInfo); + + CRowTypeInfo cRowTypeInfo = new CRowTypeInfo(sideOutTypeInfo); + dsOut.getTransformation().setOutputType(cRowTypeInfo); + String targetTableName = joinInfo.getNewTableName(); String targetTableAlias = joinInfo.getNewTableAlias(); @@ -808,10 +819,18 @@ private void joinFun(Object pollObj, Map localTableCache, replaceInfoList.add(replaceInfo); if (!tableEnv.isRegistered(joinInfo.getNewTableName())){ - tableEnv.registerDataStream(joinInfo.getNewTableName(), dsOut, String.join(",", sideOutTypeInfo.getFieldNames())); + tableEnv.registerDataStream(joinInfo.getNewTableName(), dsOut); } } + private TypeInformation projectedTypeInfo(int[] fields, TableSchema schema) { + String[] fieldNames = schema.getFieldNames(); + TypeInformation[] fieldTypes = schema.getFieldTypes(); + + String[] projectedNames = Arrays.stream(fields).mapToObj(i -> fieldNames[i]).toArray(String[]::new); + TypeInformation[] projectedTypes = Arrays.stream(fields).mapToObj(i -> fieldTypes[i]).toArray(TypeInformation[]::new); + return new RowTypeInfo(projectedTypes, projectedNames); + } private boolean checkFieldsInfo(CreateTmpTableParser.SqlParserResult result, Table table) { diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 4b5ae9a25..251b88034 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -36,6 +36,7 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.hbase.async.HBaseClient; @@ -122,14 +123,14 @@ public void open(Configuration parameters) throws Exception { } @Override - public void asyncInvoke(Row input, ResultFuture resultFuture) throws Exception { - Row inputRow = Row.copy(input); + public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + CRow inputCopy = new CRow(input.row(), input.change()); Map refData = Maps.newHashMap(); for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = inputRow.getField(conValIndex); + Object equalObj = inputCopy.row().getField(conValIndex); if(equalObj == null){ - dealMissKey(inputRow, resultFuture); + dealMissKey(inputCopy, resultFuture); return; } refData.put(sideInfo.getEqualFieldList().get(i), equalObj); @@ -138,34 +139,34 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except String rowKeyStr = ((HbaseAsyncSideInfo)sideInfo).getRowKeyBuilder().getRowKey(refData); //get from cache - if(openCache()){ + if (openCache()) { CacheObj val = getFromCache(rowKeyStr); - if(val != null){ - if(ECacheContentType.MissVal == val.getType()){ - dealMissKey(inputRow, resultFuture); + if (val != null) { + if (ECacheContentType.MissVal == val.getType()) { + dealMissKey(inputCopy, resultFuture); return; - }else if(ECacheContentType.SingleLine == val.getType()){ + } else if (ECacheContentType.SingleLine == val.getType()) { try { - Row row = fillData(inputRow, val); - resultFuture.complete(Collections.singleton(row)); + Row row = fillData(inputCopy.row(), val); + resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, inputRow); + dealFillDataError(resultFuture, e, inputCopy); } - }else if(ECacheContentType.MultiLine == val.getType()){ + } else if (ECacheContentType.MultiLine == val.getType()) { try { - for(Object one : (List)val.getContent()){ - Row row = fillData(inputRow, one); - resultFuture.complete(Collections.singleton(row)); + for (Object one : (List) val.getContent()) { + Row row = fillData(inputCopy.row(), one); + resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); } } catch (Exception e) { - dealFillDataError(resultFuture, e, inputRow); + dealFillDataError(resultFuture, e, inputCopy); } } return; } } - rowKeyMode.asyncGetData(tableName, rowKeyStr, inputRow, resultFuture, sideInfo.getSideCache()); + rowKeyMode.asyncGetData(tableName, rowKeyStr, inputCopy, resultFuture, sideInfo.getSideCache()); } @Override diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java index 07590702b..1506bc440 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java @@ -25,6 +25,7 @@ import org.apache.calcite.sql.JoinType; import com.google.common.collect.Maps; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.hbase.async.HBaseClient; @@ -72,12 +73,12 @@ public AbsRowKeyModeDealer(Map colRefType, String[] colNames, HB this.sideFieldIndex = sideFieldIndex; } - protected void dealMissKey(Row input, ResultFuture resultFuture){ + protected void dealMissKey(CRow input, ResultFuture resultFuture){ if(joinType == JoinType.LEFT){ try { //保留left 表数据 - Row row = fillData(input, null); - resultFuture.complete(Collections.singleton(row)); + Row row = fillData(input.row(), null); + resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { resultFuture.completeExceptionally(e); } @@ -109,6 +110,6 @@ protected Row fillData(Row input, Object sideInput){ return row; } - public abstract void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFuture resultFuture, - AbsSideCache sideCache); + public abstract void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, + AbsSideCache sideCache); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java index 552bb354c..a41ffe916 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java @@ -30,6 +30,7 @@ import org.apache.calcite.sql.JoinType; import com.google.common.collect.Lists; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.hbase.async.BinaryPrefixComparator; import org.hbase.async.Bytes; @@ -65,7 +66,7 @@ public PreRowKeyModeDealerDealer(Map colRefType, String[] colNam } @Override - public void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFuture resultFuture, + public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, AbsSideCache sideCache) { Scanner prefixScanner = hBaseClient.newScanner(tableName); ScanFilter scanFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(Bytes.UTF8(rowKeyStr))); @@ -79,7 +80,7 @@ public void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFu } - private String dealOneRow(ArrayList> args, String rowKeyStr, Row input, ResultFuture resultFuture, AbsSideCache sideCache) { + private String dealOneRow(ArrayList> args, String rowKeyStr, CRow input, ResultFuture resultFuture, AbsSideCache sideCache) { if(args == null || args.size() == 0){ dealMissKey(input, resultFuture); if (openCache) { @@ -88,7 +89,7 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, } List cacheContent = Lists.newArrayList(); - List rowList = Lists.newArrayList(); + List rowList = Lists.newArrayList(); for(List oneRow : args){ try { @@ -117,11 +118,11 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, sideVal.add(val); } - Row row = fillData(input, sideVal); + Row row = fillData(input.row(), sideVal); if (openCache) { cacheContent.add(sideVal); } - rowList.add(row); + rowList.add(new CRow(row, input.change())); } }catch (Exception e) { resultFuture.completeExceptionally(e); @@ -144,7 +145,7 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, return ""; } - private String dealFail(Object arg2, Row input, ResultFuture resultFuture){ + private String dealFail(Object arg2, CRow input, ResultFuture resultFuture){ LOG.error("record:" + input); LOG.error("get side record exception:" + arg2); resultFuture.complete(null); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index f91e8924a..b20c316db 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -30,6 +30,7 @@ import org.apache.calcite.sql.JoinType; import com.google.common.collect.Lists; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.hbase.async.GetRequest; import org.hbase.async.HBaseClient; @@ -60,7 +61,7 @@ public RowKeyEqualModeDealer(Map colRefType, String[] colNames, @Override - public void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFuture resultFuture, + public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, AbsSideCache sideCache){ //TODO 是否有查询多个col family 和多个col的方法 GetRequest getRequest = new GetRequest(tableName, rowKeyStr); @@ -92,11 +93,11 @@ public void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFu sideVal.add(val); } - Row row = fillData(input, sideVal); + Row row = fillData(input.row(), sideVal); if(openCache){ sideCache.putCache(rowKeyStr, CacheObj.buildCacheObj(ECacheContentType.SingleLine, row)); } - resultFuture.complete(Collections.singleton(row)); + resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { resultFuture.completeExceptionally(e); } diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index aa85857df..4a7457418 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -13,6 +13,7 @@ import org.apache.flink.configuration.Configuration; import com.google.common.collect.Maps; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; @@ -120,8 +121,8 @@ private void connKuDu() throws KuduException { @Override - public void asyncInvoke(Row input, ResultFuture resultFuture) throws Exception { - Row inputRow = Row.copy(input); + public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + CRow inputCopy = new CRow(input.row(), input.change()); //scannerBuilder 设置为null重新加载过滤条件 scannerBuilder = null; connKuDu(); @@ -129,9 +130,9 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except Schema schema = table.getSchema(); // @wenbaoup fix bug for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { - Object equalObj = inputRow.getField(sideInfo.getEqualValIndex().get(i)); + Object equalObj = inputCopy.row().getField(sideInfo.getEqualValIndex().get(i)); if (equalObj == null) { - dealMissKey(inputRow, resultFuture); + dealMissKey(inputCopy, resultFuture); return; } //增加过滤条件 @@ -159,25 +160,25 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except CacheObj val = getFromCache(key); if (val != null) { if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(inputRow, resultFuture); + dealMissKey(inputCopy, resultFuture); return; } else if (ECacheContentType.SingleLine == val.getType()) { try { - Row row = fillData(inputRow, val); - resultFuture.complete(Collections.singleton(row)); + Row row = fillData(inputCopy.row(), val); + resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, inputRow); + dealFillDataError(resultFuture, e, inputCopy); } } else if (ECacheContentType.MultiLine == val.getType()) { try { - List rowList = Lists.newArrayList(); + List rowList = Lists.newArrayList(); for (Object jsonArray : (List) val.getContent()) { - Row row = fillData(inputRow, jsonArray); - rowList.add(row); + Row row = fillData(inputCopy.row(), jsonArray); + rowList.add(new CRow(row, inputCopy.change())); } resultFuture.complete(rowList); } catch (Exception e) { - dealFillDataError(resultFuture, e, inputRow); + dealFillDataError(resultFuture, e, inputCopy); } } else { resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); @@ -187,10 +188,10 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except } List> cacheContent = Lists.newArrayList(); AsyncKuduScanner asyncKuduScanner = scannerBuilder.build(); - List rowList = Lists.newArrayList(); + List rowList = Lists.newArrayList(); Deferred data = asyncKuduScanner.nextRows(); //从之前的同步修改为调用异步的Callback - data.addCallbackDeferring(new GetListRowCB(inputRow, cacheContent, rowList, asyncKuduScanner, resultFuture, key)); + data.addCallbackDeferring(new GetListRowCB(inputCopy, cacheContent, rowList, asyncKuduScanner, resultFuture, key)); } @@ -243,18 +244,18 @@ public void close() throws Exception { } class GetListRowCB implements Callback>, RowResultIterator> { - private Row input; + private CRow input; private List> cacheContent; - private List rowList; + private List rowList; private AsyncKuduScanner asyncKuduScanner; - private ResultFuture resultFuture; + private ResultFuture resultFuture; private String key; public GetListRowCB() { } - GetListRowCB(Row input, List> cacheContent, List rowList, AsyncKuduScanner asyncKuduScanner, ResultFuture resultFuture, String key) { + GetListRowCB(CRow input, List> cacheContent, List rowList, AsyncKuduScanner asyncKuduScanner, ResultFuture resultFuture, String key) { this.input = input; this.cacheContent = cacheContent; this.rowList = rowList; @@ -274,11 +275,11 @@ public Deferred> call(RowResultIterator results) throws Exception { KuduUtil.setMapValue(columnSchema.getType(), oneRow, sideFieldName, result); } } - Row row = fillData(input, oneRow); + Row row = fillData(input.row(), oneRow); if (openCache()) { cacheContent.add(oneRow); } - rowList.add(row); + rowList.add(new CRow(row, input.change())); } if (asyncKuduScanner.hasMoreRows()) { return asyncKuduScanner.nextRows().addCallbackDeferring(this); diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 1558f8bec..8ac7d83ee 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -44,6 +44,7 @@ import com.google.common.collect.Lists; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.bson.Document; @@ -123,14 +124,14 @@ public void connMongoDB() throws Exception { } @Override - public void asyncInvoke(Row input, ResultFuture resultFuture) throws Exception { - Row inputRow = Row.copy(input); + public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + CRow inputCopy = new CRow(input.row(), input.change()); BasicDBObject basicDBObject = new BasicDBObject(); for (int i = 0; i < sideInfo.getEqualFieldList().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = inputRow.getField(conValIndex); + Object equalObj = inputCopy.row().getField(conValIndex); if (equalObj == null) { - dealMissKey(inputRow, resultFuture); + dealMissKey(inputCopy, resultFuture); return; } basicDBObject.put(sideInfo.getEqualFieldList().get(i), equalObj); @@ -154,13 +155,13 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except if (val != null) { if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(inputRow, resultFuture); + dealMissKey(inputCopy, resultFuture); return; } else if (ECacheContentType.MultiLine == val.getType()) { - List rowList = Lists.newArrayList(); + List rowList = Lists.newArrayList(); for (Object jsonArray : (List) val.getContent()) { - Row row = fillData(inputRow, jsonArray); - rowList.add(row); + Row row = fillData(inputCopy.row(), jsonArray); + rowList.add(new CRow(row, inputCopy.change())); } resultFuture.complete(rowList); } else { @@ -176,11 +177,11 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except @Override public void apply(final Document document) { atomicInteger.incrementAndGet(); - Row row = fillData(inputRow, document); + Row row = fillData(inputCopy.row(), document); if (openCache()) { cacheContent.add(document); } - resultFuture.complete(Collections.singleton(row)); + resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); } }; SingleResultCallback callbackWhenFinished = new SingleResultCallback() { diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index f3b5db422..7123e257b 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -34,6 +34,7 @@ import io.vertx.ext.sql.SQLConnection; import com.google.common.collect.Lists; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -79,13 +80,13 @@ public RdbAsyncReqRow(SideInfo sideInfo) { } @Override - public void asyncInvoke(Row input, ResultFuture resultFuture) throws Exception { - Row inputRow = Row.copy(input); + public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + CRow copyCrow = new CRow(input.row(), input.change()); JsonArray inputParams = new JsonArray(); for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = inputRow.getField(conValIndex); + Object equalObj = copyCrow.row().getField(conValIndex); if (equalObj == null) { - dealMissKey(inputRow, resultFuture); + dealMissKey(copyCrow, resultFuture); return; } inputParams.add(equalObj); @@ -96,14 +97,14 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except CacheObj val = getFromCache(key); if (val != null) { if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(inputRow, resultFuture); + dealMissKey(copyCrow, resultFuture); return; } else if (ECacheContentType.MultiLine == val.getType()) { try { - List rowList = getRows(inputRow, null, (List) val.getContent()); + List rowList = getRows(copyCrow, null, (List) val.getContent()); resultFuture.complete(rowList); } catch (Exception e) { - dealFillDataError(resultFuture, e, inputRow); + dealFillDataError(resultFuture, e, copyCrow); } } else { resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); @@ -131,14 +132,14 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except List results = rs.result().getResults(); if (results.size() > 0) { try { - List rowList = getRows(inputRow, cacheContent, results); + List rowList = getRows(copyCrow, cacheContent, results); dealCacheData(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); resultFuture.complete(rowList); } catch (Exception e){ - dealFillDataError(resultFuture, e, inputRow); + dealFillDataError(resultFuture, e, copyCrow); } } else { - dealMissKey(inputRow, resultFuture); + dealMissKey(copyCrow, resultFuture); dealCacheData(key, CacheMissVal.getMissKeyObj()); } @@ -152,14 +153,14 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except }); } - protected List getRows(Row inputRow, List cacheContent, List results) { - List rowList = Lists.newArrayList(); + protected List getRows(CRow inputRow, List cacheContent, List results) { + List rowList = Lists.newArrayList(); for (JsonArray line : results) { - Row row = fillData(inputRow, line); + Row row = fillData(inputRow.row(), line); if (null != cacheContent && openCache()) { cacheContent.add(line); } - rowList.add(row); + rowList.add(new CRow(row, inputRow.change())); } return rowList; } diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index bb67c3c47..d20875e01 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -36,6 +36,7 @@ import com.google.common.collect.Maps; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Collections; @@ -116,14 +117,14 @@ public Row fillData(Row input, Object sideInput) { } @Override - public void asyncInvoke(Row input, ResultFuture resultFuture) throws Exception { - Row inputRow = Row.copy(input); + public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + CRow inputCopy = new CRow(input.row(),input.change()); List keyData = Lists.newLinkedList(); for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = inputRow.getField(conValIndex); + Object equalObj = inputCopy.row().getField(conValIndex); if(equalObj == null){ - dealMissKey(inputRow, resultFuture); + dealMissKey(inputCopy, resultFuture); return; } String value = equalObj.toString(); @@ -137,14 +138,14 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except CacheObj val = getFromCache(key); if(val != null){ if(ECacheContentType.MissVal == val.getType()){ - dealMissKey(inputRow, resultFuture); + dealMissKey(inputCopy, resultFuture); return; }else if(ECacheContentType.MultiLine == val.getType()){ try { - Row row = fillData(inputRow, val.getContent()); - resultFuture.complete(Collections.singleton(row)); + Row row = fillData(inputCopy.row(), val.getContent()); + resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, inputRow); + dealFillDataError(resultFuture, e, inputCopy); } }else{ RuntimeException exception = new RuntimeException("not support cache obj type " + val.getType()); @@ -157,8 +158,8 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except Map keyValue = Maps.newHashMap(); List value = async.keys(key + ":*").get(); String[] values = value.toArray(new String[value.size()]); - if (values.length == 0){ - dealMissKey(inputRow, resultFuture); + if (values.length == 0) { + dealMissKey(inputCopy, resultFuture); } else { RedisFuture>> future = ((RedisStringAsyncCommands) async).mget(values); future.thenAccept(new Consumer>>() { @@ -171,15 +172,15 @@ public void accept(List> keyValues) { keyValue.put(splitKeys[3], keyValues.get(i).getValue()); } try { - Row row = fillData(inputRow, keyValue); - dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.MultiLine, keyValue)); - resultFuture.complete(Collections.singleton(row)); + Row row = fillData(inputCopy.row(), keyValue); + dealCacheData(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, keyValue)); + resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, inputRow); + dealFillDataError(resultFuture, e, inputCopy); } } else { - dealMissKey(inputRow, resultFuture); - dealCacheData(key,CacheMissVal.getMissKeyObj()); + dealMissKey(inputCopy, resultFuture); + dealCacheData(key, CacheMissVal.getMissKeyObj()); } } }); From 43aaad0c10eb9637f93f2f0014e47344c576a668 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Wed, 12 Feb 2020 02:56:11 +0800 Subject: [PATCH 014/523] =?UTF-8?q?=E5=A2=9E=E5=8A=A0elasticsearch6-sink?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/elasticsearch6Sink.md | 56 ++++++ elasticsearch6/elasticsearch6-sink/pom.xml | 126 +++++++++++++ .../sink/elasticsearch6/CustomerSinkFunc.java | 119 ++++++++++++ .../elasticsearch6/Elasticsearch6Sink.java | 173 ++++++++++++++++++ .../sql/sink/elasticsearch6/Es6Util.java | 64 +++++++ .../ExtendES6ApiCallBridge.java | 139 ++++++++++++++ .../MetricElasticsearch6Sink.java | 70 +++++++ .../table/ElasticsearchSinkParser.java | 74 ++++++++ .../table/ElasticsearchTableInfo.java | 139 ++++++++++++++ elasticsearch6/pom.xml | 36 ++++ pom.xml | 1 + 11 files changed, 997 insertions(+) create mode 100644 docs/elasticsearch6Sink.md create mode 100644 elasticsearch6/elasticsearch6-sink/pom.xml create mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java create mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Elasticsearch6Sink.java create mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Es6Util.java create mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendES6ApiCallBridge.java create mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java create mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchSinkParser.java create mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchTableInfo.java create mode 100644 elasticsearch6/pom.xml diff --git a/docs/elasticsearch6Sink.md b/docs/elasticsearch6Sink.md new file mode 100644 index 000000000..f8a398217 --- /dev/null +++ b/docs/elasticsearch6Sink.md @@ -0,0 +1,56 @@ +## 1.格式： +``` +CREATE TABLE tableName( + colName colType, + bb INT + )WITH( + type ='elasticsearch6', + address ='ip:port[,ip:port]', + cluster='clusterName', + estype ='esType', + index ='index', + id ='num[,num]', + parallelism ='1' + ) +``` +## 2.支持的版本 + elasticsearch 6.8.6 + +## 3.表结构定义 + +|参数名称|含义| +|----|---| +|tableName|在 sql 中使用的名称;即注册到flink-table-env上的名称| +|colName|列名称| +|colType|列类型 [colType支持的类型](colType.md)| + +## 4.参数： +|参数名称|含义|是否必填|默认值| +|----|---|---|----| +|type|表明 输出表类型[mysq|hbase|elasticsearch]|是|| +|address | 连接ES Transport地址(tcp地址)|是|| +|cluster | ES 集群名称 |是|| +|index | 选择的ES上的index名称|是|| +|estype | 选择ES上的type名称|是|| +|id | 生成id的规则(当前是根据指定的字段pos获取字段信息,拼接生成id;|是|| +| |若id为空字符串或索引都超出范围，则随机生成id值)||| +|authMesh | 是否进行用户名密码认证 | 否 | false| +|userName | 用户名 | 否，authMesh='true'时为必填 || +|password | 密码 | 否，authMesh='true'时为必填 || +|parallelism | 并行度设置|否|1| + +## 5.样例： +``` +CREATE TABLE MyResult( + aa INT, + bb INT + )WITH( + type ='elasticsearch', + address ='172.16.10.47:9500', + cluster='es_47_menghan', + estype ='type1', + index ='xc_es_test', + id ='0,1', + parallelism ='1' + ) + ``` \ No newline at end of file diff --git a/elasticsearch6/elasticsearch6-sink/pom.xml b/elasticsearch6/elasticsearch6-sink/pom.xml new file mode 100644 index 000000000..cb5fb753c --- /dev/null +++ b/elasticsearch6/elasticsearch6-sink/pom.xml @@ -0,0 +1,126 @@ + + + + sql.elasticsearch6 + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.sink.elasticsearch6 + elasticsearch6-sink + + + 6.8.6 + + + + + org.apache.flink + flink-streaming-java_2.11 + ${flink.version} + provided + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} + + + + ch.qos.logback + logback-core + 1.1.7 + + + + ch.qos.logback + logback-classic + 1.1.7 + + + + org.apache.logging.log4j + log4j-to-slf4j + 2.7 + + + + org.apache.flink + flink-connector-elasticsearch6_2.11 + ${flink.version} + + + + org.apache.flink + flink-connector-elasticsearch-base_2.11 + ${flink.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java new file mode 100644 index 000000000..091e18ede --- /dev/null +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.elasticsearch6; + +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.metrics.Counter; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction; +import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer; +import org.apache.flink.types.Row; + +import org.apache.commons.lang3.StringUtils; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.Requests; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * @author yinxi + * @date 2020/1/9 - 15:10 + */ +public class CustomerSinkFunc implements ElasticsearchSinkFunction { + + private final Logger logger = LoggerFactory.getLogger(CustomerSinkFunc.class); + + private String index; + + private String type; + + private List idFieldIndexList; + + private List fieldNames; + + private List fieldTypes; + + public transient Counter outRecords; + + /** 默认分隔符为'_' */ + private char sp = '_'; + + public CustomerSinkFunc(String index, String type, List fieldNames, List fieldTypes, List idFieldIndexes){ + this.index = index; + this.type = type; + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + this.idFieldIndexList = idFieldIndexes; + } + + @Override + public void process(Tuple2 tuple2, RuntimeContext ctx, RequestIndexer indexer) { + try{ + Tuple2 tupleTrans = tuple2; + Boolean retract = tupleTrans.getField(0); + Row element = tupleTrans.getField(1); + if(!retract){ + return; + } + + + indexer.add(createIndexRequest(element)); + outRecords.inc(); + }catch (Throwable e){ + logger.error("", e); + } + } + + public void setOutRecords(Counter outRecords) { + this.outRecords = outRecords; + } + + private IndexRequest createIndexRequest(Row element) { + + List idFieldList = new ArrayList<>(); + for(int index : idFieldIndexList){ + if(index >= element.getArity()){ + continue; + } + + idFieldList.add(element.getField(index).toString()); + } + + Map dataMap = Es6Util.rowToJsonMap(element,fieldNames,fieldTypes); + int length = Math.min(element.getArity(), fieldNames.size()); + for(int i=0; i, IStreamSinkGener { + + private final Logger logger = LoggerFactory.getLogger(Elasticsearch6Sink.class); + + private String clusterName; + + private int bulkFlushMaxActions = 1; + + private List esAddressList; + + private String index = ""; + + private String type = ""; + + private List idIndexList; + + protected String[] fieldNames; + + protected String[] columnTypes; + + private TypeInformation[] fieldTypes; + + private int parallelism = -1; + + private ElasticsearchTableInfo esTableInfo; + + + @Override + public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + return this; + } + + @Override + public TupleTypeInfo> getOutputType() { + return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), getRecordType()); + } + + @Override + public TypeInformation getRecordType() { + return new RowTypeInfo(fieldTypes, fieldNames); + } + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } + + + private RichSinkFunction createEsSinkFunction() { + + + Map userConfig = new HashMap<>(); + userConfig.put("cluster.name", clusterName); + // This instructs the sink to emit after every element, otherwise they would be buffered + userConfig.put(org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "" + bulkFlushMaxActions); + List transports = new ArrayList<>(); + + for (String address : esAddressList) { + String[] infoArray = address.split(":"); + int port = 9200; + String host = infoArray[0]; + if (infoArray.length > 1) { + port = Integer.valueOf(infoArray[1].trim()); + } + + try { + transports.add(new HttpHost(host.trim(), port, "http")); + } catch (Exception e) { + logger.error("", e); + throw new RuntimeException(e); + } + } + + CustomerSinkFunc customerSinkFunc = new CustomerSinkFunc(index, type, Arrays.asList(fieldNames), Arrays.asList(columnTypes), idIndexList); + + return new MetricElasticsearch6Sink(userConfig, transports, customerSinkFunc, esTableInfo); + } + + @Override + public void emitDataStream(DataStream> dataStream) { + RichSinkFunction richSinkFunction = createEsSinkFunction(); + DataStreamSink streamSink = dataStream.addSink(richSinkFunction); + if (parallelism > 0) { + streamSink.setParallelism(parallelism); + } + } + + public void setParallelism(int parallelism) { + this.parallelism = parallelism; + } + + public void setBulkFlushMaxActions(int bulkFlushMaxActions) { + this.bulkFlushMaxActions = bulkFlushMaxActions; + } + + @Override + public Elasticsearch6Sink genStreamSink(TargetTableInfo targetTableInfo) { + ElasticsearchTableInfo elasticsearchTableInfo = (ElasticsearchTableInfo) targetTableInfo; + esTableInfo = elasticsearchTableInfo; + clusterName = elasticsearchTableInfo.getClusterName(); + String address = elasticsearchTableInfo.getAddress(); + String[] addr = address.split(","); + esAddressList = Arrays.asList(addr); + index = elasticsearchTableInfo.getIndex(); + type = elasticsearchTableInfo.getEsType(); + String id = elasticsearchTableInfo.getId(); + String[] idField = StringUtils.split(id, ","); + idIndexList = new ArrayList<>(); + + for (int i = 0; i < idField.length; ++i) { + idIndexList.add(Integer.valueOf(idField[i])); + } + + columnTypes = elasticsearchTableInfo.getFieldTypes(); + return this; + } +} diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Es6Util.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Es6Util.java new file mode 100644 index 000000000..ecd07b746 --- /dev/null +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Es6Util.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.elasticsearch6; + +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import com.dtstack.flink.sql.util.DtStringUtil; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author yinxi + * @date 2020/1/9 - 15:08 + */ +public class Es6Util { + + public static Map rowToJsonMap(Row row, List fields, List types) { + Preconditions.checkArgument(row.getArity() == fields.size()); + Map jsonMap = new HashMap<>(); + int i = 0; + for(; i < fields.size(); ++i) { + String field = fields.get(i); + String[] parts = field.split("\\."); + Map currMap = jsonMap; + for(int j = 0; j < parts.length - 1; ++j) { + String key = parts[j]; + if(currMap.get(key) == null) { + currMap.put(key, new HashMap()); + } + currMap = (Map) currMap.get(key); + } + String key = parts[parts.length - 1]; + Object col = row.getField(i); + if(col != null) { + Object value = DtStringUtil.col2string(col, types.get(i)); + currMap.put(key, value); + } + + } + + return jsonMap; + } + + +} diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendES6ApiCallBridge.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendES6ApiCallBridge.java new file mode 100644 index 000000000..4ab596fc9 --- /dev/null +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendES6ApiCallBridge.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.elasticsearch6; + +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchApiCallBridge; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase; +import org.apache.flink.util.Preconditions; + +import com.dtstack.flink.sql.sink.elasticsearch6.table.ElasticsearchTableInfo; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.elasticsearch.action.bulk.BackoffPolicy; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkProcessor; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.TimeValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * @author yinxi + * @date 2020/1/9 - 15:09 + */ +public class ExtendES6ApiCallBridge implements ElasticsearchApiCallBridge { + + private static final Logger LOG = LoggerFactory.getLogger(ExtendES6ApiCallBridge.class); + + private final List HttpAddresses; + + protected ElasticsearchTableInfo es6TableInfo; + + public ExtendES6ApiCallBridge(List HttpAddresses, ElasticsearchTableInfo es6TableInfo) { + Preconditions.checkArgument(HttpAddresses != null && !HttpAddresses.isEmpty()); + this.HttpAddresses = HttpAddresses; + this.es6TableInfo = es6TableInfo; + } + + @Override + public RestHighLevelClient createClient(Map clientConfig) { + + RestClientBuilder restClientBuilder = RestClient.builder(HttpAddresses.toArray(new HttpHost[HttpAddresses.size()])); + if (es6TableInfo.isAuthMesh()) { + // 进行用户和密码认证 + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(es6TableInfo.getUserName(), es6TableInfo.getPassword())); + restClientBuilder.setHttpClientConfigCallback(httpAsyncClientBuilder -> + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + + RestHighLevelClient rhlClient = new RestHighLevelClient(restClientBuilder); + + if (LOG.isInfoEnabled()) { + LOG.info("Pinging Elasticsearch cluster via hosts {} ...", HttpAddresses); + } + + try{ + if (!rhlClient.ping()) { + throw new RuntimeException("There are no reachable Elasticsearch nodes!"); + } + } catch (IOException e){ + LOG.warn("", e); + } + + + if (LOG.isInfoEnabled()) { + LOG.info("Created Elasticsearch RestHighLevelClient connected to {}", HttpAddresses.toString()); + } + + return rhlClient; + } + + @Override + public BulkProcessor.Builder createBulkProcessorBuilder(RestHighLevelClient client, BulkProcessor.Listener listener) { + return BulkProcessor.builder(client::bulkAsync, listener); + } + + @Override + public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) { + if (!bulkItemResponse.isFailed()) { + return null; + } else { + return bulkItemResponse.getFailure().getCause(); + } + } + + @Override + public void configureBulkProcessorBackoff( + BulkProcessor.Builder builder, + @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { + + BackoffPolicy backoffPolicy; + if (flushBackoffPolicy != null) { + switch (flushBackoffPolicy.getBackoffType()) { + case CONSTANT: + backoffPolicy = BackoffPolicy.constantBackoff( + new TimeValue(flushBackoffPolicy.getDelayMillis()), + flushBackoffPolicy.getMaxRetryCount()); + break; + case EXPONENTIAL: + default: + backoffPolicy = BackoffPolicy.exponentialBackoff( + new TimeValue(flushBackoffPolicy.getDelayMillis()), + flushBackoffPolicy.getMaxRetryCount()); + } + } else { + backoffPolicy = BackoffPolicy.noBackoff(); + } + + builder.setBackoffPolicy(backoffPolicy); + } + +} diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java new file mode 100644 index 000000000..ffd14bf43 --- /dev/null +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.elasticsearch6; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.Meter; +import org.apache.flink.metrics.MeterView; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction; +import org.apache.flink.streaming.connectors.elasticsearch.util.NoOpFailureHandler; + +import com.dtstack.flink.sql.metric.MetricConstant; +import com.dtstack.flink.sql.sink.elasticsearch6.table.ElasticsearchTableInfo; +import org.elasticsearch.client.RestHighLevelClient; + +import java.util.List; +import java.util.Map; + +/** + * @author yinxi + * @date 2020/1/9 - 15:05 + */ +public class MetricElasticsearch6Sink extends ElasticsearchSinkBase { + + protected CustomerSinkFunc customerSinkFunc; + + protected transient Meter outRecordsRate; + + protected Map userConfig; + + + public MetricElasticsearch6Sink(Map userConfig, List transportAddresses, + ElasticsearchSinkFunction elasticsearchSinkFunction, + ElasticsearchTableInfo es6TableInfo) { + super(new ExtendES6ApiCallBridge(transportAddresses, es6TableInfo), userConfig, elasticsearchSinkFunction, new NoOpFailureHandler()); + this.customerSinkFunc = (CustomerSinkFunc) elasticsearchSinkFunction; + this.userConfig = userConfig; + } + + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + initMetric(); + } + + + public void initMetric() { + Counter counter = getRuntimeContext().getMetricGroup().counter(MetricConstant.DT_NUM_RECORDS_OUT); + customerSinkFunc.setOutRecords(counter); + outRecordsRate = getRuntimeContext().getMetricGroup().meter(MetricConstant.DT_NUM_RECORDS_OUT_RATE, new MeterView(counter, 20)); + } +} + diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchSinkParser.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchSinkParser.java new file mode 100644 index 000000000..e662a54f3 --- /dev/null +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchSinkParser.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.elasticsearch6.table; + +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.util.MathUtil; + +import java.util.Map; + +/** + * @author yinxi + * @date 2020/1/9 - 15:06 + */ +public class ElasticsearchSinkParser extends AbsTableParser { + + private static final String KEY_ES6_ADDRESS = "address"; + + private static final String KEY_ES6_CLUSTER = "cluster"; + + private static final String KEY_ES6_INDEX = "index"; + + private static final String KEY_ES6_TYPE = "estype"; + + private static final String KEY_ES6_ID_FIELD_INDEX_LIST = "id"; + + private static final String KEY_ES6_AUTHMESH = "authMesh"; + + private static final String KEY_ES6_USERNAME = "userName"; + + private static final String KEY_ES6_PASSWORD = "password"; + + @Override + protected boolean fieldNameNeedsUpperCase() { + return false; + } + + @Override + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + ElasticsearchTableInfo elasticsearchTableInfo = new ElasticsearchTableInfo(); + elasticsearchTableInfo.setName(tableName); + parseFieldsInfo(fieldsInfo, elasticsearchTableInfo); + elasticsearchTableInfo.setAddress((String) props.get(KEY_ES6_ADDRESS.toLowerCase())); + elasticsearchTableInfo.setClusterName((String) props.get(KEY_ES6_CLUSTER.toLowerCase())); + elasticsearchTableInfo.setId((String) props.get(KEY_ES6_ID_FIELD_INDEX_LIST.toLowerCase())); + elasticsearchTableInfo.setIndex((String) props.get(KEY_ES6_INDEX.toLowerCase())); + elasticsearchTableInfo.setEsType((String) props.get(KEY_ES6_TYPE.toLowerCase())); + + String authMeshStr = (String) props.get(KEY_ES6_AUTHMESH.toLowerCase()); + if (authMeshStr != null && "true".equalsIgnoreCase(authMeshStr)) { + elasticsearchTableInfo.setAuthMesh(MathUtil.getBoolean(authMeshStr)); + elasticsearchTableInfo.setUserName(MathUtil.getString(props.get(KEY_ES6_USERNAME.toLowerCase()))); + elasticsearchTableInfo.setPassword(MathUtil.getString(props.get(KEY_ES6_PASSWORD.toLowerCase()))); + } + elasticsearchTableInfo.check(); + return elasticsearchTableInfo; + } +} diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchTableInfo.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchTableInfo.java new file mode 100644 index 000000000..eb7dc1c4f --- /dev/null +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchTableInfo.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.elasticsearch6.table; + +import com.dtstack.flink.sql.table.TargetTableInfo; +import com.google.common.base.Preconditions; + +/** + * @author yinxi + * @date 2020/1/9 - 15:06 + */ +public class ElasticsearchTableInfo extends TargetTableInfo { + + private static final String CURR_TYPE = "elasticsearch6"; + + private String address; + + private String index; + + private String id; + + private String clusterName; + + private String esType; + + private boolean authMesh = false; + + private String userName; + + private String password; + + public String getEsType() { + return esType; + } + + public void setEsType(String esType) { + this.esType = esType; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + + @Override + public String getType() { + //return super.getType().toLowerCase() + TARGET_SUFFIX; + return super.getType().toLowerCase(); + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public boolean isAuthMesh() { + return authMesh; + } + + public void setAuthMesh(boolean authMesh) { + this.authMesh = authMesh; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public ElasticsearchTableInfo() { + setType(CURR_TYPE); + } + + @Override + public boolean check() { + Preconditions.checkNotNull(address, "elasticsearch6 type of address is required"); + Preconditions.checkNotNull(index, "elasticsearch6 type of index is required"); + Preconditions.checkNotNull(esType, "elasticsearch6 type of type is required"); + Preconditions.checkNotNull(id, "elasticsearch6 type of id is required"); + Preconditions.checkNotNull(clusterName, "elasticsearch6 type of clusterName is required"); + + if (isAuthMesh()) { + Preconditions.checkNotNull(userName, "elasticsearch6 type of userName is required"); + Preconditions.checkNotNull(password, "elasticsearch6 type of password is required"); + } + + return true; + } + +} + diff --git a/elasticsearch6/pom.xml b/elasticsearch6/pom.xml new file mode 100644 index 000000000..624bd174a --- /dev/null +++ b/elasticsearch6/pom.xml @@ -0,0 +1,36 @@ + + + + flink.sql + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.elasticsearch6 + pom + + elasticsearch6-sink + + + + + + junit + junit + 3.8.1 + test + + + + com.dtstack.flink + sql.core + 1.0-SNAPSHOT + provided + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 20ad02810..2794f9004 100644 --- a/pom.xml +++ b/pom.xml @@ -33,6 +33,7 @@ impala db2 polardb + elasticsearch6 From 56ef5226e72d9f014dca88ea6e6725ad3628ac43 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 12 Feb 2020 14:15:59 +0800 Subject: [PATCH 015/523] retract stream all mode --- .../side/cassandra/CassandraAllReqRow.java | 15 ++++----- .../com/dtstack/flink/sql/side/AllReqRow.java | 4 +-- .../dtstack/flink/sql/side/SideSqlExec.java | 24 -------------- .../flink/sql/side/hbase/HbaseAllReqRow.java | 31 +++++++++---------- .../flink/sql/side/kudu/KuduAllReqRow.java | 11 ++++--- .../flink/sql/side/mongo/MongoAllReqRow.java | 15 ++++----- .../flink/sql/side/rdb/all/RdbAllReqRow.java | 16 +++++----- .../flink/sql/side/redis/RedisAllReqRow.java | 19 ++++++------ 8 files changed, 57 insertions(+), 78 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index 7acdabd87..e30b6dfc8 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -38,6 +38,7 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; @@ -129,14 +130,14 @@ protected void reloadCache() { @Override - public void flatMap(Row value, Collector out) throws Exception { + public void flatMap(CRow input, Collector out) throws Exception { List inputParams = Lists.newArrayList(); for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = value.getField(conValIndex); + Object equalObj = input.row().getField(conValIndex); if (equalObj == null) { if(sideInfo.getJoinType() == JoinType.LEFT){ - Row data = fillData(value, null); - out.collect(data); + Row data = fillData(input.row(), null); + out.collect(new CRow(data, input.change())); } return; } @@ -148,8 +149,8 @@ public void flatMap(Row value, Collector out) throws Exception { List> cacheList = cacheRef.get().get(key); if (CollectionUtils.isEmpty(cacheList)) { if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value, null); - out.collect(row); + Row row = fillData(input.row(), null); + out.collect(new CRow(row, input.change())); } else { return; } @@ -158,7 +159,7 @@ public void flatMap(Row value, Collector out) throws Exception { } for (Map one : cacheList) { - out.collect(fillData(value, one)); + out.collect(new CRow(fillData(input.row(), one), input.change())); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java index e788cf139..7d16ee726 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.factory.DTThreadFactory; import org.apache.flink.api.common.functions.RichFlatMapFunction; import org.apache.flink.configuration.Configuration; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import java.sql.SQLException; import java.util.concurrent.Executors; @@ -37,7 +37,7 @@ * @author xuchao */ -public abstract class AllReqRow extends RichFlatMapFunction implements ISideReqRow { +public abstract class AllReqRow extends RichFlatMapFunction implements ISideReqRow { protected SideInfo sideInfo; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 390b24f8f..fa32d05bd 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -254,27 +254,6 @@ public RowTypeInfo buildOutRowTypeInfo(List sideJoinFieldInfo, HashBa return new RowTypeInfo(sideOutTypes, sideOutNames); } - /** - * 对时间类型进行类型转换 - * @param leftTypeInfo - * @return - */ - private RowTypeInfo buildLeftTableOutType(RowTypeInfo leftTypeInfo) { - TypeInformation[] sideOutTypes = new TypeInformation[leftTypeInfo.getFieldNames().length]; - TypeInformation[] fieldTypes = leftTypeInfo.getFieldTypes(); - for (int i = 0; i < sideOutTypes.length; i++) { - sideOutTypes[i] = convertTimeAttributeType(fieldTypes[i]); - } - RowTypeInfo rowTypeInfo = new RowTypeInfo(sideOutTypes, leftTypeInfo.getFieldNames()); - return rowTypeInfo; - } - - private TypeInformation convertTimeAttributeType(TypeInformation typeInformation) { - if (typeInformation instanceof TimeIndicatorTypeInfo) { - return TypeInformation.of(Timestamp.class); - } - return typeInformation; - } //需要考虑更多的情况 private void replaceFieldName(SqlNode sqlNode, HashBasedTable mappingTable, String targetTableName, String tableAlias) { @@ -786,9 +765,6 @@ private void joinFun(Object pollObj, Map localTableCache, //join side table before keyby ===> Reducing the size of each dimension table cache of async if (sideTableInfo.isPartitionedJoin()) { - // RowTypeInfo leftTableOutType = buildLeftTableOutType(leftTypeInfo); - // adaptStream.getTransformation().setOutputType(leftTableOutType); - List leftJoinColList = getConditionFields(joinInfo.getCondition(), joinInfo.getLeftTableAlias(), sideTableInfo); List fieldNames = Arrays.asList(targetTable.getSchema().getFieldNames()); int[] keyIndex = leftJoinColList.stream().mapToInt(fieldNames::indexOf).toArray(); diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 813d6af55..11380eea6 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -26,6 +26,7 @@ import org.apache.commons.collections.map.HashedMap; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; @@ -114,41 +115,39 @@ protected void reloadCache() { } @Override - public void flatMap(Row value, Collector out) throws Exception { + public void flatMap(CRow input, Collector out) throws Exception { Map refData = Maps.newHashMap(); for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = value.getField(conValIndex); - if(equalObj == null){ - if(sideInfo.getJoinType() == JoinType.LEFT){ - Row data = fillData(value, null); - out.collect(data); + Object equalObj = input.row().getField(conValIndex); + if (equalObj == null) { + if (sideInfo.getJoinType() == JoinType.LEFT) { + Row data = fillData(input.row(), null); + out.collect(new CRow(data, input.change())); } return; } refData.put(sideInfo.getEqualFieldList().get(i), equalObj); } - String rowKeyStr = ((HbaseAllSideInfo)sideInfo).getRowKeyBuilder().getRowKey(refData); + String rowKeyStr = ((HbaseAllSideInfo) sideInfo).getRowKeyBuilder().getRowKey(refData); Map cacheList = null; SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; - if (hbaseSideTableInfo.isPreRowKey()) - { - for (Map.Entry> entry : cacheRef.get().entrySet()){ - if (entry.getKey().startsWith(rowKeyStr)) - { + if (hbaseSideTableInfo.isPreRowKey()) { + for (Map.Entry> entry : cacheRef.get().entrySet()) { + if (entry.getKey().startsWith(rowKeyStr)) { cacheList = cacheRef.get().get(entry.getKey()); - Row row = fillData(value, cacheList); - out.collect(row); + Row row = fillData(input.row(), cacheList); + out.collect(new CRow(row, input.change())); } } } else { cacheList = cacheRef.get().get(rowKeyStr); - Row row = fillData(value, cacheList); - out.collect(row); + Row row = fillData(input.row(), cacheList); + out.collect(new CRow(row, input.change())); } } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index b688ec225..047665928 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -12,6 +12,7 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; @@ -110,10 +111,10 @@ protected void reloadCache() { @Override - public void flatMap(Row value, Collector out) throws Exception { + public void flatMap(CRow input, Collector out) throws Exception { List inputParams = Lists.newArrayList(); for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = value.getField(conValIndex); + Object equalObj = input.row().getField(conValIndex); if (equalObj == null) { out.collect(null); } @@ -124,14 +125,14 @@ public void flatMap(Row value, Collector out) throws Exception { List> cacheList = cacheRef.get().get(key); if (CollectionUtils.isEmpty(cacheList)) { if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value, null); - out.collect(row); + Row row = fillData(input.row(), null); + out.collect(new CRow(row, input.change())); } return; } for (Map one : cacheList) { - out.collect(fillData(value, one)); + out.collect(new CRow(fillData(input.row(), one), input.change())); } } diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index 9b0f85a24..b416e4820 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -39,6 +39,7 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; @@ -128,14 +129,14 @@ protected void reloadCache() { } @Override - public void flatMap(Row value, Collector out) throws Exception { + public void flatMap(CRow input, Collector out) throws Exception { List inputParams = Lists.newArrayList(); for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = value.getField(conValIndex); + Object equalObj = input.row().getField(conValIndex); if (equalObj == null) { if(sideInfo.getJoinType() == JoinType.LEFT){ - Row data = fillData(value, null); - out.collect(data); + Row data = fillData(input.row(), null); + out.collect(new CRow(data, input.change())); } return; } @@ -147,8 +148,8 @@ public void flatMap(Row value, Collector out) throws Exception { List> cacheList = cacheRef.get().get(key); if (CollectionUtils.isEmpty(cacheList)) { if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value, null); - out.collect(row); + Row row = fillData(input.row(), null); + out.collect(new CRow(row, input.change())); } else { return; } @@ -157,7 +158,7 @@ public void flatMap(Row value, Collector out) throws Exception { } for (Map one : cacheList) { - out.collect(fillData(value, one)); + out.collect(new CRow(fillData(input.row(), one), input.change())); } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java index c07e4ca85..48d2407aa 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java @@ -26,6 +26,7 @@ import org.apache.commons.collections.CollectionUtils; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; @@ -114,14 +115,14 @@ protected void reloadCache() { @Override - public void flatMap(Row value, Collector out) throws Exception { + public void flatMap(CRow value, Collector out) throws Exception { List inputParams = Lists.newArrayList(); for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = value.getField(conValIndex); + Object equalObj = value.row().getField(conValIndex); if (equalObj == null) { if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value, null); - out.collect(row); + Row row = fillData(value.row(), null); + out.collect(new CRow(row, value.change())); } return; } @@ -132,8 +133,8 @@ public void flatMap(Row value, Collector out) throws Exception { List> cacheList = cacheRef.get().get(key); if (CollectionUtils.isEmpty(cacheList)) { if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value, null); - out.collect(row); + Row row = fillData(value.row(), null); + out.collect(new CRow(row, value.change())); } else { return; } @@ -142,9 +143,8 @@ public void flatMap(Row value, Collector out) throws Exception { } for (Map one : cacheList) { - out.collect(fillData(value, one)); + out.collect(new CRow(fillData(value.row(), one), value.change())); } - } private String buildKey(List equalValList) { diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 7437d74e5..59997f5cb 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -26,6 +26,7 @@ import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; import org.slf4j.Logger; @@ -89,14 +90,14 @@ protected void reloadCache() { } @Override - public void flatMap(Row row, Collector out) throws Exception { + public void flatMap(CRow input, Collector out) throws Exception { Map inputParams = Maps.newHashMap(); for(Integer conValIndex : sideInfo.getEqualValIndex()){ - Object equalObj = row.getField(conValIndex); + Object equalObj = input.row().getField(conValIndex); if(equalObj == null){ - if(sideInfo.getJoinType() == JoinType.LEFT){ - Row data = fillData(row, null); - out.collect(data); + if (sideInfo.getJoinType() == JoinType.LEFT) { + Row data = fillData(input.row(), null); + out.collect(new CRow(data, input.change())); } return; } @@ -109,8 +110,8 @@ public void flatMap(Row row, Collector out) throws Exception { if (cacheMap == null){ if(sideInfo.getJoinType() == JoinType.LEFT){ - Row data = fillData(row, null); - out.collect(data); + Row data = fillData(input.row(), null); + out.collect(new CRow(data, input.change())); }else{ return; } @@ -118,8 +119,8 @@ public void flatMap(Row row, Collector out) throws Exception { return; } - Row newRow = fillData(row, cacheMap); - out.collect(newRow); + Row newRow = fillData(input.row(), cacheMap); + out.collect(new CRow(newRow, input.change())); } private String buildKey(Map inputParams) { From 01ff006b3ffc3c7bc08050e54994483e24cd4dc5 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Wed, 12 Feb 2020 21:14:06 +0800 Subject: [PATCH 016/523] =?UTF-8?q?=E5=A2=9E=E5=8A=A0elasticsearch6-side?= =?UTF-8?q?=E5=8A=9F=E8=83=BD(=E6=9C=AA=E5=AE=8C=E6=88=90)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../elasticsearch6-all-side/pom.xml | 100 +++++++ .../Elasticsearch6AllSideInfo.java | 135 +++++++++ .../Elasticsearh6AllReqRow.java | 263 ++++++++++++++++++ .../elasticsearch6-side-core/pom.xml | 20 ++ .../table/Elasticsearch6SideParser.java | 54 ++++ .../table/Elasticsearch6SideTableInfo.java | 108 +++++++ elasticsearch6/elasticsearch6-side/pom.xml | 21 ++ 7 files changed, 701 insertions(+) create mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/pom.xml create mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java create mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearh6AllReqRow.java create mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/pom.xml create mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java create mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java create mode 100644 elasticsearch6/elasticsearch6-side/pom.xml diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/pom.xml b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/pom.xml new file mode 100644 index 000000000..cdf45134a --- /dev/null +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/pom.xml @@ -0,0 +1,100 @@ + + + + sql.side.elasticsearch6 + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.side.all.elasticsearch6 + elasticsearch6-all-side + jar + + + 1.0-SNAPSHOT + 6.8.6 + + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} + + + + com.dtstack.flink + sql.side.elasticsearch6.core + ${sql.side.elasticsearch6.core.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java new file mode 100644 index 000000000..c7c5d96ea --- /dev/null +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java @@ -0,0 +1,135 @@ +package com.dtstack.flink.sql.side.elasticsearch6; + +import org.apache.flink.api.java.typeutils.RowTypeInfo; + +import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; +import com.dtstack.flink.sql.util.ParseUtils; +import com.google.common.collect.Lists; +import org.apache.calcite.sql.SqlNode; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author yinxi + * @date 2020/1/13 - 1:01 + */ +public class Elasticsearch6AllSideInfo extends SideInfo { + + public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + } + + @Override + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + Elasticsearch6SideTableInfo es6SideTableInfo = (Elasticsearch6SideTableInfo) sideTableInfo; + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + + + sqlCondition = getSelectFromStatement(getEstype(es6SideTableInfo), Arrays.asList(sideSelectFields.split(",")), sideTableInfo.getPredicateInfoes()); + System.out.println("-------- all side sql query-------\n" + sqlCondition); + } + + //基于rdb开发side,但是那些between,in,not in之类的不知道怎么处理 + + public String getAdditionalWhereClause() { + return ""; + } + + private String getSelectFromStatement(String tableName, List selectFields, List predicateInfoes) { + String fromClause = selectFields.stream().map(this::quoteIdentifier).collect(Collectors.joining(", ")); + String predicateClause = predicateInfoes.stream().map(this::buildFilterCondition).collect(Collectors.joining(" AND ")); + String whereClause = buildWhereClause(predicateClause); + String sql = "SELECT " + fromClause + " FROM " + tableName + whereClause; + return sql; + } + + private String buildWhereClause(String predicateClause) { + String additionalWhereClause = getAdditionalWhereClause(); + String whereClause = (!StringUtils.isEmpty(predicateClause) || !StringUtils.isEmpty(additionalWhereClause) ? " WHERE " + predicateClause : ""); + whereClause += (StringUtils.isEmpty(predicateClause)) ? additionalWhereClause.replaceFirst("AND", "") : additionalWhereClause; + return whereClause; + } + + @Override + public void parseSelectFields(JoinInfo joinInfo) { + String sideTableName = joinInfo.getSideTableName(); + String nonSideTableName = joinInfo.getNonSideTable(); + List fields = Lists.newArrayList(); + + int sideIndex = 0; + for (int i = 0; i < outFieldInfoList.size(); i++) { + FieldInfo fieldInfo = outFieldInfoList.get(i); + if (fieldInfo.getTable().equalsIgnoreCase(sideTableName)) { + fields.add(fieldInfo.getFieldName()); + sideFieldIndex.put(i, sideIndex); + sideFieldNameIndex.put(i, fieldInfo.getFieldName()); + sideIndex++; + } else if (fieldInfo.getTable().equalsIgnoreCase(nonSideTableName)) { + int nonSideIndex = rowTypeInfo.getFieldIndex(fieldInfo.getFieldName()); + inFieldIndex.put(i, nonSideIndex); + } else { + throw new RuntimeException("unknown table " + fieldInfo.getTable()); + } + } + + if (fields.size() == 0) { + throw new RuntimeException("select non field from table " + sideTableName); + } + + //add join on condition field to select fields + SqlNode conditionNode = joinInfo.getCondition(); + + List sqlNodeList = Lists.newArrayList(); + + ParseUtils.parseAnd(conditionNode, sqlNodeList); + + for (SqlNode sqlNode : sqlNodeList) { + dealOneEqualCon(sqlNode, sideTableName); + } + + if (CollectionUtils.isEmpty(equalFieldList)) { + throw new RuntimeException("no join condition found after table " + joinInfo.getLeftTableName()); + } + + for (String equalField : equalFieldList) { + if (fields.contains(equalField)) { + continue; + } + + fields.add(equalField); + } + + sideSelectFields = String.join(",", fields); + } + + public String buildFilterCondition(PredicateInfo info) { + switch (info.getOperatorKind()) { + case "IN": + case "NOT_IN": + return quoteIdentifier(info.getFieldName()) + " " + info.getOperatorName() + " ( " + info.getCondition() + " )"; + case "NOT_EQUALS": + return quoteIdentifier(info.getFieldName()) + " != " + info.getCondition(); + case "BETWEEN": + return quoteIdentifier(info.getFieldName()) + " BETWEEN " + info.getCondition(); + case "IS_NOT_NULL": + case "IS_NULL": + return quoteIdentifier(info.getFieldName()) + " " + info.getOperatorName(); + default: + return quoteIdentifier(info.getFieldName()) + " " + info.getOperatorName() + " " + info.getCondition(); + } + } + + public String getEstype(Elasticsearch6SideTableInfo es6SdideTableInfo) { + return es6SdideTableInfo.getEsType(); + } + + public String quoteIdentifier(String identifier) { + return " " + identifier + " "; + } +} diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearh6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearh6AllReqRow.java new file mode 100644 index 000000000..bfbe72bb7 --- /dev/null +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearh6AllReqRow.java @@ -0,0 +1,263 @@ +package com.dtstack.flink.sql.side.elasticsearch6; + +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; + +import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.calcite.sql.JoinType; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.QueryStringQueryBuilder; +import org.elasticsearch.search.Scroll; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +/** + * @author yinxi + * @date 2020/1/13 - 1:00 + */ +public class Elasticsearh6AllReqRow extends AllReqRow { + + private static final Logger LOG = LoggerFactory.getLogger(Elasticsearh6AllReqRow.class); + private static final int CONN_RETRY_NUM = 3; + private AtomicReference>>> cacheRef = new AtomicReference<>(); + + public Elasticsearh6AllReqRow(SideInfo sideInfo) { + super(sideInfo); + } + + @Override + public void flatMap(Row value, Collector out) throws Exception { + List inputParams = Lists.newArrayList(); + for (Integer conValIndex : sideInfo.getEqualValIndex()) { + Object equalObj = value.getField(conValIndex); + if (equalObj == null) { + if (sideInfo.getJoinType() == JoinType.LEFT) { + Row row = fillData(value, null); + out.collect(row); + } + + return; + } + + inputParams.add(equalObj); + } + + String key = buildKey(inputParams); + List> cacheList = cacheRef.get().get(key); + if(CollectionUtils.isEmpty(cacheList)){ + if(sideInfo.getJoinType() == JoinType.LEFT){ + Row row = fillData(value, null); + out.collect(row); + }else{ + return; + } + + return; + } + } + + @Override + public Row fillData(Row input, Object sideInput) { + Map cacheInfo = (Map) sideInput; + Row row = new Row(sideInfo.getOutFieldInfoList().size()); + for (Map.Entry entry : sideInfo.getInFieldIndex().entrySet()) { + Object obj = input.getField(entry.getValue()); + boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); + + //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. + if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { + obj = ((Timestamp) obj).getTime(); + } + + row.setField(entry.getKey(), obj); + } + + for (Map.Entry entry : sideInfo.getSideFieldNameIndex().entrySet()) { + if (cacheInfo == null) { + row.setField(entry.getKey(), null); + } else { + row.setField(entry.getKey(), cacheInfo.get(entry.getValue())); + } + } + + return row; + } + + private String buildKey(List equalValList) { + StringBuilder sb = new StringBuilder(""); + for (Object equalVal : equalValList) { + sb.append(equalVal).append("_"); + } + + return sb.toString(); + } + + private String buildKey(Map val, List equalFieldList) { + StringBuilder sb = new StringBuilder(""); + for (String equalField : equalFieldList) { + sb.append(val.get(equalField)).append("_"); + } + + return sb.toString(); + } + + @Override + protected void initCache() throws SQLException { + Map>> newCache = Maps.newConcurrentMap(); + cacheRef.set(newCache); + try { + loadData(newCache); + } catch (Exception e) { + LOG.error("", e); + } + } + + @Override + protected void reloadCache() { + //reload cacheRef and replace to old cacheRef + Map>> newCache = Maps.newConcurrentMap(); + try { + loadData(newCache); + } catch (Exception e) { + LOG.error("", e); + } + + cacheRef.set(newCache); + LOG.info("----- elasticsearch6 all cacheRef reload end:{}", Calendar.getInstance()); + } + + private void loadData(Map>> tmpCache) throws IOException { + Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); + RestHighLevelClient rhlClient = null; + + try { + for (int i = 0; i < CONN_RETRY_NUM; i++) { + try { + rhlClient = getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword()); + break; + } catch (Exception e) { + if (i == CONN_RETRY_NUM - 1) { + throw new RuntimeException("", e); + } + + try { + String connInfo = "address: " + tableInfo.getAddress() + ";userName:" + tableInfo.getUserName() + ",pwd:" + tableInfo.getPassword(); + LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); + Thread.sleep(5 * 1000); + } catch (InterruptedException e1) { + LOG.error("", e1); + } + } + } + + // load data from table + String sql = sideInfo.getSqlCondition(); + + Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L)); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(getBathSize()); + if(StringUtils.isNotEmpty(sql)){ + searchSourceBuilder.query(QueryBuilders.wrapperQuery(sql)); + } + +// if(genericInputSplit.getTotalNumberOfSplits() > 1){ +// searchSourceBuilder.slice(new SliceBuilder(genericInputSplit.getSplitNumber(), genericInputSplit.getTotalNumberOfSplits())); +// } + + SearchRequest searchRequest = new SearchRequest(tableInfo.getIndex()); + searchRequest.types(tableInfo.getEsType()); + searchRequest.scroll(scroll); + searchRequest.source(searchSourceBuilder); + + SearchResponse searchResponse = rhlClient.search(searchRequest); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); + String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); + + Map oneRow = Maps.newHashMap(); + for (SearchHit searchHit : searchHits) { + for(String fieldName : sideFieldNames){ + Object object = searchHit.getSourceAsMap().get(fieldName.trim()); + int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); + object = SwitchUtil.getTarget(object, fields[fieldIndex]); + oneRow.put(fieldName.trim(), object); + } + + } + + String cacheKey = buildKey(oneRow, sideInfo.getEqualFieldList()); + List> list = tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()); + list.add(oneRow); + + } catch (Exception e) { + LOG.error("", e); + } finally { + if (rhlClient != null) { + rhlClient.close(); + } + } + + } + + public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, String userName, String password) { + List httpHostList = new ArrayList<>(); + String[] address = esAddress.split(","); + for (String addr : address) { + String[] infoArray = addr.split(":"); + int port = 9200; + String host = infoArray[0].trim(); + if (infoArray.length > 1) { + port = Integer.valueOf(infoArray[1].trim()); + } + httpHostList.add(new HttpHost(host, port, "http")); + } + + RestClientBuilder restClientBuilder = RestClient.builder(httpHostList.toArray(new HttpHost[httpHostList.size()])); + if (isAuthMesh) { + // 进行用户和密码认证 + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password)); + restClientBuilder.setHttpClientConfigCallback(httpAsyncClientBuilder -> + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + + RestHighLevelClient rhlClient = new RestHighLevelClient(restClientBuilder); + + return rhlClient; + + } + + public int getBathSize() { + return 1000; + } +} diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/pom.xml b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/pom.xml new file mode 100644 index 000000000..679881182 --- /dev/null +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/pom.xml @@ -0,0 +1,20 @@ + + + + sql.side.elasticsearch6 + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.side.elasticsearch6.core + elasticsearch6-side-core + 1.0-SNAPSHOT + jar + + + + \ No newline at end of file diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java new file mode 100644 index 000000000..a40f33575 --- /dev/null +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java @@ -0,0 +1,54 @@ +package com.dtstack.flink.sql.side.elasticsearch6.table; + +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.util.MathUtil; +import org.apache.commons.lang3.StringUtils; + +import java.util.Map; + +/** + * @author yinxi + * @date 2020/1/13 - 1:07 + */ +public class Elasticsearch6SideParser extends AbsSideTableParser { + + private static final String KEY_ES6_ADDRESS = "address"; + + private static final String KEY_ES6_CLUSTER = "cluster"; + + private static final String KEY_ES6_INDEX = "index"; + + private static final String KEY_ES6_TYPE = "estype"; + + private static final String KEY_ES6_AUTHMESH = "authMesh"; + + private static final String KEY_ES6_USERNAME = "userName"; + + private static final String KEY_ES6_PASSWORD = "password"; + + @Override + protected boolean fieldNameNeedsUpperCase() { + return false; + } + + @Override + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + Elasticsearch6SideTableInfo elasticsearch6SideTableInfo = new Elasticsearch6SideTableInfo(); + elasticsearch6SideTableInfo.setName(tableName); + parseFieldsInfo(fieldsInfo, elasticsearch6SideTableInfo); + elasticsearch6SideTableInfo.setAddress((String) props.get(KEY_ES6_ADDRESS.toLowerCase())); + elasticsearch6SideTableInfo.setClusterName((String) props.get(KEY_ES6_CLUSTER.toLowerCase())); + elasticsearch6SideTableInfo.setIndex((String) props.get(KEY_ES6_INDEX.toLowerCase())); + elasticsearch6SideTableInfo.setEsType((String) props.get(KEY_ES6_TYPE.toLowerCase())); + + String authMeshStr = (String) props.get(KEY_ES6_AUTHMESH.toLowerCase()); + if (authMeshStr != null && StringUtils.equalsIgnoreCase("true", authMeshStr)) { + elasticsearch6SideTableInfo.setAuthMesh(MathUtil.getBoolean(authMeshStr)); + elasticsearch6SideTableInfo.setUserName(MathUtil.getString(props.get(KEY_ES6_USERNAME.toLowerCase()))); + elasticsearch6SideTableInfo.setPassword(MathUtil.getString(props.get(KEY_ES6_PASSWORD.toLowerCase()))); + } + elasticsearch6SideTableInfo.check(); + return elasticsearch6SideTableInfo; + } +} diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java new file mode 100644 index 000000000..5964e8631 --- /dev/null +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java @@ -0,0 +1,108 @@ +package com.dtstack.flink.sql.side.elasticsearch6.table; + +import com.dtstack.flink.sql.side.SideTableInfo; +import com.google.common.base.Preconditions; + +/** + * @author yinxi + * @date 2020/1/13 - 15:00 + */ +public class Elasticsearch6SideTableInfo extends SideTableInfo { + + private static final String CURR_TYPE = "elasticsearch6"; + + private String address; + + private String index; + + private String clusterName; + + private String esType; + + private boolean authMesh = false; + + private String userName; + + private String password; + + public String getEsType() { + return esType; + } + + public void setEsType(String esType) { + this.esType = esType; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + + @Override + public String getType() { + //return super.getType().toLowerCase() + TARGET_SUFFIX; + return super.getType().toLowerCase(); + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public boolean isAuthMesh() { + return authMesh; + } + + public void setAuthMesh(boolean authMesh) { + this.authMesh = authMesh; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public Elasticsearch6SideTableInfo() { + setType(CURR_TYPE); + } + + @Override + public boolean check() { + Preconditions.checkNotNull(address, "elasticsearch6 type of address is required"); + Preconditions.checkNotNull(index, "elasticsearch6 type of index is required"); + Preconditions.checkNotNull(esType, "elasticsearch6 type of type is required"); + Preconditions.checkNotNull(clusterName, "elasticsearch6 type of clusterName is required"); + + if (isAuthMesh()) { + Preconditions.checkNotNull(userName, "elasticsearch6 type of userName is required"); + Preconditions.checkNotNull(password, "elasticsearch6 type of password is required"); + } + + return true; + } +} diff --git a/elasticsearch6/elasticsearch6-side/pom.xml b/elasticsearch6/elasticsearch6-side/pom.xml new file mode 100644 index 000000000..e4f8f6876 --- /dev/null +++ b/elasticsearch6/elasticsearch6-side/pom.xml @@ -0,0 +1,21 @@ + + + + sql.elasticsearch6 + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.side.elasticsearch6 + pom + elasticsearch6-side + + + + elasticsearch6-side-core + + + \ No newline at end of file From dc3a29eecccf9adb55c474e5f2710f1da2cb2c11 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Thu, 13 Feb 2020 06:25:04 +0800 Subject: [PATCH 017/523] =?UTF-8?q?=E5=A2=9E=E5=8A=A0elasticsearch6-side?= =?UTF-8?q?=E5=8A=9F=E8=83=BD(ALL=E8=83=BD=E8=AF=BB=E5=8F=96=E6=95=B0?= =?UTF-8?q?=E6=8D=AE)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Elasticsearch6AllSideInfo.java | 100 +++---- .../Elasticsearh6AllReqRow.java | 27 +- .../side/elasticsearch6/util/MathUtil.java | 255 ++++++++++++++++++ .../side/elasticsearch6/util/SwitchUtil.java | 82 ++++++ 4 files changed, 395 insertions(+), 69 deletions(-) create mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java create mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java index c7c5d96ea..853f07dd7 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java @@ -1,19 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.dtstack.flink.sql.side.elasticsearch6; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.dtstack.flink.sql.side.*; -import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; -import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; /** * @author yinxi @@ -21,39 +37,53 @@ */ public class Elasticsearch6AllSideInfo extends SideInfo { + public static SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { - Elasticsearch6SideTableInfo es6SideTableInfo = (Elasticsearch6SideTableInfo) sideTableInfo; - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + getSelectFromStatement(sideTableInfo.getPredicateInfoes()); - sqlCondition = getSelectFromStatement(getEstype(es6SideTableInfo), Arrays.asList(sideSelectFields.split(",")), sideTableInfo.getPredicateInfoes()); - System.out.println("-------- all side sql query-------\n" + sqlCondition); } - //基于rdb开发side,但是那些between,in,not in之类的不知道怎么处理 + private void getSelectFromStatement(List predicateInfoes) { - public String getAdditionalWhereClause() { - return ""; - } + if (predicateInfoes.size() != 0) { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + for (PredicateInfo info : sideTableInfo.getPredicateInfoes()) { + boolQueryBuilder = buildFilterCondition(boolQueryBuilder, info); + } + + searchSourceBuilder.query(boolQueryBuilder); + } - private String getSelectFromStatement(String tableName, List selectFields, List predicateInfoes) { - String fromClause = selectFields.stream().map(this::quoteIdentifier).collect(Collectors.joining(", ")); - String predicateClause = predicateInfoes.stream().map(this::buildFilterCondition).collect(Collectors.joining(" AND ")); - String whereClause = buildWhereClause(predicateClause); - String sql = "SELECT " + fromClause + " FROM " + tableName + whereClause; - return sql; } - private String buildWhereClause(String predicateClause) { - String additionalWhereClause = getAdditionalWhereClause(); - String whereClause = (!StringUtils.isEmpty(predicateClause) || !StringUtils.isEmpty(additionalWhereClause) ? " WHERE " + predicateClause : ""); - whereClause += (StringUtils.isEmpty(predicateClause)) ? additionalWhereClause.replaceFirst("AND", "") : additionalWhereClause; - return whereClause; + public BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info){ + switch (info.getOperatorKind()) { + case "GREATER_THAN_OR_EQUAL": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gte(info.getCondition())); + case "GREATER_THAN": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gt(info.getCondition())); + case "LESS_THAN_OR_EQUAL": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lte(info.getCondition())); + case "LESS_THAN": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lt(info.getCondition())); + case "EQUALS": + return boolQueryBuilder.must(QueryBuilders.termQuery(info.getFieldName(), info.getCondition())); + default: + try { + throw new Exception("Predicate does not match!"); + } catch (Exception e) { + e.printStackTrace(); + } + } + + return boolQueryBuilder; } @Override @@ -108,28 +138,4 @@ public void parseSelectFields(JoinInfo joinInfo) { sideSelectFields = String.join(",", fields); } - public String buildFilterCondition(PredicateInfo info) { - switch (info.getOperatorKind()) { - case "IN": - case "NOT_IN": - return quoteIdentifier(info.getFieldName()) + " " + info.getOperatorName() + " ( " + info.getCondition() + " )"; - case "NOT_EQUALS": - return quoteIdentifier(info.getFieldName()) + " != " + info.getCondition(); - case "BETWEEN": - return quoteIdentifier(info.getFieldName()) + " BETWEEN " + info.getCondition(); - case "IS_NOT_NULL": - case "IS_NULL": - return quoteIdentifier(info.getFieldName()) + " " + info.getOperatorName(); - default: - return quoteIdentifier(info.getFieldName()) + " " + info.getOperatorName() + " " + info.getCondition(); - } - } - - public String getEstype(Elasticsearch6SideTableInfo es6SdideTableInfo) { - return es6SdideTableInfo.getEsType(); - } - - public String quoteIdentifier(String identifier) { - return " " + identifier + " "; - } } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearh6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearh6AllReqRow.java index bfbe72bb7..ce3f00a34 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearh6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearh6AllReqRow.java @@ -7,11 +7,11 @@ import com.dtstack.flink.sql.side.AllReqRow; import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; +import com.dtstack.flink.sql.side.elasticsearch6.util.SwitchUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; @@ -22,10 +22,6 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.QueryStringQueryBuilder; -import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; @@ -179,24 +175,11 @@ private void loadData(Map>> tmpCache) throws IO } } - // load data from table - String sql = sideInfo.getSqlCondition(); - - Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L)); - - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.size(getBathSize()); - if(StringUtils.isNotEmpty(sql)){ - searchSourceBuilder.query(QueryBuilders.wrapperQuery(sql)); - } - -// if(genericInputSplit.getTotalNumberOfSplits() > 1){ -// searchSourceBuilder.slice(new SliceBuilder(genericInputSplit.getSplitNumber(), genericInputSplit.getTotalNumberOfSplits())); -// } - + // load data from tableA + SearchSourceBuilder searchSourceBuilder = Elasticsearch6AllSideInfo.searchSourceBuilder; + searchSourceBuilder.size(getFetchSize()); SearchRequest searchRequest = new SearchRequest(tableInfo.getIndex()); searchRequest.types(tableInfo.getEsType()); - searchRequest.scroll(scroll); searchRequest.source(searchSourceBuilder); SearchResponse searchResponse = rhlClient.search(searchRequest); @@ -257,7 +240,7 @@ public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, Strin } - public int getBathSize() { + public int getFetchSize() { return 1000; } } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java new file mode 100644 index 000000000..7765ddf17 --- /dev/null +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.dtstack.flink.sql.side.elasticsearch6.util; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Date; +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; + +/** + * Date: 2017/4/21 + * Company: www.dtstack.com + * + * @author xuchao + */ + +public class MathUtil { + + public static Long getLongVal(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Long.valueOf((String) obj); + } else if (obj instanceof Long) { + return (Long) obj; + } else if (obj instanceof Integer) { + return Long.valueOf(obj.toString()); + } else if (obj instanceof BigDecimal) { + return ((BigDecimal) obj).longValue(); + } else if (obj instanceof BigInteger) { + return ((BigInteger) obj).longValue(); + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Long."); + } + + public static Long getLongVal(Object obj, long defaultVal) { + if (obj == null) { + return defaultVal; + } + + return getLongVal(obj); + } + + public static Integer getIntegerVal(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Integer.valueOf((String) obj); + } else if (obj instanceof Integer) { + return (Integer) obj; + } else if (obj instanceof Long) { + return ((Long) obj).intValue(); + } else if (obj instanceof Double) { + return ((Double) obj).intValue(); + } else if (obj instanceof BigDecimal) { + return ((BigDecimal) obj).intValue(); + } else if (obj instanceof BigInteger) { + return ((BigInteger) obj).intValue(); + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Integer."); + } + + public static Integer getIntegerVal(Object obj, int defaultVal) { + if (obj == null) { + return defaultVal; + } + + return getIntegerVal(obj); + } + + public static Float getFloatVal(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Float.valueOf((String) obj); + } else if (obj instanceof Float) { + return (Float) obj; + } else if (obj instanceof BigDecimal) { + return ((BigDecimal) obj).floatValue(); + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Float."); + } + + public static Float getFloatVal(Object obj, float defaultVal) { + if (obj == null) { + return defaultVal; + } + + return getFloatVal(obj); + } + + public static Double getDoubleVal(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Double.valueOf((String) obj); + } else if (obj instanceof Float) { + return (Double) obj; + } else if (obj instanceof BigDecimal) { + return ((BigDecimal) obj).doubleValue(); + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Double."); + } + + public static Double getDoubleVal(Object obj, double defaultVal) { + if (obj == null) { + return defaultVal; + } + + return getDoubleVal(obj); + } + + + public static Boolean getBoolean(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Boolean.valueOf((String) obj); + } else if (obj instanceof Boolean) { + return (Boolean) obj; + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Boolean."); + } + + public static Boolean getBoolean(Object obj, boolean defaultVal) { + if (obj == null) { + return defaultVal; + } + + return getBoolean(obj); + } + + public static String getString(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return (String) obj; + } + + return obj.toString(); + } + + public static Byte getByte(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Byte.valueOf((String) obj); + } else if (obj instanceof Byte) { + return (Byte) obj; + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Byte."); + } + + public static Short getShort(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Short.valueOf((String) obj); + } else if (obj instanceof Short) { + return (Short) obj; + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Short."); + } + + public static BigDecimal getBigDecimal(Object obj) { + if (obj == null) { + return null; + } + if (obj instanceof String) { + return new BigDecimal((String) obj); + } else if (obj instanceof BigDecimal) { + return (BigDecimal) obj; + } else if (obj instanceof BigInteger) { + return new BigDecimal((BigInteger) obj); + } else if (obj instanceof Number) { + return new BigDecimal(((Number) obj).doubleValue()); + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to BigDecimal."); + } + + public static Date getDate(Object obj) { + if (obj == null) { + return null; + } + if (obj instanceof String) { + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + try { + return new Date(format.parse((String) obj).getTime()); + } catch (ParseException e) { + throw new RuntimeException("String convert to Date fail."); + } + } else if (obj instanceof Timestamp) { + return new Date(((Timestamp) obj).getTime()); + } else if (obj instanceof Date) { + return (Date) obj; + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); + } + + public static Timestamp getTimestamp(Object obj) { + if (obj == null) { + return null; + } + if (obj instanceof Timestamp) { + return (Timestamp) obj; + } else if (obj instanceof Date) { + return new Timestamp(((Date) obj).getTime()); + } else if (obj instanceof String) { + return new Timestamp(getDate(obj).getTime()); + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); + } + +} \ No newline at end of file diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java new file mode 100644 index 000000000..5f2e08651 --- /dev/null +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.flink.sql.side.elasticsearch6.util; + +/** + * Reason: + * Date: 2018/12/3 + * Company: www.dtstack.com + * + * @author maqi + */ +public class SwitchUtil { + public static Object getTarget(Object obj, String targetType) { + targetType = targetType.toLowerCase(); + switch (targetType) { + + case "smallint": + case "smallintunsigned": + case "tinyint": + case "tinyintunsigned": + case "mediumint": + case "mediumintunsigned": + case "integer": + case "int": + return MathUtil.getIntegerVal(obj); + + case "bigint": + case "bigintunsigned": + case "intunsigned": + case "integerunsigned": + return MathUtil.getLongVal(obj); + + case "boolean": + return MathUtil.getBoolean(obj); + + case "blob": + return MathUtil.getByte(obj); + + case "varchar": + case "char": + case "text": + return MathUtil.getString(obj); + + case "real": + case "float": + case "realunsigned": + case "floatunsigned": + return MathUtil.getFloatVal(obj); + + case "double": + case "doubleunsigned": + return MathUtil.getDoubleVal(obj); + + case "decimal": + case "decimalunsigned": + return MathUtil.getBigDecimal(obj); + + case "date": + return MathUtil.getDate(obj); + + case "timestamp": + case "datetime": + return MathUtil.getTimestamp(obj); + } + return obj; + } +} From afb4bc6db2e954facbd66c0c9c37a3b9159e9204 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 17 Feb 2020 10:18:17 +0800 Subject: [PATCH 018/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=BA=A7=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/pom.xml | 6 +++++- core/src/main/java/com/dtstack/flink/sql/Main.java | 10 +++++++++- .../java/com/dtstack/flink/sql/option/Options.java | 10 ++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 305873632..cfb290635 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -115,7 +115,11 @@ flink-statebackend-rocksdb_2.11 ${flink.version} - + + ch.qos.logback + logback-classic + 1.1.7 + diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index 435117a46..d8cb2e1cd 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -77,6 +77,8 @@ import java.util.Set; import com.dtstack.flink.sql.option.Options; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.LoggerContext; /** * Date: 2018/6/26 @@ -105,9 +107,9 @@ public static void main(String[] args) throws Exception { String pluginLoadMode = options.getPluginLoadMode(); String deployMode = options.getMode(); String confProp = options.getConfProp(); - sql = URLDecoder.decode(sql, Charsets.UTF_8.name()); SqlParser.setLocalSqlPluginRoot(localSqlPluginPath); + setLogLevel(options.getLogLevel()); List addJarFileList = Lists.newArrayList(); if (!Strings.isNullOrEmpty(addJarListStr)) { @@ -304,4 +306,10 @@ private static StreamExecutionEnvironment getStreamExeEnv(Properties confPropert StreamEnvConfigManager.streamExecutionEnvironmentConfig(env, confProperties); return env; } + private static void setLogLevel(String level){ + LoggerContext loggerContext= (LoggerContext) LoggerFactory.getILoggerFactory(); + //设置全局日志级别 + ch.qos.logback.classic.Logger logger = loggerContext.getLogger("root"); + logger.setLevel(Level.toLevel(level, Level.INFO)); + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/option/Options.java b/core/src/main/java/com/dtstack/flink/sql/option/Options.java index 6adee2a01..d66b0af0a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/Options.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/Options.java @@ -69,6 +69,8 @@ public class Options { @OptionRequired(description = "plugin load mode, by classpath or shipfile") private String pluginLoadMode = EPluginLoadMode.CLASSPATH.name(); + private String logLevel = "logLevel"; + public String getMode() { return mode; } @@ -172,4 +174,12 @@ public String getPluginLoadMode() { public void setPluginLoadMode(String pluginLoadMode) { this.pluginLoadMode = pluginLoadMode; } + + public String getLogLevel() { + return logLevel; + } + + public void setLogLevel(String logLevel) { + this.logLevel = logLevel; + } } From 74b2a9b66b8bb91dbe03204053b0e44298fb9806 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 17 Feb 2020 19:07:50 +0800 Subject: [PATCH 019/523] support get execution plan --- .../com/dtstack/flink/sql/CommonProcess.java | 234 ++++++++++++++++++ .../java/com/dtstack/flink/sql/GetPlan.java | 129 ++++++++++ .../main/java/com/dtstack/flink/sql/Main.java | 207 +--------------- .../com/dtstack/flink/sql/TestGetPlan.java | 87 +++++++ 4 files changed, 457 insertions(+), 200 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/CommonProcess.java create mode 100644 core/src/main/java/com/dtstack/flink/sql/GetPlan.java create mode 100644 core/src/test/java/com/dtstack/flink/sql/TestGetPlan.java diff --git a/core/src/main/java/com/dtstack/flink/sql/CommonProcess.java b/core/src/main/java/com/dtstack/flink/sql/CommonProcess.java new file mode 100644 index 000000000..61bc983d1 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/CommonProcess.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql; + +import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.dtstack.flink.sql.config.CalciteConfig; +import com.dtstack.flink.sql.enums.ClusterMode; +import com.dtstack.flink.sql.enums.ECacheType; +import com.dtstack.flink.sql.environment.MyLocalStreamEnvironment; +import com.dtstack.flink.sql.environment.StreamEnvConfigManager; +import com.dtstack.flink.sql.exec.FlinkSQLExec; +import com.dtstack.flink.sql.function.FunctionManager; +import com.dtstack.flink.sql.parser.CreateFuncParser; +import com.dtstack.flink.sql.parser.CreateTmpTableParser; +import com.dtstack.flink.sql.parser.InsertSqlParser; +import com.dtstack.flink.sql.parser.SqlTree; +import com.dtstack.flink.sql.side.SideSqlExec; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.sink.StreamSinkFactory; +import com.dtstack.flink.sql.source.StreamSourceFactory; +import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.util.PluginUtil; +import com.dtstack.flink.sql.watermarker.WaterMarkerAssigner; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; +import org.apache.calcite.sql.SqlInsert; +import org.apache.calcite.sql.SqlNode; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.StreamQueryConfig; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.types.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +/** + * 提取任务执行时共同的流程方法 + * Date: 2020/2/17 + * Company: www.dtstack.com + * @author maqi + */ +public class CommonProcess { + + private static final String CLASS_FILE_NAME_FMT = "class_path_%d"; + private static final Logger LOG = LoggerFactory.getLogger(Main.class); + + public static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironment tableEnv, SqlTree sqlTree, Map sideTableMap, Map registerTableCache, StreamQueryConfig queryConfig) throws Exception { + SideSqlExec sideSqlExec = new SideSqlExec(); + sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); + for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { + sideSqlExec.registerTmpTable(result, sideTableMap, tableEnv, registerTableCache); + } + + for (InsertSqlParser.SqlParseResult result : sqlTree.getExecSqlList()) { + if (LOG.isInfoEnabled()) { + LOG.info("exe-sql:\n" + result.getExecSql()); + } + boolean isSide = false; + for (String tableName : result.getTargetTableList()) { + if (sqlTree.getTmpTableMap().containsKey(tableName)) { + CreateTmpTableParser.SqlParserResult tmp = sqlTree.getTmpTableMap().get(tableName); + String realSql = DtStringUtil.replaceIgnoreQuota(result.getExecSql(), "`", ""); + + SqlNode sqlNode = org.apache.calcite.sql.parser.SqlParser.create(realSql, CalciteConfig.MYSQL_LEX_CONFIG).parseStmt(); + String tmpSql = ((SqlInsert) sqlNode).getSource().toString(); + tmp.setExecSql(tmpSql); + sideSqlExec.registerTmpTable(tmp, sideTableMap, tableEnv, registerTableCache); + } else { + for (String sourceTable : result.getSourceTableList()) { + if (sideTableMap.containsKey(sourceTable)) { + isSide = true; + break; + } + } + if (isSide) { + //sql-dimensional table contains the dimension table of execution + sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig); + } else { + FlinkSQLExec.sqlUpdate(tableEnv, result.getExecSql(), queryConfig); + if (LOG.isInfoEnabled()) { + LOG.info("exec sql: " + result.getExecSql()); + } + } + } + } + } + } + + public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrlList, TableEnvironment tableEnv) + throws IllegalAccessException, InvocationTargetException { + // udf和tableEnv须由同一个类加载器加载 + ClassLoader levelClassLoader = tableEnv.getClass().getClassLoader(); + URLClassLoader classLoader = null; + List funcList = sqlTree.getFunctionList(); + for (CreateFuncParser.SqlParserResult funcInfo : funcList) { + //classloader + if (classLoader == null) { + classLoader = ClassLoaderManager.loadExtraJar(jarUrlList, (URLClassLoader) levelClassLoader); + } + FunctionManager.registerUDF(funcInfo.getType(), funcInfo.getClassName(), funcInfo.getName(), tableEnv, classLoader); + } + } + + /** + * 向Flink注册源表和结果表，返回执行时插件包的全路径 + * @param sqlTree + * @param env + * @param tableEnv + * @param localSqlPluginPath + * @param remoteSqlPluginPath + * @param pluginLoadMode 插件加载模式 classpath or shipfile + * @param sideTableMap + * @param registerTableCache + * @return + * @throws Exception + */ + public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, + String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { + Set pluginClassPatshSets = Sets.newHashSet(); + WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); + for (TableInfo tableInfo : sqlTree.getTableInfoMap().values()) { + + if (tableInfo instanceof SourceTableInfo) { + + SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; + Table table = StreamSourceFactory.getStreamSource(sourceTableInfo, env, tableEnv, localSqlPluginPath); + tableEnv.registerTable(sourceTableInfo.getAdaptName(), table); + //Note --- parameter conversion function can not be used inside a function of the type of polymerization + //Create table in which the function is arranged only need adaptation sql + String adaptSql = sourceTableInfo.getAdaptSelectSql(); + Table adaptTable = adaptSql == null ? table : tableEnv.sqlQuery(adaptSql); + + RowTypeInfo typeInfo = new RowTypeInfo(adaptTable.getSchema().getFieldTypes(), adaptTable.getSchema().getFieldNames()); + DataStream adaptStream = tableEnv.toRetractStream(adaptTable, typeInfo) + .map((Tuple2 f0) -> { + return f0.f1; + }) + .returns(typeInfo); + + String fields = String.join(",", typeInfo.getFieldNames()); + + if (waterMarkerAssigner.checkNeedAssignWaterMarker(sourceTableInfo)) { + adaptStream = waterMarkerAssigner.assignWaterMarker(adaptStream, typeInfo, sourceTableInfo); + fields += ",ROWTIME.ROWTIME"; + } else { + fields += ",PROCTIME.PROCTIME"; + } + + Table regTable = tableEnv.fromDataStream(adaptStream, fields); + tableEnv.registerTable(tableInfo.getName(), regTable); + if (LOG.isInfoEnabled()) { + LOG.info("registe table {} success.", tableInfo.getName()); + } + registerTableCache.put(tableInfo.getName(), regTable); + + URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), SourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + pluginClassPatshSets.add(sourceTablePathUrl); + } else if (tableInfo instanceof TargetTableInfo) { + + TableSink tableSink = StreamSinkFactory.getTableSink((TargetTableInfo) tableInfo, localSqlPluginPath); + TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); + tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); + + URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), TargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + pluginClassPatshSets.add(sinkTablePathUrl); + } else if (tableInfo instanceof SideTableInfo) { + String sideOperator = ECacheType.ALL.name().equals(((SideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; + sideTableMap.put(tableInfo.getName(), (SideTableInfo) tableInfo); + + URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, SideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + pluginClassPatshSets.add(sideTablePathUrl); + } else { + throw new RuntimeException("not support table type:" + tableInfo.getType()); + } + } + return pluginClassPatshSets; + } + + /** + * perjob模式将job依赖的插件包路径存储到cacheFile，在外围将插件包路径传递给jobgraph + * @param env + * @param classPathSet + */ + public static void registerPluginUrlToCachedFile(StreamExecutionEnvironment env, Set classPathSet) { + int i = 0; + for (URL url : classPathSet) { + String classFileName = String.format(CLASS_FILE_NAME_FMT, i); + env.registerCachedFile(url.getPath(), classFileName, true); + i++; + } + } + + public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperties, String deployMode) throws Exception { + StreamExecutionEnvironment env = !ClusterMode.local.name().equals(deployMode) ? + StreamExecutionEnvironment.getExecutionEnvironment() : + new MyLocalStreamEnvironment(); + + StreamEnvConfigManager.streamExecutionEnvironmentConfig(env, confProperties); + return env; + } +} diff --git a/core/src/main/java/com/dtstack/flink/sql/GetPlan.java b/core/src/main/java/com/dtstack/flink/sql/GetPlan.java new file mode 100644 index 000000000..4f3ab71d4 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/GetPlan.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql; + +import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.dtstack.flink.sql.enums.ClusterMode; +import com.dtstack.flink.sql.environment.MyLocalStreamEnvironment; +import com.dtstack.flink.sql.environment.StreamEnvConfigManager; +import com.dtstack.flink.sql.option.OptionParser; +import com.dtstack.flink.sql.option.Options; +import com.dtstack.flink.sql.parser.SqlParser; +import com.dtstack.flink.sql.parser.SqlTree; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.util.PluginUtil; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.commons.io.Charsets; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.StreamQueryConfig; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.java.StreamTableEnvironment; + +import java.io.File; +import java.net.URL; +import java.net.URLDecoder; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +/** + * 获取sql任务的执行计划 + * Date: 2020/2/17 + * Company: www.dtstack.com + * @author maqi + */ +public class GetPlan { + public static final String STATUS_KEY = "status"; + public static final String MSG_KEY = "msg"; + public static final Integer FAIL = 0; + public static final Integer SUCCESS = 1; + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static String getExecutionPlan(String[] args) { + try { + Arrays.stream(args).forEach(System.out::println); + OptionParser optionParser = new OptionParser(args); + Options options = optionParser.getOptions(); + String sql = options.getSql(); + String addJarListStr = options.getAddjar(); + String localSqlPluginPath = options.getLocalSqlPluginPath(); + String deployMode = ClusterMode.local.name(); + String confProp = options.getConfProp(); + + sql = URLDecoder.decode(sql, Charsets.UTF_8.name()); + SqlParser.setLocalSqlPluginRoot(localSqlPluginPath); + + List addJarFileList = Lists.newArrayList(); + if (!Strings.isNullOrEmpty(addJarListStr)) { + addJarListStr = URLDecoder.decode(addJarListStr, Charsets.UTF_8.name()); + addJarFileList = OBJECT_MAPPER.readValue(addJarListStr, List.class); + } + + confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); + Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); + StreamExecutionEnvironment env = CommonProcess.getStreamExeEnv(confProperties, deployMode); + + StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); + StreamQueryConfig streamQueryConfig = StreamEnvConfigManager.getStreamQueryConfig(tableEnv, confProperties); + + List jarUrlList = Lists.newArrayList(); + SqlTree sqlTree = SqlParser.parseSql(sql); + + //Get External jar to load + for (String addJarPath : addJarFileList) { + File tmpFile = new File(addJarPath); + jarUrlList.add(tmpFile.toURI().toURL()); + } + + Map sideTableMap = Maps.newHashMap(); + Map registerTableCache = Maps.newHashMap(); + + //register udf + CommonProcess.registerUserDefinedFunction(sqlTree, jarUrlList, tableEnv); + //register table schema + Set classPathSets = CommonProcess.registerTable(sqlTree, env, tableEnv, localSqlPluginPath, null, null, sideTableMap, registerTableCache); + // cache classPathSets + CommonProcess.registerPluginUrlToCachedFile(env, classPathSets); + + CommonProcess.sqlTranslation(localSqlPluginPath, tableEnv, sqlTree, sideTableMap, registerTableCache, streamQueryConfig); + + ((MyLocalStreamEnvironment)env).setClasspaths(ClassLoaderManager.getClassPath()); + + String executionPlan = env.getExecutionPlan(); + return getJsonStr(SUCCESS, executionPlan); + } catch (Exception e) { + return getJsonStr(FAIL, ExceptionUtils.getFullStackTrace(e)); + } + } + + public static String getJsonStr(int status, String msg) { + ObjectNode objectNode = OBJECT_MAPPER.createObjectNode(); + objectNode.put(STATUS_KEY, status); + objectNode.put(MSG_KEY, msg); + return objectNode.toString(); + } +} diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index 435117a46..d0e9424d0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -19,60 +19,29 @@ package com.dtstack.flink.sql; -import com.dtstack.flink.sql.config.CalciteConfig; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.enums.ClusterMode; -import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.environment.MyLocalStreamEnvironment; import com.dtstack.flink.sql.environment.StreamEnvConfigManager; -import com.dtstack.flink.sql.exec.FlinkSQLExec; import com.dtstack.flink.sql.option.OptionParser; -import com.dtstack.flink.sql.parser.CreateFuncParser; -import com.dtstack.flink.sql.parser.CreateTmpTableParser; -import com.dtstack.flink.sql.parser.InsertSqlParser; import com.dtstack.flink.sql.parser.SqlParser; import com.dtstack.flink.sql.parser.SqlTree; -import com.dtstack.flink.sql.side.SideSqlExec; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; -import com.dtstack.flink.sql.sink.StreamSinkFactory; -import com.dtstack.flink.sql.source.StreamSourceFactory; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.watermarker.WaterMarkerAssigner; -import com.dtstack.flink.sql.function.FunctionManager; import com.dtstack.flink.sql.util.PluginUtil; -import org.apache.calcite.sql.SqlInsert; -import org.apache.calcite.sql.SqlNode; import org.apache.commons.io.Charsets; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.StreamQueryConfig; import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.types.Row; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; -import java.lang.reflect.InvocationTargetException; import java.net.URL; -import java.net.URLClassLoader; import java.net.URLDecoder; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.Set; @@ -86,12 +55,7 @@ public class Main { - private static final String CLASS_FILE_NAME_FMT = "class_path_%d"; - - private static final ObjectMapper objMapper = new ObjectMapper(); - - private static final Logger LOG = LoggerFactory.getLogger(Main.class); - + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); public static void main(String[] args) throws Exception { @@ -112,12 +76,12 @@ public static void main(String[] args) throws Exception { List addJarFileList = Lists.newArrayList(); if (!Strings.isNullOrEmpty(addJarListStr)) { addJarListStr = URLDecoder.decode(addJarListStr, Charsets.UTF_8.name()); - addJarFileList = objMapper.readValue(addJarListStr, List.class); + addJarFileList = OBJECT_MAPPER.readValue(addJarListStr, List.class); } confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - StreamExecutionEnvironment env = getStreamExeEnv(confProperties, deployMode); + StreamExecutionEnvironment env = CommonProcess.getStreamExeEnv(confProperties, deployMode); StreamTableEnvironment tableEnv = StreamTableEnvironment.getTableEnvironment(env); StreamQueryConfig streamQueryConfig = StreamEnvConfigManager.getStreamQueryConfig(tableEnv, confProperties); @@ -134,174 +98,17 @@ public static void main(String[] args) throws Exception { Map registerTableCache = Maps.newHashMap(); //register udf - registerUserDefinedFunction(sqlTree, jarURList, tableEnv); + CommonProcess.registerUserDefinedFunction(sqlTree, jarURList, tableEnv); //register table schema - Set classPathSets = registerTable(sqlTree, env, tableEnv, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode, sideTableMap, registerTableCache); + Set classPathSets = CommonProcess.registerTable(sqlTree, env, tableEnv, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode, sideTableMap, registerTableCache); // cache classPathSets - registerPluginUrlToCachedFile(env, classPathSets); + CommonProcess.registerPluginUrlToCachedFile(env, classPathSets); - sqlTranslation(localSqlPluginPath, tableEnv, sqlTree, sideTableMap, registerTableCache, streamQueryConfig); + CommonProcess.sqlTranslation(localSqlPluginPath, tableEnv, sqlTree, sideTableMap, registerTableCache, streamQueryConfig); if (env instanceof MyLocalStreamEnvironment) { ((MyLocalStreamEnvironment) env).setClasspaths(ClassLoaderManager.getClassPath()); } - env.execute(name); } - - private static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironment tableEnv,SqlTree sqlTree,Map sideTableMap,Map registerTableCache, StreamQueryConfig queryConfig) throws Exception { - SideSqlExec sideSqlExec = new SideSqlExec(); - sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); - for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { - sideSqlExec.registerTmpTable(result, sideTableMap, tableEnv, registerTableCache); - } - - for (InsertSqlParser.SqlParseResult result : sqlTree.getExecSqlList()) { - if (LOG.isInfoEnabled()) { - LOG.info("exe-sql:\n" + result.getExecSql()); - } - boolean isSide = false; - for (String tableName : result.getTargetTableList()) { - if (sqlTree.getTmpTableMap().containsKey(tableName)) { - CreateTmpTableParser.SqlParserResult tmp = sqlTree.getTmpTableMap().get(tableName); - String realSql = DtStringUtil.replaceIgnoreQuota(result.getExecSql(), "`", ""); - - SqlNode sqlNode = org.apache.calcite.sql.parser.SqlParser.create(realSql, CalciteConfig.MYSQL_LEX_CONFIG).parseStmt(); - String tmpSql = ((SqlInsert) sqlNode).getSource().toString(); - tmp.setExecSql(tmpSql); - sideSqlExec.registerTmpTable(tmp, sideTableMap, tableEnv, registerTableCache); - } else { - for (String sourceTable : result.getSourceTableList()) { - if (sideTableMap.containsKey(sourceTable)) { - isSide = true; - break; - } - } - if (isSide) { - //sql-dimensional table contains the dimension table of execution - sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig); - }else{ - FlinkSQLExec.sqlUpdate(tableEnv, result.getExecSql(), queryConfig); - if(LOG.isInfoEnabled()){ - LOG.info("exec sql: " + result.getExecSql()); - } - } - } - } - } - - - } - - - private static void registerUserDefinedFunction(SqlTree sqlTree, List jarURList, TableEnvironment tableEnv) - throws IllegalAccessException, InvocationTargetException { - // udf和tableEnv须由同一个类加载器加载 - ClassLoader levelClassLoader = tableEnv.getClass().getClassLoader(); - URLClassLoader classLoader = null; - List funcList = sqlTree.getFunctionList(); - for (CreateFuncParser.SqlParserResult funcInfo : funcList) { - //classloader - if (classLoader == null) { - classLoader = ClassLoaderManager.loadExtraJar(jarURList, (URLClassLoader) levelClassLoader); - } - FunctionManager.registerUDF(funcInfo.getType(), funcInfo.getClassName(), funcInfo.getName(), tableEnv, classLoader); - } - } - - /** - * 向Flink注册源表和结果表，返回执行时插件包的全路径 - * @param sqlTree - * @param env - * @param tableEnv - * @param localSqlPluginPath - * @param remoteSqlPluginPath - * @param pluginLoadMode 插件加载模式 classpath or shipfile - * @param sideTableMap - * @param registerTableCache - * @return - * @throws Exception - */ - private static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { - Set pluginClassPatshSets = Sets.newHashSet(); - WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); - for (TableInfo tableInfo : sqlTree.getTableInfoMap().values()) { - - if (tableInfo instanceof SourceTableInfo) { - - SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; - Table table = StreamSourceFactory.getStreamSource(sourceTableInfo, env, tableEnv, localSqlPluginPath); - tableEnv.registerTable(sourceTableInfo.getAdaptName(), table); - //Note --- parameter conversion function can not be used inside a function of the type of polymerization - //Create table in which the function is arranged only need adaptation sql - String adaptSql = sourceTableInfo.getAdaptSelectSql(); - Table adaptTable = adaptSql == null ? table : tableEnv.sqlQuery(adaptSql); - - RowTypeInfo typeInfo = new RowTypeInfo(adaptTable.getSchema().getTypes(), adaptTable.getSchema().getColumnNames()); - DataStream adaptStream = tableEnv.toRetractStream(adaptTable, typeInfo) - .map((Tuple2 f0) -> { return f0.f1; }) - .returns(typeInfo); - - String fields = String.join(",", typeInfo.getFieldNames()); - - if (waterMarkerAssigner.checkNeedAssignWaterMarker(sourceTableInfo)) { - adaptStream = waterMarkerAssigner.assignWaterMarker(adaptStream, typeInfo, sourceTableInfo); - fields += ",ROWTIME.ROWTIME"; - } else { - fields += ",PROCTIME.PROCTIME"; - } - - Table regTable = tableEnv.fromDataStream(adaptStream, fields); - tableEnv.registerTable(tableInfo.getName(), regTable); - if (LOG.isInfoEnabled()) { - LOG.info("registe table {} success.", tableInfo.getName()); - } - registerTableCache.put(tableInfo.getName(), regTable); - - URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), SourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); - pluginClassPatshSets.add(sourceTablePathUrl); - } else if (tableInfo instanceof TargetTableInfo) { - - TableSink tableSink = StreamSinkFactory.getTableSink((TargetTableInfo) tableInfo, localSqlPluginPath); - TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); - tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); - - URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), TargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); - pluginClassPatshSets.add(sinkTablePathUrl); - } else if (tableInfo instanceof SideTableInfo) { - String sideOperator = ECacheType.ALL.name().equals(((SideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; - sideTableMap.put(tableInfo.getName(), (SideTableInfo) tableInfo); - - URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, SideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); - pluginClassPatshSets.add(sideTablePathUrl); - } else { - throw new RuntimeException("not support table type:" + tableInfo.getType()); - } - } - return pluginClassPatshSets; - } - - /** - * perjob模式将job依赖的插件包路径存储到cacheFile，在外围将插件包路径传递给jobgraph - * @param env - * @param classPathSet - */ - private static void registerPluginUrlToCachedFile(StreamExecutionEnvironment env, Set classPathSet) { - int i = 0; - for (URL url : classPathSet) { - String classFileName = String.format(CLASS_FILE_NAME_FMT, i); - env.registerCachedFile(url.getPath(), classFileName, true); - i++; - } - } - - private static StreamExecutionEnvironment getStreamExeEnv(Properties confProperties, String deployMode) throws Exception { - StreamExecutionEnvironment env = !ClusterMode.local.name().equals(deployMode) ? - StreamExecutionEnvironment.getExecutionEnvironment() : - new MyLocalStreamEnvironment(); - - StreamEnvConfigManager.streamExecutionEnvironmentConfig(env, confProperties); - return env; - } } diff --git a/core/src/test/java/com/dtstack/flink/sql/TestGetPlan.java b/core/src/test/java/com/dtstack/flink/sql/TestGetPlan.java new file mode 100644 index 000000000..ed8190887 --- /dev/null +++ b/core/src/test/java/com/dtstack/flink/sql/TestGetPlan.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql; + +import com.dtstack.flink.sql.classloader.DtClassLoader; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.lang.reflect.Method; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Date: 2020/2/17 + * Company: www.dtstack.com + * @author maqi + */ +public class TestGetPlan { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Test + public void testGetExecutionPlan() throws Exception { + List urls = new ArrayList(); + urls.addAll(getJarUrl("/Users/maqi/code/dtstack/dt-center-flinkStreamSQL/plugins/")); + urls.addAll(getJarUrl("/Users/maqi/tmp/flink/flink-1.8.1/lib")); + + ClassLoader parentClassLoader = Thread.currentThread().getContextClassLoader(); + DtClassLoader childClassLoader = new DtClassLoader(urls.toArray(new URL[urls.size()]), parentClassLoader); + + Thread.currentThread().setContextClassLoader(childClassLoader); + Class aClass = childClassLoader.loadClass("com.dtstack.flink.sql.GetPlan"); + + String[] params = { + "-mode", + "local", + "-sql", + "%0d%0aCREATE+TABLE+MyTable(%0d%0a++++id+INT%2c%0d%0a++++channel+VARCHAR%2c%0d%0a++++pv+varchar%2c%0d%0a++++xctime+varchar%2c%0d%0a++++name+varchar%0d%0a+)WITH(%0d%0a++++type+%3d%27kafka11%27%2c%0d%0a++++bootstrapServers+%3d%27172.16.8.107%3a9092%27%2c%0d%0a++++zookeeperQuorum+%3d%27172.16.8.107%3a2181%2fkafka%27%2c%0d%0a++++offsetReset+%3d%27latest%27%2c%0d%0a++++topic+%3d%27mqTest02%27%2c%0d%0a++++timezone%3d%27Asia%2fShanghai%27%2c%0d%0a++++topicIsPattern+%3d%27false%27%2c%0d%0a++++parallelism+%3d%271%27%0d%0a+)%3b%0d%0a%0d%0a%0d%0aCREATE+TABLE+MyTable2(%0d%0a++++id2+INT%2c%0d%0a++++channel2+VARCHAR%2c%0d%0a++++pv2+varchar%2c%0d%0a++++xctime2+varchar%2c%0d%0a++++name2+varchar%0d%0a+)WITH(%0d%0a++++type+%3d%27kafka11%27%2c%0d%0a++++bootstrapServers+%3d%27172.16.8.107%3a9092%27%2c%0d%0a++++zookeeperQuorum+%3d%27172.16.8.107%3a2181%2fkafka%27%2c%0d%0a++++offsetReset+%3d%27latest%27%2c%0d%0a++++topic+%3d%27mqTest03%27%2c%0d%0a++++timezone%3d%27Asia%2fShanghai%27%2c%0d%0a++++topicIsPattern+%3d%27false%27%2c%0d%0a++++parallelism+%3d%271%27%0d%0a+)%3b%0d%0a%0d%0a%0d%0aCREATE+TABLE+sideTableA(%0d%0a++++id1+INT%2c%0d%0a++++channel1+varchar%2c%0d%0a++++time_info+varchar%2c%0d%0a++++name1+varchar%2c%0d%0a++++PRIMARY+KEY(channel1)+%2c%0d%0a++++PERIOD+FOR+SYSTEM_TIME%0d%0a+)WITH(%0d%0a++++type+%3d%27mysql%27%2c%0d%0a++++url+%3d%27jdbc%3amysql%3a%2f%2f172.16.8.109%3a3306%2ftest%27%2c%0d%0a++++userName+%3d%27dtstack%27%2c%0d%0a++++password+%3d%27abc123%27%2c%0d%0a++++tableName+%3d%27dimA%27%2c%0d%0a++++parallelism+%3d%271%27%2c%0d%0a++++cache+%3d+%27LRU%27%0d%0a%0d%0a+)%3b%0d%0a%0d%0a+CREATE+TABLE+sideTableB(%0d%0a++++id+INT%2c%0d%0a++++channel1+varchar%2c%0d%0a++++address+varchar%2c%0d%0a++++PRIMARY+KEY(channel1)%2c%0d%0a++++PERIOD+FOR+SYSTEM_TIME%0d%0a+)WITH(%0d%0a++++type+%3d%27mysql%27%2c%0d%0a++++url+%3d%27jdbc%3amysql%3a%2f%2f172.16.8.109%3a3306%2ftest%27%2c%0d%0a++++userName+%3d%27dtstack%27%2c%0d%0a++++password+%3d%27abc123%27%2c%0d%0a++++tableName+%3d%27dimB%27%2c%0d%0a++++parallelism+%3d%271%27%2c%0d%0a++++cache+%3d+%27LRU%27%2c%0d%0a++++asyncTimeout+%3d+%271000000%27%0d%0a+)%3b%0d%0a%0d%0aCREATE+TABLE+MyResult(%0d%0a++++xctime+VARCHAR%2c%0d%0a++++name+VARCHAR%2c%0d%0a++++time_info+VARCHAR%2c%0d%0a++++address+VARCHAR%0d%0a+)WITH(%0d%0a+++++type+%3d%27mysql%27%2c%0d%0a++++url+%3d%27jdbc%3amysql%3a%2f%2f172.16.8.109%3a3306%2ftest%27%2c%0d%0a++++userName+%3d%27dtstack%27%2c%0d%0a++++password+%3d%27abc123%27%2c%0d%0a++++tableName+%3d%27dimC%27%2c%0d%0a++++parallelism+%3d%271%27%2c%0d%0a+)%3b%0d%0a%0d%0ainsert+%0d%0ainto%0d%0a++MyResult%0d%0a++++select%0d%0a++++++++t1.xctime+as+xctime%2c%0d%0a++++++++t1.name+as+name%2c%0d%0a++++++++t3.channel1+as+time_info%2c%0d%0a++++++++t3.address+as+address+++++%0d%0a++++from%0d%0a++++++++MyTable+as+t1%0d%0a++++left+join%0d%0a++++++++--MyTable2+m2+%0d%0a++++++++sideTableA+m2+++++++++++++++++++++++++++%0d%0a++++++++on++t1.channel+%3d+m2.channel1%0d%0a++++left+join+sideTableB+t3%0d%0a++++++++on+t1.channel+%3d+t3.channel1%0d%0a%0d%0a%0d%0a", + "-localSqlPluginPath", + "/Users/maqi/code/dtstack/dt-center-flinkStreamSQL/plugins", + "-remoteSqlPluginPath", + "/Users/maqi/code/dtstack/dt-center-flinkStreamSQL/plugins", + "-name","test"}; + + Method getExecutionPlan = aClass.getMethod("getExecutionPlan", String[].class); + String jsonStr = (String) getExecutionPlan.invoke(aClass.newInstance(), (Object)params); + + ObjectNode jsonNodes = OBJECT_MAPPER.readValue(jsonStr, ObjectNode.class); + Assert.assertEquals(jsonNodes.get(GetPlan.STATUS_KEY).asLong(), GetPlan.SUCCESS.longValue()); + } + + + public static List getJarUrl(String directory) { + File flinkLibDir = new File(directory); + return Arrays.stream(flinkLibDir.listFiles(((dir1, name) -> name.endsWith(".jar")))).map(file -> { + try { + return file.toURI().toURL(); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + + }).collect(Collectors.toList()); + } +} From a5353ba890e7037796292ad9277755e484592da2 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Tue, 18 Feb 2020 01:55:36 +0800 Subject: [PATCH 020/523] =?UTF-8?q?=E5=A2=9E=E5=8A=A0elasticsearch6-side?= =?UTF-8?q?=E5=8A=9F=E8=83=BD(=E5=BC=82=E6=AD=A5=E8=AF=BB=E5=8F=96?= =?UTF-8?q?=E6=95=B0=E6=8D=AE)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../elasticsearch6-all-side/pom.xml | 14 + ...qRow.java => Elasticsearch6AllReqRow.java} | 98 ++++-- .../Elasticsearch6AllSideInfo.java | 29 +- .../elasticsearch6-async-side/pom.xml | 101 ++++++ .../Elasticsearch6AsyncReqRow.java | 324 ++++++++++++++++++ .../Elasticsearch6AsyncSideInfo.java | 172 ++++++++++ .../side/elasticsearch6/util/MathUtil.java | 255 ++++++++++++++ .../side/elasticsearch6/util/SwitchUtil.java | 82 +++++ .../elasticsearch6-side-core/pom.xml | 11 + .../table/Elasticsearch6SideParser.java | 5 +- .../table/Elasticsearch6SideTableInfo.java | 27 +- elasticsearch6/elasticsearch6-side/pom.xml | 4 +- elasticsearch6/pom.xml | 1 + 13 files changed, 1077 insertions(+), 46 deletions(-) rename elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/{Elasticsearh6AllReqRow.java => Elasticsearch6AllReqRow.java} (75%) create mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/pom.xml create mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java create mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java create mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java create mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/pom.xml b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/pom.xml index cdf45134a..3e4bacebc 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/pom.xml +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/pom.xml @@ -31,6 +31,20 @@ sql.side.elasticsearch6.core ${sql.side.elasticsearch6.core.version} + + + junit + junit + 3.8.1 + test + + + junit + junit + 4.12 + test + + diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearh6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java similarity index 75% rename from elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearh6AllReqRow.java rename to elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index ce3f00a34..8a4fce94e 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearh6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -12,6 +12,7 @@ import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; @@ -40,13 +41,13 @@ * @author yinxi * @date 2020/1/13 - 1:00 */ -public class Elasticsearh6AllReqRow extends AllReqRow { +public class Elasticsearch6AllReqRow extends AllReqRow { + + private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AllReqRow.class); - private static final Logger LOG = LoggerFactory.getLogger(Elasticsearh6AllReqRow.class); - private static final int CONN_RETRY_NUM = 3; private AtomicReference>>> cacheRef = new AtomicReference<>(); - public Elasticsearh6AllReqRow(SideInfo sideInfo) { + public Elasticsearch6AllReqRow(SideInfo sideInfo) { super(sideInfo); } @@ -155,44 +156,55 @@ private void loadData(Map>> tmpCache) throws IO Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); RestHighLevelClient rhlClient = null; - try { - for (int i = 0; i < CONN_RETRY_NUM; i++) { - try { - rhlClient = getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword()); - break; - } catch (Exception e) { - if (i == CONN_RETRY_NUM - 1) { - throw new RuntimeException("", e); - } - - try { - String connInfo = "address: " + tableInfo.getAddress() + ";userName:" + tableInfo.getUserName() + ",pwd:" + tableInfo.getPassword(); - LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); - Thread.sleep(5 * 1000); - } catch (InterruptedException e1) { - LOG.error("", e1); - } - } - } + try{ + rhlClient = getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword(), tableInfo.getTimeout()); // load data from tableA - SearchSourceBuilder searchSourceBuilder = Elasticsearch6AllSideInfo.searchSourceBuilder; + SearchSourceBuilder searchSourceBuilder = tableInfo.getSearchSourceBuilder(); searchSourceBuilder.size(getFetchSize()); - SearchRequest searchRequest = new SearchRequest(tableInfo.getIndex()); - searchRequest.types(tableInfo.getEsType()); + SearchRequest searchRequest = new SearchRequest(); + + // determine existence of index + String index = tableInfo.getIndex().trim(); + if(!StringUtils.isEmpty(index)){ + // strip leading and trailing spaces from a string + String[] indexes = StringUtils.split(index, ","); + for(int i=0; i < indexes.length; i++ ){ + indexes[i] = indexes[i].trim(); + } + + searchRequest.indices(indexes); + + } + + // determine existence of type + String type = tableInfo.getEsType().trim(); + if(!StringUtils.isEmpty(type)){ + // strip leading and trailing spaces from a string + String[] types = StringUtils.split(type, ","); + for(int i=0; i < types.length; i++ ){ + types[i] = types[i].trim(); + } + + searchRequest.types(types); + } + + // add query condition searchRequest.source(searchSourceBuilder); + // get query reults SearchResponse searchResponse = rhlClient.search(searchRequest); SearchHit[] searchHits = searchResponse.getHits().getHits(); - String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); - String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); + + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); + String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); Map oneRow = Maps.newHashMap(); for (SearchHit searchHit : searchHits) { for(String fieldName : sideFieldNames){ Object object = searchHit.getSourceAsMap().get(fieldName.trim()); int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); - object = SwitchUtil.getTarget(object, fields[fieldIndex]); + object = SwitchUtil.getTarget(object, sideFieldTypes[fieldIndex]); oneRow.put(fieldName.trim(), object); } @@ -212,11 +224,11 @@ private void loadData(Map>> tmpCache) throws IO } - public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, String userName, String password) { + public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, String userName, String password, Integer timeout) { List httpHostList = new ArrayList<>(); - String[] address = esAddress.split(","); + String[] address = StringUtils.split(esAddress, ","); for (String addr : address) { - String[] infoArray = addr.split(":"); + String[] infoArray = StringUtils.split(addr, ":"); int port = 9200; String host = infoArray[0].trim(); if (infoArray.length > 1) { @@ -226,6 +238,11 @@ public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, Strin } RestClientBuilder restClientBuilder = RestClient.builder(httpHostList.toArray(new HttpHost[httpHostList.size()])); + + if (timeout != null) { + restClientBuilder.setMaxRetryTimeoutMillis(timeout * 1000); + } + if (isAuthMesh) { // 进行用户和密码认证 final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); @@ -236,6 +253,23 @@ public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, Strin RestHighLevelClient rhlClient = new RestHighLevelClient(restClientBuilder); + if (LOG.isInfoEnabled()) { + LOG.info("Pinging Elasticsearch cluster via hosts {} ...", httpHostList); + } + + try{ + if (!rhlClient.ping()) { + throw new RuntimeException("There are no reachable Elasticsearch nodes!"); + } + } catch (IOException e){ + LOG.warn("", e); + } + + + if (LOG.isInfoEnabled()) { + LOG.info("Created Elasticsearch RestHighLevelClient connected to {}", httpHostList.toString()); + } + return rhlClient; } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java index 853f07dd7..acf39f989 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java @@ -21,13 +21,17 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; @@ -37,7 +41,7 @@ */ public class Elasticsearch6AllSideInfo extends SideInfo { - public static SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AllSideInfo.class); public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); @@ -46,12 +50,14 @@ public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, Lis @Override public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { - getSelectFromStatement(sideTableInfo.getPredicateInfoes()); + Elasticsearch6SideTableInfo elasticsearch6SideTableInfo = (Elasticsearch6SideTableInfo) sideTableInfo; - } + elasticsearch6SideTableInfo.setSearchSourceBuilder(getSelectFromStatement(sideTableInfo.getPredicateInfoes())); - private void getSelectFromStatement(List predicateInfoes) { + } + private SearchSourceBuilder getSelectFromStatement(List predicateInfoes) { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); if (predicateInfoes.size() != 0) { BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); for (PredicateInfo info : sideTableInfo.getPredicateInfoes()) { @@ -61,10 +67,15 @@ private void getSelectFromStatement(List predicateInfoes) { searchSourceBuilder.query(boolQueryBuilder); } + return searchSourceBuilder; } - public BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info){ + public BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info) { switch (info.getOperatorKind()) { + case "IN": + return boolQueryBuilder.must(QueryBuilders.termsQuery(info.getFieldName(), StringUtils.split(info.getCondition().trim(), ","))); + case "NOT_IN": + return boolQueryBuilder.mustNot(QueryBuilders.termsQuery(info.getFieldName(), StringUtils.split(info.getCondition().trim(), ","))); case "GREATER_THAN_OR_EQUAL": return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gte(info.getCondition())); case "GREATER_THAN": @@ -75,15 +86,19 @@ public BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lt(info.getCondition())); case "EQUALS": return boolQueryBuilder.must(QueryBuilders.termQuery(info.getFieldName(), info.getCondition())); + case "NOT_EQUALS": + return boolQueryBuilder.mustNot(QueryBuilders.termQuery(info.getFieldName(), info.getCondition())); default: try { - throw new Exception("Predicate does not match!"); + throw new Exception("elasticsearch6 does not support this operation: " + info.getOperatorName()); } catch (Exception e) { + e.printStackTrace(); + LOG.error(e.getMessage()); } + return boolQueryBuilder; } - return boolQueryBuilder; } @Override diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/pom.xml b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/pom.xml new file mode 100644 index 000000000..db2d49f8c --- /dev/null +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/pom.xml @@ -0,0 +1,101 @@ + + + + sql.side.elasticsearch6 + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.side.async.elasticsearch6 + elasticsearch6-async-side + jar + + + 1.0-SNAPSHOT + 6.8.6 + + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} + + + + com.dtstack.flink + sql.side.elasticsearch6.core + ${sql.side.elasticsearch6.core.version} + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java new file mode 100644 index 000000000..11b0d383e --- /dev/null +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.elasticsearch6; + +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; + +import com.dtstack.flink.sql.enums.ECacheContentType; +import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.cache.CacheObj; +import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; +import com.dtstack.flink.sql.side.elasticsearch6.util.SwitchUtil; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * @author yinxi + * @date 2020/2/13 - 13:10 + */ +public class Elasticsearch6AsyncReqRow extends AsyncReqRow { + + private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AsyncReqRow.class); + + private static final int CONN_RETRY_NUM = 3; + + public final static int DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE = 1; + + public final static int DEFAULT_VERTX_WORKER_POOL_SIZE = Runtime.getRuntime().availableProcessors() * 2; + + public final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; + + public final static int DEFAULT_IDLE_CONNECTION_TEST_PEROID = 60; + + public final static boolean DEFAULT_TEST_CONNECTION_ON_CHECKIN = true; + + public final static String PREFERRED_TEST_QUERY_SQL = "select 1 from dual"; + + private transient RestHighLevelClient rhlClient; + + public Elasticsearch6AsyncReqRow(SideInfo sideInfo) { + super(sideInfo); + } + + @Override + public void asyncInvoke(Row input, ResultFuture resultFuture) throws Exception { + Row inputRow = Row.copy(input); + List inputParams = Lists.newArrayList(); +// JsonArray inputParams = new JsonArray(); + for (Integer conValIndex : sideInfo.getEqualValIndex()) { + Object equalObj = inputRow.getField(conValIndex); + if (equalObj == null) { + dealMissKey(inputRow, resultFuture); + return; + } + inputParams.add(equalObj); + } + + String key = buildCacheKey(inputParams); + if (openCache()) { + CacheObj val = getFromCache(key); + if (val != null) { + if (ECacheContentType.MissVal == val.getType()) { + dealMissKey(inputRow, resultFuture); + return; + } else if (ECacheContentType.MultiLine == val.getType()) { + try { + List rowList = getRows(inputRow, null, (List) val.getContent()); + resultFuture.complete(rowList); + } catch (Exception e) { + dealFillDataError(resultFuture, e, inputRow); + } + } else { + resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); + } + return; + } + } + + Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); + RestHighLevelClient rhlClient = null; + try{ + rhlClient = getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword(), tableInfo.getTimeout()); + + // load data from tableA + SearchSourceBuilder searchSourceBuilder = tableInfo.getSearchSourceBuilder(); +// searchSourceBuilder.size(getFetchSize()); + SearchRequest searchRequest = new SearchRequest(); + + // determine existence of index + String index = tableInfo.getIndex().trim(); + if(!StringUtils.isEmpty(index)){ + // strip leading and trailing spaces from a string + String[] indexes = StringUtils.split(index, ","); + for(int i=0; i < indexes.length; i++ ){ + indexes[i] = indexes[i].trim(); + } + + searchRequest.indices(indexes); + + } + + // determine existence of type + String type = tableInfo.getEsType().trim(); + if(!StringUtils.isEmpty(type)){ + // strip leading and trailing spaces from a string + String[] types = StringUtils.split(type, ","); + for(int i=0; i < types.length; i++ ){ + types[i] = types[i].trim(); + } + + searchRequest.types(types); + } + + // add query condition + searchRequest.source(searchSourceBuilder); + + // 异步查询数据 + + rhlClient.searchAsync(searchRequest, RequestOptions.DEFAULT ,new ActionListener() { + + //成功响应时 + @Override + public void onResponse(SearchResponse searchResponse) { + SearchHit[] searchHits = searchResponse.getHits().getHits(); + List cacheContent = Lists.newArrayList(); + List results = Lists.newArrayList(); + if(searchHits.length > 0){ + try{ + for(SearchHit searchHit : searchHits){ + Map object = searchHit.getSourceAsMap(); + results.add(object); + } + + List rowList = getRows(inputRow, cacheContent, results); + dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); + resultFuture.complete(rowList); + }catch (Exception e){ + dealFillDataError(resultFuture, e, inputRow); + } + }else{ + dealMissKey(inputRow, resultFuture); + dealCacheData(key, CacheMissVal.getMissKeyObj()); + } + + + } + + // 响应失败处理 + @Override + public void onFailure(Exception e) { + resultFuture.completeExceptionally(new RuntimeException("Response timeout!")); + } + }); + + + + + } catch (Exception e) { + LOG.error("", e); + } finally { + if (rhlClient != null) { + rhlClient.close(); + } + } + + } + + protected List getRows(Row inputRow, List cacheContent, List results) { + List rowList = Lists.newArrayList(); + for (Object line : results) { + Row row = fillData(inputRow, line); + if (null != cacheContent && openCache()) { + cacheContent.add(line); + } + rowList.add(row); + } + return rowList; + } + + @Override + public Row fillData(Row input, Object line) { +// JsonArray jsonArray = (JsonArray) line; + Map cacheInfo = (Map) line; + Row row = new Row(sideInfo.getOutFieldInfoList().size()); + String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); + for (Map.Entry entry : sideInfo.getInFieldIndex().entrySet()) { + Object obj = input.getField(entry.getValue()); + boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); + if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { + obj = ((Timestamp) obj).getTime(); + } + + row.setField(entry.getKey(), obj); + } + + for (Map.Entry entry : sideInfo.getSideFieldIndex().entrySet()) { + if (cacheInfo == null) { + row.setField(entry.getKey(), null); + } else { + Object object = SwitchUtil.getTarget(cacheInfo.get(entry.getValue()), fields[entry.getValue()]); + row.setField(entry.getKey(), object); + } + } + + return row; + } + + @Override + public void close() throws Exception { + super.close(); + if (rhlClient != null) { + rhlClient.close(); + } + + } + + public String buildCacheKey(List equalValList) { + StringBuilder sb = new StringBuilder(); + for (Object ele : equalValList) { + sb.append(ele.toString()) + .append("_"); + } + + return sb.toString(); + } + +// public void setRdbSQLClient(SQLClient rdbSQLClient) { +// this.rdbSQLClient = rdbSQLClient; +// } + + public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, String userName, String password, Integer timeout) { + + List httpHostList = new ArrayList<>(); + String[] address = StringUtils.split(esAddress, ","); + for (String addr : address) { + String[] infoArray = StringUtils.split(addr, ":"); + int port = 9200; + String host = infoArray[0].trim(); + if (infoArray.length > 1) { + port = Integer.valueOf(infoArray[1].trim()); + } + httpHostList.add(new HttpHost(host, port, "http")); + } + + RestClientBuilder restClientBuilder = RestClient.builder(httpHostList.toArray(new HttpHost[httpHostList.size()])); + + if (timeout != null) { + restClientBuilder.setMaxRetryTimeoutMillis(timeout * 1000); + } + + if (isAuthMesh) { + // 进行用户和密码认证 + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password)); + restClientBuilder.setHttpClientConfigCallback(httpAsyncClientBuilder -> + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + + RestHighLevelClient rhlClient = new RestHighLevelClient(restClientBuilder); + + if (LOG.isInfoEnabled()) { + LOG.info("Pinging Elasticsearch cluster via hosts {} ...", httpHostList); + } + + try { + if (!rhlClient.ping()) { + throw new RuntimeException("There are no reachable Elasticsearch nodes!"); + } + } catch (IOException e) { + LOG.warn("", e); + } + + + if (LOG.isInfoEnabled()) { + LOG.info("Created Elasticsearch RestHighLevelClient connected to {}", httpHostList.toString()); + } + + return rhlClient; + + } + +} + diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java new file mode 100644 index 000000000..dd5ba1e46 --- /dev/null +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.elasticsearch6; + +import org.apache.flink.api.java.typeutils.RowTypeInfo; + +import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; +import com.dtstack.flink.sql.util.ParseUtils; +import com.google.common.collect.Lists; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.commons.lang.StringUtils; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import java.util.Arrays; +import java.util.List; + +/** + * @author yinxi + * @date 2020/2/13 - 13:09 + */ +public class Elasticsearch6AsyncSideInfo extends SideInfo { + + + public Elasticsearch6AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + } + + @Override + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + Elasticsearch6SideTableInfo elasticsearch6SideTableInfo = (Elasticsearch6SideTableInfo) sideTableInfo; + + String sideTableName = joinInfo.getSideTableName(); + SqlNode conditionNode = joinInfo.getCondition(); + + List sqlNodeList = Lists.newArrayList(); + List sqlJoinCompareOperate= Lists.newArrayList(); + + ParseUtils.parseAnd(conditionNode, sqlNodeList); + ParseUtils.parseJoinCompareOperate(conditionNode, sqlJoinCompareOperate); + + for (SqlNode sqlNode : sqlNodeList) { + dealOneEqualCon(sqlNode, sideTableName); + } + + // set query condition + elasticsearch6SideTableInfo.setSearchSourceBuilder(getSelectFromStatement(sideTableInfo.getPredicateInfoes())); + + } + + private SearchSourceBuilder getSelectFromStatement(List predicateInfoes) { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + + if (predicateInfoes.size() != 0) { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + for (PredicateInfo info : sideTableInfo.getPredicateInfoes()) { + boolQueryBuilder = buildFilterCondition(boolQueryBuilder, info); + } + + searchSourceBuilder.query(boolQueryBuilder); + } + + return searchSourceBuilder; + } + + public BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info){ + switch (info.getOperatorKind()) { + case "GREATER_THAN_OR_EQUAL": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gte(info.getCondition())); + case "GREATER_THAN": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gt(info.getCondition())); + case "LESS_THAN_OR_EQUAL": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lte(info.getCondition())); + case "LESS_THAN": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lt(info.getCondition())); + case "EQUALS": + return boolQueryBuilder.must(QueryBuilders.termQuery(info.getFieldName(), info.getCondition())); + case "NOT_EQUALS": + return boolQueryBuilder.mustNot(QueryBuilders.termQuery(info.getFieldName(), info.getCondition())); + case "LIKE": + return boolQueryBuilder.must(QueryBuilders.fuzzyQuery(info.getFieldName(), info.getCondition())); + case "IN": + return boolQueryBuilder.must(QueryBuilders.termsQuery(info.getFieldName(), Arrays.asList(StringUtils.split(info.getCondition().trim(), ",")))); + case "NOT_IN": + return boolQueryBuilder.mustNot(QueryBuilders.termsQuery(info.getFieldName(), Arrays.asList(StringUtils.split(info.getCondition().trim(), ",")))); + default: + try { + throw new Exception("Predicate does not match!"); + } catch (Exception e) { + e.printStackTrace(); + } + + return boolQueryBuilder; + } + + } + + + @Override + public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { + if (!SqlKind.COMPARISON.contains(sqlNode.getKind())) { + throw new RuntimeException("not compare operator."); + } + + SqlIdentifier left = (SqlIdentifier) ((SqlBasicCall) sqlNode).getOperands()[0]; + SqlIdentifier right = (SqlIdentifier) ((SqlBasicCall) sqlNode).getOperands()[1]; + + String leftTableName = left.getComponent(0).getSimple(); + String leftField = left.getComponent(1).getSimple(); + + String rightTableName = right.getComponent(0).getSimple(); + String rightField = right.getComponent(1).getSimple(); + + if (leftTableName.equalsIgnoreCase(sideTableName)) { + equalFieldList.add(leftField); + int equalFieldIndex = -1; + for (int i = 0; i < rowTypeInfo.getFieldNames().length; i++) { + String fieldName = rowTypeInfo.getFieldNames()[i]; + if (fieldName.equalsIgnoreCase(rightField)) { + equalFieldIndex = i; + } + } + if (equalFieldIndex == -1) { + throw new RuntimeException("can't deal equal field: " + sqlNode); + } + + equalValIndex.add(equalFieldIndex); + + } else if (rightTableName.equalsIgnoreCase(sideTableName)) { + + equalFieldList.add(rightField); + int equalFieldIndex = -1; + for (int i = 0; i < rowTypeInfo.getFieldNames().length; i++) { + String fieldName = rowTypeInfo.getFieldNames()[i]; + if (fieldName.equalsIgnoreCase(leftField)) { + equalFieldIndex = i; + } + } + if (equalFieldIndex == -1) { + throw new RuntimeException("can't deal equal field: " + sqlNode.toString()); + } + + equalValIndex.add(equalFieldIndex); + + } else { + throw new RuntimeException("resolve equalFieldList error:" + sqlNode.toString()); + } + + } + +} diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java new file mode 100644 index 000000000..7765ddf17 --- /dev/null +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.dtstack.flink.sql.side.elasticsearch6.util; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Date; +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; + +/** + * Date: 2017/4/21 + * Company: www.dtstack.com + * + * @author xuchao + */ + +public class MathUtil { + + public static Long getLongVal(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Long.valueOf((String) obj); + } else if (obj instanceof Long) { + return (Long) obj; + } else if (obj instanceof Integer) { + return Long.valueOf(obj.toString()); + } else if (obj instanceof BigDecimal) { + return ((BigDecimal) obj).longValue(); + } else if (obj instanceof BigInteger) { + return ((BigInteger) obj).longValue(); + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Long."); + } + + public static Long getLongVal(Object obj, long defaultVal) { + if (obj == null) { + return defaultVal; + } + + return getLongVal(obj); + } + + public static Integer getIntegerVal(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Integer.valueOf((String) obj); + } else if (obj instanceof Integer) { + return (Integer) obj; + } else if (obj instanceof Long) { + return ((Long) obj).intValue(); + } else if (obj instanceof Double) { + return ((Double) obj).intValue(); + } else if (obj instanceof BigDecimal) { + return ((BigDecimal) obj).intValue(); + } else if (obj instanceof BigInteger) { + return ((BigInteger) obj).intValue(); + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Integer."); + } + + public static Integer getIntegerVal(Object obj, int defaultVal) { + if (obj == null) { + return defaultVal; + } + + return getIntegerVal(obj); + } + + public static Float getFloatVal(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Float.valueOf((String) obj); + } else if (obj instanceof Float) { + return (Float) obj; + } else if (obj instanceof BigDecimal) { + return ((BigDecimal) obj).floatValue(); + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Float."); + } + + public static Float getFloatVal(Object obj, float defaultVal) { + if (obj == null) { + return defaultVal; + } + + return getFloatVal(obj); + } + + public static Double getDoubleVal(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Double.valueOf((String) obj); + } else if (obj instanceof Float) { + return (Double) obj; + } else if (obj instanceof BigDecimal) { + return ((BigDecimal) obj).doubleValue(); + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Double."); + } + + public static Double getDoubleVal(Object obj, double defaultVal) { + if (obj == null) { + return defaultVal; + } + + return getDoubleVal(obj); + } + + + public static Boolean getBoolean(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Boolean.valueOf((String) obj); + } else if (obj instanceof Boolean) { + return (Boolean) obj; + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Boolean."); + } + + public static Boolean getBoolean(Object obj, boolean defaultVal) { + if (obj == null) { + return defaultVal; + } + + return getBoolean(obj); + } + + public static String getString(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return (String) obj; + } + + return obj.toString(); + } + + public static Byte getByte(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Byte.valueOf((String) obj); + } else if (obj instanceof Byte) { + return (Byte) obj; + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Byte."); + } + + public static Short getShort(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof String) { + return Short.valueOf((String) obj); + } else if (obj instanceof Short) { + return (Short) obj; + } + + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Short."); + } + + public static BigDecimal getBigDecimal(Object obj) { + if (obj == null) { + return null; + } + if (obj instanceof String) { + return new BigDecimal((String) obj); + } else if (obj instanceof BigDecimal) { + return (BigDecimal) obj; + } else if (obj instanceof BigInteger) { + return new BigDecimal((BigInteger) obj); + } else if (obj instanceof Number) { + return new BigDecimal(((Number) obj).doubleValue()); + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to BigDecimal."); + } + + public static Date getDate(Object obj) { + if (obj == null) { + return null; + } + if (obj instanceof String) { + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + try { + return new Date(format.parse((String) obj).getTime()); + } catch (ParseException e) { + throw new RuntimeException("String convert to Date fail."); + } + } else if (obj instanceof Timestamp) { + return new Date(((Timestamp) obj).getTime()); + } else if (obj instanceof Date) { + return (Date) obj; + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); + } + + public static Timestamp getTimestamp(Object obj) { + if (obj == null) { + return null; + } + if (obj instanceof Timestamp) { + return (Timestamp) obj; + } else if (obj instanceof Date) { + return new Timestamp(((Date) obj).getTime()); + } else if (obj instanceof String) { + return new Timestamp(getDate(obj).getTime()); + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); + } + +} \ No newline at end of file diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java new file mode 100644 index 000000000..5f2e08651 --- /dev/null +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.flink.sql.side.elasticsearch6.util; + +/** + * Reason: + * Date: 2018/12/3 + * Company: www.dtstack.com + * + * @author maqi + */ +public class SwitchUtil { + public static Object getTarget(Object obj, String targetType) { + targetType = targetType.toLowerCase(); + switch (targetType) { + + case "smallint": + case "smallintunsigned": + case "tinyint": + case "tinyintunsigned": + case "mediumint": + case "mediumintunsigned": + case "integer": + case "int": + return MathUtil.getIntegerVal(obj); + + case "bigint": + case "bigintunsigned": + case "intunsigned": + case "integerunsigned": + return MathUtil.getLongVal(obj); + + case "boolean": + return MathUtil.getBoolean(obj); + + case "blob": + return MathUtil.getByte(obj); + + case "varchar": + case "char": + case "text": + return MathUtil.getString(obj); + + case "real": + case "float": + case "realunsigned": + case "floatunsigned": + return MathUtil.getFloatVal(obj); + + case "double": + case "doubleunsigned": + return MathUtil.getDoubleVal(obj); + + case "decimal": + case "decimalunsigned": + return MathUtil.getBigDecimal(obj); + + case "date": + return MathUtil.getDate(obj); + + case "timestamp": + case "datetime": + return MathUtil.getTimestamp(obj); + } + return obj; + } +} diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/pom.xml b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/pom.xml index 679881182..73c46fbb5 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/pom.xml +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/pom.xml @@ -15,6 +15,17 @@ 1.0-SNAPSHOT jar + + 6.8.6 + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} + + \ No newline at end of file diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java index a40f33575..fee2d2d8e 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java @@ -19,7 +19,7 @@ public class Elasticsearch6SideParser extends AbsSideTableParser { private static final String KEY_ES6_INDEX = "index"; - private static final String KEY_ES6_TYPE = "estype"; + private static final String KEY_ES6_TYPE = "esType"; private static final String KEY_ES6_AUTHMESH = "authMesh"; @@ -27,6 +27,8 @@ public class Elasticsearch6SideParser extends AbsSideTableParser { private static final String KEY_ES6_PASSWORD = "password"; + private static final String KEY_ES6_TIMEOUT = "timeout"; + @Override protected boolean fieldNameNeedsUpperCase() { return false; @@ -41,6 +43,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Mappom elasticsearch6-side - - + elasticsearch6-all-side + elasticsearch6-async-side elasticsearch6-side-core diff --git a/elasticsearch6/pom.xml b/elasticsearch6/pom.xml index 624bd174a..38472cd4e 100644 --- a/elasticsearch6/pom.xml +++ b/elasticsearch6/pom.xml @@ -13,6 +13,7 @@ pom elasticsearch6-sink + elasticsearch6-side From daf1e24d8094a5b7c38208f49c928848e96aca56 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 18 Feb 2020 11:17:13 +0800 Subject: [PATCH 021/523] hbase retract sink modify --- .../sql/sink/clickhouse/ClickhouseSink.java | 15 +++--- .../com/dtstack/flink/sql/sink/db/DbSink.java | 2 - .../sql/sink/hbase/HbaseOutputFormat.java | 29 ++++++------ .../flink/sql/sink/impala/EAuthMech.java | 46 +++++++++++++++++++ .../flink/sql/sink/impala/ImpalaDialect.java | 4 +- .../flink/sql/sink/impala/ImpalaSink.java | 35 ++++++-------- .../flink/sql/sink/kudu/KuduOutputFormat.java | 25 ++++++++-- .../flink/sql/sink/mysql/MysqlSink.java | 12 +++-- .../flink/sql/sink/oracle/OracleSink.java | 13 ++++-- .../flink/sql/sink/polardb/PolardbSink.java | 3 -- .../sql/sink/postgresql/PostgresqlSink.java | 15 ++++-- .../rdb/format/JDBCUpsertOutputFormat.java | 5 +- .../flink/sql/sink/rdb/writer/JDBCWriter.java | 2 +- .../sql/sink/sqlserver/SqlserverSink.java | 10 ++-- 14 files changed, 143 insertions(+), 73 deletions(-) create mode 100644 impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/EAuthMech.java diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java index 6e7a3eef4..bbb6a6fe4 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java @@ -30,9 +30,6 @@ public class ClickhouseSink extends RdbSink implements IStreamSinkGener { - - private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - public ClickhouseSink() { super(new ClickhouseDialect()); } @@ -40,9 +37,12 @@ public ClickhouseSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL).setDialect(jdbcDialect) - .setUsername(userName).setPassword(password) - .setTableName(tableName).build(); + .setDBUrl(dbURL) + .setDialect(jdbcDialect) + .setUsername(userName) + .setPassword(password) + .setTableName(tableName) + .build(); return JDBCUpsertOutputFormat.builder() .setOptions(jdbcOptions) @@ -52,7 +52,8 @@ public JDBCUpsertOutputFormat getOutputFormat() { .setFieldTypes(sqlTypes) .setKeyFields(primaryKeys) .setAllReplace(allReplace) - .setUpdateMode(updateMode).build(); + .setUpdateMode(updateMode) + .build(); } diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java index 8e6fb4bd7..119af515c 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java @@ -9,8 +9,6 @@ public class DbSink extends RdbSink { - private static final String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; - public DbSink() { super(new DbDialect()); } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 79c92e2e2..409cf312f 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -46,7 +46,7 @@ import java.util.Set; /** - * author: jingzhen@dtstack.com + * @author: jingzhen@dtstack.com * date: 2017-6-29 */ public class HbaseOutputFormat extends DtRichOutputFormat { @@ -94,16 +94,16 @@ public void open(int taskNumber, int numTasks) throws IOException { @Override public void writeRecord(Tuple2 tuple2) { Tuple2 tupleTrans = tuple2; - Boolean retract = tupleTrans.getField(0); - if (!retract && StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { - dealDelete(tupleTrans); - } else { - dealInsert(tupleTrans); + Boolean retract = tupleTrans.f0; + Row row = tupleTrans.f1; + if (retract) { + dealInsert(row); + } else if (!retract && StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + dealDelete(row); } } - protected void dealInsert(Tuple2 tupleTrans) { - Row record = tupleTrans.getField(1); + protected void dealInsert(Row record) { Put put = getPutByRow(record); if (put == null) { return; @@ -112,11 +112,11 @@ protected void dealInsert(Tuple2 tupleTrans) { try { table.put(put); } catch (IOException e) { - outDirtyRecords.inc(); if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { - LOG.error("record insert failed ..", record.toString()); + LOG.error("record insert failed ..{}", record.toString()); LOG.error("", e); } + outDirtyRecords.inc(); } if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0) { @@ -125,19 +125,18 @@ protected void dealInsert(Tuple2 tupleTrans) { outRecords.inc(); } - protected void dealDelete(Tuple2 tupleTrans) { - Row record = tupleTrans.getField(1); + protected void dealDelete(Row record) { String rowKey = buildRowKey(record); if (!StringUtils.isEmpty(rowKey)) { Delete delete = new Delete(Bytes.toBytes(rowKey)); try { table.delete(delete); } catch (IOException e) { - outDirtyRecords.inc(); if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { - LOG.error("record insert failed ..", record.toString()); + LOG.error("record insert failed ..{}", record.toString()); LOG.error("", e); } + outDirtyRecords.inc(); } if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0) { LOG.info(record.toString()); @@ -170,7 +169,7 @@ private String buildRowKey(Row record) { List rowKeyValues = getRowKeyValues(record); // all rowkey not null if (rowKeyValues.size() != rowkey.length) { - LOG.error("row key value must not null,record is ..", record); + LOG.error("row key value must not null,record is ..{}", record); outDirtyRecords.inc(); return ""; } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/EAuthMech.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/EAuthMech.java new file mode 100644 index 000000000..3753a54bd --- /dev/null +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/EAuthMech.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.impala; + +/** + * impala kdnc AuthMech params + * Date: 2020/2/18 + * Company: www.dtstack.com + * @author maqi + */ +public enum EAuthMech { + // 0 for No Authentication + NoAuthentication(0), + // 1 for Kerberos + Kerberos(1), + // 2 for User Name + UserName(2), + // 3 for User Name and Password + NameANDPassword(3); + + private int type; + + EAuthMech(int type) { + this.type = type; + } + + public int getType() { + return this.type; + } +} diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java index d8523eeb7..7bb24481a 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java @@ -34,7 +34,7 @@ public class ImpalaDialect implements JDBCDialect { private static final long serialVersionUID = 1L; - private static final String IMPALA_PARTITION_KEYWORD = " partition"; + private static final String IMPALA_PARTITION_KEYWORD = "partition"; @Override public boolean canHandle(String url) { @@ -76,7 +76,7 @@ public String getInsertIntoStatement(String schema, String tableName, String[] f .map(field -> field.replaceAll("\"", "'")) .collect(Collectors.joining(", ")); - String partitionStatement = StringUtils.isEmpty(partitionFieldStr) ? "" : IMPALA_PARTITION_KEYWORD + "(" + partitionFieldStr + ")"; + String partitionStatement = StringUtils.isEmpty(partitionFieldStr) ? "" : " " + IMPALA_PARTITION_KEYWORD + "(" + partitionFieldStr + ")"; return "INSERT INTO " + schemaInfo + quoteIdentifier(tableName) + "(" + columns + ")" + partitionStatement + " VALUES (" + placeholders + ")"; diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index d60f302e1..41fa00926 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -22,23 +22,12 @@ import com.dtstack.flink.sql.sink.impala.table.ImpalaTableInfo; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.RdbSink; -import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.table.TargetTableInfo; -import com.dtstack.flink.sql.util.JDBCUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction; -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import java.io.IOException; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; /** * Date: 2019/11/11 @@ -58,9 +47,12 @@ public ImpalaSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(getImpalaJdbcUrl()).setDialect(jdbcDialect) - .setUsername(userName).setPassword(password) - .setTableName(tableName).build(); + .setDBUrl(getImpalaJdbcUrl()) + .setDialect(jdbcDialect) + .setUsername(userName) + .setPassword(password) + .setTableName(tableName) + .build(); return JDBCUpsertOutputFormat.builder() .setOptions(jdbcOptions) @@ -71,7 +63,8 @@ public JDBCUpsertOutputFormat getOutputFormat() { .setKeyFields(primaryKeys) .setPartitionFields(impalaTableInfo.getPartitionFields()) .setAllReplace(allReplace) - .setUpdateMode(updateMode).build(); + .setUpdateMode(updateMode) + .build(); } @@ -79,19 +72,19 @@ public String getImpalaJdbcUrl() { Integer authMech = impalaTableInfo.getAuthMech(); String newUrl = dbURL; StringBuffer urlBuffer = new StringBuffer(dbURL); - if (authMech == 0) { + if (authMech == EAuthMech.NoAuthentication.getType()) { return newUrl; - } else if (authMech == 1) { + } else if (authMech == EAuthMech.Kerberos.getType()) { String keyTabFilePath = impalaTableInfo.getKeyTabFilePath(); String krb5FilePath = impalaTableInfo.getKrb5FilePath(); String principal = impalaTableInfo.getPrincipal(); String krbRealm = impalaTableInfo.getKrbRealm(); - String krbHostFQDN = impalaTableInfo.getKrbHostFQDN(); + String krbHostFqdn = impalaTableInfo.getKrbHostFQDN(); String krbServiceName = impalaTableInfo.getKrbServiceName(); urlBuffer.append(";" .concat("AuthMech=1;") .concat("KrbRealm=").concat(krbRealm).concat(";") - .concat("KrbHostFQDN=").concat(krbHostFQDN).concat(";") + .concat("KrbHostFQDN=").concat(krbHostFqdn).concat(";") .concat("KrbServiceName=").concat(krbServiceName).concat(";") ); newUrl = urlBuffer.toString(); @@ -106,7 +99,7 @@ public String getImpalaJdbcUrl() { throw new RuntimeException("loginUserFromKeytab error ..", e); } - } else if (authMech == 2) { + } else if (authMech == EAuthMech.UserName.getType()) { urlBuffer.append(";" .concat("AuthMech=3;") .concat("UID=").concat(userName).concat(";") @@ -114,7 +107,7 @@ public String getImpalaJdbcUrl() { .concat("UseSasl=0") ); newUrl = urlBuffer.toString(); - } else if (authMech == 3) { + } else if (authMech == EAuthMech.NameANDPassword.getType()) { urlBuffer.append(";" .concat("AuthMech=3;") .concat("UID=").concat(userName).concat(";") diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index c36c7eea6..780319d82 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -23,7 +23,13 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; -import org.apache.kudu.client.*; +import org.apache.kudu.client.AsyncKuduClient; +import org.apache.kudu.client.AsyncKuduSession; +import org.apache.kudu.client.KuduClient; +import org.apache.kudu.client.KuduException; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.Operation; +import org.apache.kudu.client.PartialRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,13 +38,24 @@ import java.sql.Timestamp; import java.util.Date; +/** + * @author gituser + * @modify xiuzhu + */ public class KuduOutputFormat extends DtRichOutputFormat { private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger(KuduOutputFormat.class); - public enum WriteMode {INSERT, UPDATE, UPSERT} + public enum WriteMode { + // insert + INSERT, + // update + UPDATE, + // update or insert + UPSERT + } private String kuduMasters; @@ -106,7 +123,7 @@ public void writeRecord(Tuple2 record) throws IOException { Row row = tupleTrans.getField(1); if (row.getArity() != fieldNames.length) { if(outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0) { - LOG.error("record insert failed ..", row.toString()); + LOG.error("record insert failed ..{}", row.toString()); LOG.error("cause by row.getArity() != fieldNames.length"); } outDirtyRecords.inc(); @@ -125,7 +142,7 @@ public void writeRecord(Tuple2 record) throws IOException { outRecords.inc(); } catch (KuduException e) { if(outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0){ - LOG.error("record insert failed ..", row.toString().substring(0, 100)); + LOG.error("record insert failed ..{}", row.toString().substring(0, 100)); LOG.error("", e); } outDirtyRecords.inc(); diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java index 269b7b258..8c1146ede 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java @@ -42,9 +42,12 @@ public MysqlSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL).setDialect(jdbcDialect) - .setUsername(userName).setPassword(password) - .setTableName(tableName).build(); + .setDBUrl(dbURL) + .setDialect(jdbcDialect) + .setUsername(userName) + .setPassword(password) + .setTableName(tableName) + .build(); return JDBCUpsertOutputFormat.builder() .setOptions(jdbcOptions) @@ -54,6 +57,7 @@ public JDBCUpsertOutputFormat getOutputFormat() { .setFieldTypes(sqlTypes) .setKeyFields(primaryKeys) .setAllReplace(allReplace) - .setUpdateMode(updateMode).build(); + .setUpdateMode(updateMode) + .build(); } } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index 176c46a23..f4a945a04 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -38,9 +38,13 @@ public OracleSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL).setDialect(jdbcDialect) - .setUsername(userName).setPassword(password) - .setTableName(tableName).setScheam(schema).build(); + .setDBUrl(dbURL) + .setDialect(jdbcDialect) + .setUsername(userName) + .setPassword(password) + .setTableName(tableName) + .setScheam(schema) + .build(); return JDBCUpsertOutputFormat.builder() .setOptions(jdbcOptions) @@ -50,6 +54,7 @@ public JDBCUpsertOutputFormat getOutputFormat() { .setFieldTypes(sqlTypes) .setKeyFields(primaryKeys) .setAllReplace(allReplace) - .setUpdateMode(updateMode).build(); + .setUpdateMode(updateMode) + .build(); } } diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java index e8d695b8e..cce8083d0 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java @@ -6,9 +6,6 @@ public class PolardbSink extends RdbSink { - - private static final String POLARDB_DRIVER = "com.mysql.cj.jdbc.Driver"; - public PolardbSink() { super(new PolardbDialect()); } diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java index 65379ed19..49cf43965 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java @@ -26,7 +26,7 @@ import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** - * + * @author maqi */ public class PostgresqlSink extends RdbSink implements IStreamSinkGener { public PostgresqlSink() { @@ -36,9 +36,13 @@ public PostgresqlSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL).setDialect(jdbcDialect) - .setUsername(userName).setPassword(password) - .setTableName(tableName).setScheam(schema).build(); + .setDBUrl(dbURL) + .setDialect(jdbcDialect) + .setUsername(userName) + .setPassword(password) + .setTableName(tableName) + .setScheam(schema) + .build(); return JDBCUpsertOutputFormat.builder() .setOptions(jdbcOptions) @@ -48,6 +52,7 @@ public JDBCUpsertOutputFormat getOutputFormat() { .setFieldTypes(sqlTypes) .setKeyFields(primaryKeys) .setAllReplace(allReplace) - .setUpdateMode(updateMode).build(); + .setUpdateMode(updateMode) + .build(); } } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index 9cb44b194..ff7b24979 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -149,11 +149,12 @@ public void open(int taskNumber, int numTasks) throws IOException { public synchronized void writeRecord(Tuple2 tuple2) throws IOException { checkConnectionOpen(); try { - // Receive data - outRecords.inc(); if (outRecords.getCount() % RECEIVEDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { LOG.info("Receive data : {}", tuple2); } + // Receive data + outRecords.inc(); + jdbcWriter.addRecord(tuple2); batchCount++; if (batchCount >= flushMaxSize) { diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/JDBCWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/JDBCWriter.java index efac8b3b8..f9ce36846 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/JDBCWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/JDBCWriter.java @@ -30,7 +30,7 @@ */ public interface JDBCWriter extends Serializable { - public static final int DIRTYDATA_PRINT_FREQUENTY = 1000; + int DIRTYDATA_PRINT_FREQUENTY = 1000; /** * Open the writer by JDBC Connection. It can create Statement from Connection. diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java index 5cb382f17..69aecbf23 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java @@ -38,9 +38,13 @@ public SqlserverSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL).setDialect(jdbcDialect) - .setUsername(userName).setPassword(password) - .setTableName(tableName).setScheam(schema).build(); + .setDBUrl(dbURL) + .setDialect(jdbcDialect) + .setUsername(userName) + .setPassword(password) + .setTableName(tableName) + .setScheam(schema) + .build(); return JDBCUpsertOutputFormat.builder() .setOptions(jdbcOptions) From e035d74f50efde24162697ef6d4c3e1d3e3a45aa Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 19 Feb 2020 17:25:56 +0800 Subject: [PATCH 022/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=87=8D=E5=90=AF?= =?UTF-8?q?=E7=AD=96=E7=95=A5=E7=9A=84=E6=8C=87=E5=AE=9A=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/constrant/ConfigConstrant.java | 2 ++ .../environment/StreamEnvConfigManager.java | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java index dcda4c093..0dc060d79 100644 --- a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java @@ -63,6 +63,8 @@ public class ConfigConstrant { public static final String CHECKPOINTS_DIRECTORY_KEY = "state.checkpoints.dir"; public static final String STATE_BACKEND_INCREMENTAL_KEY = "state.backend.incremental"; + public static final String RESTOREENABLE = "restore.enable"; + // restart plocy public static final int failureRate = 3; diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index ab74edabc..9da34bcc2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -101,11 +101,15 @@ public static void streamExecutionEnvironmentConfig(StreamExecutionEnvironment s } }); - streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( - ConfigConstrant.failureRate, - Time.of(ConfigConstrant.failureInterval, TimeUnit.MINUTES), - Time.of(ConfigConstrant.delayInterval, TimeUnit.SECONDS) - )); + if(isRestore(confProperties).get()){ + streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( + ConfigConstrant.failureRate, + Time.of(ConfigConstrant.failureInterval, TimeUnit.MINUTES), + Time.of(ConfigConstrant.delayInterval, TimeUnit.SECONDS) + )); + } else { + streamEnv.setRestartStrategy(RestartStrategies.noRestart()); + } // checkpoint config Optional checkpointingEnabled = isCheckpointingEnabled(confProperties); @@ -163,6 +167,11 @@ public static Optional getAutoWatermarkInterval(Properties properties) { return StringUtils.isNotBlank(autoWatermarkInterval) ? Optional.of(Long.valueOf(autoWatermarkInterval)) : Optional.empty(); } + public static Optional isRestore(Properties properties){ + String restoreEnable = properties.getProperty(ConfigConstrant.RESTOREENABLE, "true"); + return Optional.of(Boolean.valueOf(restoreEnable)); + } + /** * #ProcessingTime(默认), IngestionTime, EventTime * @param properties From 32281cce469ea590a543002813cee93c845dfa7b Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 20 Feb 2020 20:39:59 +0800 Subject: [PATCH 023/523] modify spelling --- .../main/java/com/dtstack/flink/sql/sink/db/DbSink.java | 9 ++++++--- .../com/dtstack/flink/sql/sink/oracle/OracleSink.java | 2 +- .../com/dtstack/flink/sql/sink/polardb/PolardbSink.java | 9 ++++++--- .../flink/sql/sink/postgresql/PostgresqlSink.java | 2 +- .../java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java | 8 ++++---- .../dtstack/flink/sql/sink/sqlserver/SqlserverSink.java | 2 +- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java index 119af515c..9942d4438 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java @@ -15,9 +15,12 @@ public DbSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL).setDialect(jdbcDialect) - .setUsername(userName).setPassword(password) - .setTableName(tableName).build(); + .setDBUrl(dbURL) + .setDialect(jdbcDialect) + .setUsername(userName) + .setPassword(password) + .setTableName(tableName) + .build(); return JDBCUpsertOutputFormat.builder() .setOptions(jdbcOptions) diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index f4a945a04..ee0239a1f 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -43,7 +43,7 @@ public JDBCUpsertOutputFormat getOutputFormat() { .setUsername(userName) .setPassword(password) .setTableName(tableName) - .setScheam(schema) + .setSchema(schema) .build(); return JDBCUpsertOutputFormat.builder() diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java index cce8083d0..44d558140 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java @@ -13,9 +13,12 @@ public PolardbSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL).setDialect(jdbcDialect) - .setUsername(userName).setPassword(password) - .setTableName(tableName).build(); + .setDBUrl(dbURL) + .setDialect(jdbcDialect) + .setUsername(userName) + .setPassword(password) + .setTableName(tableName) + .build(); return JDBCUpsertOutputFormat.builder() .setOptions(jdbcOptions) diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java index 49cf43965..cf45a6bd9 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java @@ -41,7 +41,7 @@ public JDBCUpsertOutputFormat getOutputFormat() { .setUsername(userName) .setPassword(password) .setTableName(tableName) - .setScheam(schema) + .setSchema(schema) .build(); return JDBCUpsertOutputFormat.builder() diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java index 77d9f19d0..e9a4f391f 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java @@ -105,7 +105,7 @@ public static class Builder { private String driverName; private String username; private String password; - private String scheam; + private String schema; private JDBCDialect dialect; /** @@ -144,8 +144,8 @@ public Builder setDriverName(String driverName) { /** * optional, schema info */ - public Builder setScheam(String scheam) { - this.scheam = scheam; + public Builder setSchema(String schema) { + this.schema = schema; return this; } @@ -173,7 +173,7 @@ public JDBCOptions build() { }); } - return new JDBCOptions(dbURL, tableName, driverName, username, password, scheam, dialect); + return new JDBCOptions(dbURL, tableName, driverName, username, password, schema, dialect); } } } diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java index 69aecbf23..dbbeb820e 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java @@ -43,7 +43,7 @@ public JDBCUpsertOutputFormat getOutputFormat() { .setUsername(userName) .setPassword(password) .setTableName(tableName) - .setScheam(schema) + .setSchema(schema) .build(); return JDBCUpsertOutputFormat.builder() From d52ed944a0d6366ef606b276bdff9a6d3eb32dd0 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 20 Feb 2020 20:56:26 +0800 Subject: [PATCH 024/523] rm retractJdbcOutputformat --- .../flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java deleted file mode 100644 index e69de29bb..000000000 From 63395f781be66883db3c8229afb942e2d0f82402 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Fri, 21 Feb 2020 08:03:52 +0800 Subject: [PATCH 025/523] =?UTF-8?q?elasticsearch6=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E9=83=A8=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/elasticsearch6Side | 94 +++++ .../elasticsearch6-all-side/pom.xml | 6 - .../Elasticsearch6AllReqRow.java | 268 +++++++++++--- .../Elasticsearch6AllSideInfo.java | 61 +--- .../Elasticsearch6AsyncReqRow.java | 341 ++++++++++++------ .../Elasticsearch6AsyncSideInfo.java | 67 +--- .../side/elasticsearch6/util/MathUtil.java | 255 ------------- .../side/elasticsearch6/util/SwitchUtil.java | 82 ----- .../table/Elasticsearch6SideParser.java | 27 +- .../table/Elasticsearch6SideTableInfo.java | 33 +- .../side/elasticsearch6/util/ClassUtil.java | 202 +++++++++++ .../side/elasticsearch6/util/MathUtil.java | 2 + .../side/elasticsearch6/util/SwitchUtil.java | 1 + 13 files changed, 796 insertions(+), 643 deletions(-) create mode 100644 docs/elasticsearch6Side delete mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java delete mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java create mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/ClassUtil.java rename elasticsearch6/elasticsearch6-side/{elasticsearch6-all-side => elasticsearch6-side-core}/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java (98%) rename elasticsearch6/elasticsearch6-side/{elasticsearch6-all-side => elasticsearch6-side-core}/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java (99%) diff --git a/docs/elasticsearch6Side b/docs/elasticsearch6Side new file mode 100644 index 000000000..936ac7295 --- /dev/null +++ b/docs/elasticsearch6Side @@ -0,0 +1,94 @@ + +## 1.格式： +``` + CREATE TABLE tableName( + colName cloType, + ... + PRIMARY KEY(keyInfo), + PERIOD FOR SYSTEM_TIME + )WITH( + type='elasticsearch6', + address ='ip:port[,ip:port]', + cluster='clusterName', + estype ='esType', + index ='index', + authMesh='true', + userName='dbUserName', + password='dbPwd', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1', + partitionedJoin='false' + ); +``` + +# 2.支持版本 + mysql-5.6.35 + +## 3.表结构定义 + + |参数名称|含义| + |----|---| + | tableName | mysql表名称| + | colName | 列名称| + | colType | 列类型 [colType支持的类型](colType.md)| + | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| + | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| + +## 4.参数 + + |参数名称|含义|是否必填|默认值| + |----|---|---|----| + type|表明 输出表类型[mysq|hbase|elasticsearch6]|是|| + |address | 连接ES Transport地址(tcp地址)|是|| + |cluster | ES 集群名称 |是|| + |index | 选择的ES上的index名称|否|| + |esType | 选择ES上的type名称|否|| + |authMesh | 是否进行用户名密码认证 | 否 | false| + |userName | 用户名 | 否，authMesh='true'时为必填 || + |password | 密码 | 否，authMesh='true'时为必填 || + | cache | 维表缓存策略(NONE/LRU)|否|NONE| + | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| + |parallelism | 并行度设置|否|1| + + ---------- + > 缓存策略 + * NONE: 不做内存缓存 + * LRU: + * cacheSize: 缓存的条目数量 + * cacheTTLMs:缓存的过期时间(ms) + * cacheMode: (unordered|ordered)异步加载是有序还是无序,默认有序。 + * asyncCapacity:异步请求容量，默认1000 + * asyncTimeout：异步请求超时时间，默认10000毫秒 + +## 5.样例 +``` +create table sideTable( + channel varchar, + xccount int, + PRIMARY KEY(channel), + PERIOD FOR SYSTEM_TIME + )WITH( + type ='elasticsearch', + address ='172.16.10.47:9500', + cluster='es_47_menghan', + estype ='type1', + index ='xc_es_test', + authMesh='true', + userName='dtstack', + password='abc123', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + cacheMode='unordered', + asyncCapacity='1000', + asyncTimeout='10000' + parallelism ='1', + partitionedJoin='false' + ); + + +``` + + diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/pom.xml b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/pom.xml index 3e4bacebc..81639fb1e 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/pom.xml +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/pom.xml @@ -32,12 +32,6 @@ ${sql.side.elasticsearch6.core.version} - - junit - junit - 3.8.1 - test - junit junit diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index 8a4fce94e..8fdd02e87 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -1,11 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.dtstack.flink.sql.side.elasticsearch6; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import com.dtstack.flink.sql.side.AllReqRow; -import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.util.SwitchUtil; import com.google.common.collect.Lists; @@ -18,48 +37,58 @@ import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.*; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.Serializable; import java.sql.SQLException; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; /** * @author yinxi * @date 2020/1/13 - 1:00 */ -public class Elasticsearch6AllReqRow extends AllReqRow { +public class Elasticsearch6AllReqRow extends AllReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AllReqRow.class); + private static final Integer SCROLL_TIME = 1; + private static final int CONN_RETRY_NUM = 3; + private static final String KEY_WORD_TYPE = ".keyword"; private AtomicReference>>> cacheRef = new AtomicReference<>(); + private String scrollId; + private Scroll scroll; + private transient RestHighLevelClient rhlClient; - public Elasticsearch6AllReqRow(SideInfo sideInfo) { - super(sideInfo); + public Elasticsearch6AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + super(new Elasticsearch6AllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } + @Override - public void flatMap(Row value, Collector out) throws Exception { + public void flatMap(CRow value, Collector out) throws Exception { List inputParams = Lists.newArrayList(); for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = value.getField(conValIndex); + Object equalObj = value.row().getField(conValIndex); if (equalObj == null) { if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value, null); - out.collect(row); + Row row = fillData(value.row(), null); + out.collect(new CRow(row, value.change())); } return; @@ -70,16 +99,20 @@ public void flatMap(Row value, Collector out) throws Exception { String key = buildKey(inputParams); List> cacheList = cacheRef.get().get(key); - if(CollectionUtils.isEmpty(cacheList)){ - if(sideInfo.getJoinType() == JoinType.LEFT){ - Row row = fillData(value, null); - out.collect(row); - }else{ + if (CollectionUtils.isEmpty(cacheList)) { + if (sideInfo.getJoinType() == JoinType.LEFT) { + Row row = fillData(value.row(), null); + out.collect(new CRow(row, value.change())); + } else { return; } return; } + + for (Map one : cacheList) { + out.collect(new CRow(fillData(value.row(), one), value.change())); + } } @Override @@ -154,22 +187,42 @@ protected void reloadCache() { private void loadData(Map>> tmpCache) throws IOException { Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); - RestHighLevelClient rhlClient = null; - try{ - rhlClient = getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword(), tableInfo.getTimeout()); + try { + for (int i = 0; i < CONN_RETRY_NUM; i++) { + try { + rhlClient = getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword()); + break; + } catch (Exception e) { + if (i == CONN_RETRY_NUM - 1) { + throw new RuntimeException("", e); + } + + try { + String connInfo = "url: " + tableInfo.getAddress() + "; userName: " + tableInfo.getUserName() + ", pwd:" + tableInfo.getPassword(); + LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); + Thread.sleep(5 * 1000); + } catch (InterruptedException e1) { + LOG.error("", e1); + } + } + + } + // load data from tableA - SearchSourceBuilder searchSourceBuilder = tableInfo.getSearchSourceBuilder(); + SearchSourceBuilder searchSourceBuilder = getSelectFromStatement(sideInfo.getSideTableInfo().getPredicateInfoes()); searchSourceBuilder.size(getFetchSize()); SearchRequest searchRequest = new SearchRequest(); + scroll = new Scroll(TimeValue.timeValueMinutes(SCROLL_TIME)); + searchRequest.scroll(scroll); // determine existence of index String index = tableInfo.getIndex().trim(); - if(!StringUtils.isEmpty(index)){ + if (!StringUtils.isEmpty(index)) { // strip leading and trailing spaces from a string String[] indexes = StringUtils.split(index, ","); - for(int i=0; i < indexes.length; i++ ){ + for (int i = 0; i < indexes.length; i++) { indexes[i] = indexes[i].trim(); } @@ -179,10 +232,10 @@ private void loadData(Map>> tmpCache) throws IO // determine existence of type String type = tableInfo.getEsType().trim(); - if(!StringUtils.isEmpty(type)){ + if (!StringUtils.isEmpty(type)) { // strip leading and trailing spaces from a string String[] types = StringUtils.split(type, ","); - for(int i=0; i < types.length; i++ ){ + for (int i = 0; i < types.length; i++) { types[i] = types[i].trim(); } @@ -193,38 +246,74 @@ private void loadData(Map>> tmpCache) throws IO searchRequest.source(searchSourceBuilder); // get query reults - SearchResponse searchResponse = rhlClient.search(searchRequest); - SearchHit[] searchHits = searchResponse.getHits().getHits(); + searchScroll(searchRequest, tmpCache); - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); - String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); - Map oneRow = Maps.newHashMap(); - for (SearchHit searchHit : searchHits) { - for(String fieldName : sideFieldNames){ - Object object = searchHit.getSourceAsMap().get(fieldName.trim()); - int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); - object = SwitchUtil.getTarget(object, sideFieldTypes[fieldIndex]); - oneRow.put(fieldName.trim(), object); + } catch (Exception e) { + LOG.error("", e); + } finally { + if (!StringUtils.isEmpty(scrollId)) { + clearScroll(); + } + + if (rhlClient != null) { + rhlClient.close(); + } + } + + } + + public void searchScroll(SearchRequest searchRequest, Map>> tmpCache) throws IOException { + SearchResponse searchResponse = rhlClient.search(searchRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + loadToCache(searchHits, tmpCache); + + if (!StringUtils.isEmpty(scrollId)) { + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + while (true) { + SearchResponse scrollResponse = rhlClient.scroll(scrollRequest, RequestOptions.DEFAULT); + if (scrollResponse.getHits().getHits() == null || scrollResponse.getHits().getHits().length < 1) { + break; } + searchHits = searchResponse.getHits().getHits(); + loadToCache(searchHits, tmpCache); + + scrollId = scrollResponse.getScrollId(); + scrollRequest.scrollId(scrollId); } + } + } + private void loadToCache(SearchHit[] searchHits, Map>> tmpCache) { + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); + String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); + for (SearchHit searchHit : searchHits) { + Map oneRow = Maps.newHashMap(); + for (String fieldName : sideFieldNames) { + Object object = searchHit.getSourceAsMap().get(fieldName.trim()); + int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); + object = SwitchUtil.getTarget(object, sideFieldTypes[fieldIndex]); + oneRow.put(fieldName.trim(), object); + } String cacheKey = buildKey(oneRow, sideInfo.getEqualFieldList()); List> list = tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()); list.add(oneRow); - } catch (Exception e) { - LOG.error("", e); - } finally { - if (rhlClient != null) { - rhlClient.close(); - } } + } + private void clearScroll() throws IOException { + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + ClearScrollResponse clearScrollResponse = rhlClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + boolean succeeded = clearScrollResponse.isSucceeded(); + LOG.info("Clear scroll response:{}", succeeded); } - public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, String userName, String password, Integer timeout) { + public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, String userName, String password) { List httpHostList = new ArrayList<>(); String[] address = StringUtils.split(esAddress, ","); for (String addr : address) { @@ -239,14 +328,10 @@ public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, Strin RestClientBuilder restClientBuilder = RestClient.builder(httpHostList.toArray(new HttpHost[httpHostList.size()])); - if (timeout != null) { - restClientBuilder.setMaxRetryTimeoutMillis(timeout * 1000); - } - if (isAuthMesh) { // 进行用户和密码认证 final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password)); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName.trim(), password.trim())); restClientBuilder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); } @@ -257,11 +342,11 @@ public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, Strin LOG.info("Pinging Elasticsearch cluster via hosts {} ...", httpHostList); } - try{ + try { if (!rhlClient.ping()) { throw new RuntimeException("There are no reachable Elasticsearch nodes!"); } - } catch (IOException e){ + } catch (IOException e) { LOG.warn("", e); } @@ -277,4 +362,81 @@ public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, Strin public int getFetchSize() { return 1000; } + + private SearchSourceBuilder getSelectFromStatement(List predicateInfoes) { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + + if (predicateInfoes.size() > 0) { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + for (PredicateInfo info : predicateInfoes) { + boolQueryBuilder = buildFilterCondition(boolQueryBuilder, info); + } + + searchSourceBuilder.query(boolQueryBuilder); + } + + return searchSourceBuilder; + } + + public BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info) { + switch (info.getOperatorKind()) { + case "IN": + return boolQueryBuilder.must(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName()), removeSpaces(info.getCondition()))); + case "NOT_IN": + return boolQueryBuilder.mustNot(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName()), removeSpaces(info.getCondition()))); + case "GREATER_THAN_OR_EQUAL": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gte(info.getCondition())); + case "GREATER_THAN": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gt(info.getCondition())); + case "LESS_THAN_OR_EQUAL": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lte(info.getCondition())); + case "LESS_THAN": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lt(info.getCondition())); + case "BETWEEN": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gte(StringUtils.split(info.getCondition().toUpperCase(), "AND")[0].trim()) + .lte(StringUtils.split(info.getCondition().toUpperCase(), "AND")[1].trim())); + case "IS_NULL": + return boolQueryBuilder.mustNot(QueryBuilders.existsQuery(info.getFieldName())); + case "IS_NOT_NULL": + return boolQueryBuilder.must(QueryBuilders.existsQuery(info.getFieldName())); + case "EQUALS": + return boolQueryBuilder.must(QueryBuilders.termQuery(textConvertToKeyword(info.getFieldName()), info.getCondition())); + case "NOT_EQUALS": + return boolQueryBuilder.mustNot(QueryBuilders.termQuery(textConvertToKeyword(info.getFieldName()), info.getCondition())); + default: + try { + throw new Exception("elasticsearch6 does not support this operation: " + info.getOperatorName()); + } catch (Exception e) { + + e.printStackTrace(); + LOG.error(e.getMessage()); + } + return boolQueryBuilder; + } + + } + + public String[] removeSpaces(String str) { + String[] split = StringUtils.split(str, ","); + String[] result = new String[split.length]; + Arrays.asList(split).stream().map(f -> f.trim()).collect(Collectors.toList()).toArray(result); + return result; + } + + public String textConvertToKeyword(String fieldName) { + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); + String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); + int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); + String fieldType = sideFieldTypes[fieldIndex]; + switch (fieldType.toLowerCase()) { + case "varchar": + case "char": + case "text": + return fieldName + KEY_WORD_TYPE; + default: + return fieldName; + } + } + + } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java index acf39f989..bf21b3ca8 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java @@ -20,18 +20,14 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.dtstack.flink.sql.side.*; -import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; @@ -41,7 +37,6 @@ */ public class Elasticsearch6AllSideInfo extends SideInfo { - private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AllSideInfo.class); public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); @@ -50,56 +45,8 @@ public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, Lis @Override public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { - Elasticsearch6SideTableInfo elasticsearch6SideTableInfo = (Elasticsearch6SideTableInfo) sideTableInfo; - - elasticsearch6SideTableInfo.setSearchSourceBuilder(getSelectFromStatement(sideTableInfo.getPredicateInfoes())); - } - private SearchSourceBuilder getSelectFromStatement(List predicateInfoes) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - if (predicateInfoes.size() != 0) { - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - for (PredicateInfo info : sideTableInfo.getPredicateInfoes()) { - boolQueryBuilder = buildFilterCondition(boolQueryBuilder, info); - } - - searchSourceBuilder.query(boolQueryBuilder); - } - - return searchSourceBuilder; - } - - public BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info) { - switch (info.getOperatorKind()) { - case "IN": - return boolQueryBuilder.must(QueryBuilders.termsQuery(info.getFieldName(), StringUtils.split(info.getCondition().trim(), ","))); - case "NOT_IN": - return boolQueryBuilder.mustNot(QueryBuilders.termsQuery(info.getFieldName(), StringUtils.split(info.getCondition().trim(), ","))); - case "GREATER_THAN_OR_EQUAL": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gte(info.getCondition())); - case "GREATER_THAN": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gt(info.getCondition())); - case "LESS_THAN_OR_EQUAL": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lte(info.getCondition())); - case "LESS_THAN": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lt(info.getCondition())); - case "EQUALS": - return boolQueryBuilder.must(QueryBuilders.termQuery(info.getFieldName(), info.getCondition())); - case "NOT_EQUALS": - return boolQueryBuilder.mustNot(QueryBuilders.termQuery(info.getFieldName(), info.getCondition())); - default: - try { - throw new Exception("elasticsearch6 does not support this operation: " + info.getOperatorName()); - } catch (Exception e) { - - e.printStackTrace(); - LOG.error(e.getMessage()); - } - return boolQueryBuilder; - } - - } @Override public void parseSelectFields(JoinInfo joinInfo) { diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index 11b0d383e..16458d56e 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -18,18 +18,20 @@ package com.dtstack.flink.sql.side.elasticsearch6; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; -import com.dtstack.flink.sql.side.CacheMissVal; -import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.util.SwitchUtil; +import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; +import org.apache.calcite.sql.SqlNode; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; @@ -37,60 +39,57 @@ import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.*; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.Serializable; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * @author yinxi * @date 2020/2/13 - 13:10 */ -public class Elasticsearch6AsyncReqRow extends AsyncReqRow { +public class Elasticsearch6AsyncReqRow extends AsyncReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AsyncReqRow.class); - - private static final int CONN_RETRY_NUM = 3; - - public final static int DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE = 1; - - public final static int DEFAULT_VERTX_WORKER_POOL_SIZE = Runtime.getRuntime().availableProcessors() * 2; - - public final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; - - public final static int DEFAULT_IDLE_CONNECTION_TEST_PEROID = 60; - - public final static boolean DEFAULT_TEST_CONNECTION_ON_CHECKIN = true; - - public final static String PREFERRED_TEST_QUERY_SQL = "select 1 from dual"; - + private static final Integer SCROLL_TIME = 1 ; + private static final String KEY_WORD_TYPE = ".keyword"; private transient RestHighLevelClient rhlClient; + private transient Scroll scroll; + private List sqlJoinCompareOperate= Lists.newArrayList(); - public Elasticsearch6AsyncReqRow(SideInfo sideInfo) { - super(sideInfo); + public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + super(new Elasticsearch6AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + SqlNode conditionNode = joinInfo.getCondition(); + ParseUtils.parseJoinCompareOperate(conditionNode, sqlJoinCompareOperate); } + @Override - public void asyncInvoke(Row input, ResultFuture resultFuture) throws Exception { - Row inputRow = Row.copy(input); + public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + CRow copyCrow = new CRow(input.row(), input.change()); List inputParams = Lists.newArrayList(); -// JsonArray inputParams = new JsonArray(); for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = inputRow.getField(conValIndex); + Object equalObj = copyCrow.row().getField(conValIndex); if (equalObj == null) { - dealMissKey(inputRow, resultFuture); + dealMissKey(copyCrow, resultFuture); return; } inputParams.add(equalObj); @@ -101,14 +100,14 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except CacheObj val = getFromCache(key); if (val != null) { if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(inputRow, resultFuture); + dealMissKey(copyCrow, resultFuture); return; } else if (ECacheContentType.MultiLine == val.getType()) { try { - List rowList = getRows(inputRow, null, (List) val.getContent()); + List rowList = getRows(copyCrow, null, (List) val.getContent()); resultFuture.complete(rowList); } catch (Exception e) { - dealFillDataError(resultFuture, e, inputRow); + dealFillDataError(resultFuture, e, copyCrow); } } else { resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); @@ -118,109 +117,141 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except } Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); - RestHighLevelClient rhlClient = null; - try{ - rhlClient = getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword(), tableInfo.getTimeout()); - - // load data from tableA - SearchSourceBuilder searchSourceBuilder = tableInfo.getSearchSourceBuilder(); -// searchSourceBuilder.size(getFetchSize()); - SearchRequest searchRequest = new SearchRequest(); - - // determine existence of index - String index = tableInfo.getIndex().trim(); - if(!StringUtils.isEmpty(index)){ - // strip leading and trailing spaces from a string - String[] indexes = StringUtils.split(index, ","); - for(int i=0; i < indexes.length; i++ ){ - indexes[i] = indexes[i].trim(); - } - searchRequest.indices(indexes); + rhlClient = getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword()); + + // load data from tableA + SearchSourceBuilder searchSourceBuilder = getSelectFromStatement(inputParams, sideInfo.getSideTableInfo().getPredicateInfoes()); + + searchSourceBuilder.size(getFetchSize()); + SearchRequest searchRequest = new SearchRequest(); + scroll = new Scroll(TimeValue.timeValueMinutes(SCROLL_TIME)); + searchRequest.scroll(scroll); + // determine existence of index + String index = tableInfo.getIndex().trim(); + if (!StringUtils.isEmpty(index)) { + // strip leading and trailing spaces from a string + String[] indexes = StringUtils.split(index, ","); + for (int i = 0; i < indexes.length; i++) { + indexes[i] = indexes[i].trim(); } - // determine existence of type - String type = tableInfo.getEsType().trim(); - if(!StringUtils.isEmpty(type)){ - // strip leading and trailing spaces from a string - String[] types = StringUtils.split(type, ","); - for(int i=0; i < types.length; i++ ){ - types[i] = types[i].trim(); - } + searchRequest.indices(indexes); + + } - searchRequest.types(types); + // determine existence of type + String type = tableInfo.getEsType().trim(); + if (!StringUtils.isEmpty(type)) { + // strip leading and trailing spaces from a string + String[] types = StringUtils.split(type, ","); + for (int i = 0; i < types.length; i++) { + types[i] = types[i].trim(); } - // add query condition - searchRequest.source(searchSourceBuilder); - - // 异步查询数据 - - rhlClient.searchAsync(searchRequest, RequestOptions.DEFAULT ,new ActionListener() { - - //成功响应时 - @Override - public void onResponse(SearchResponse searchResponse) { - SearchHit[] searchHits = searchResponse.getHits().getHits(); - List cacheContent = Lists.newArrayList(); - List results = Lists.newArrayList(); - if(searchHits.length > 0){ - try{ - for(SearchHit searchHit : searchHits){ - Map object = searchHit.getSourceAsMap(); - results.add(object); - } - - List rowList = getRows(inputRow, cacheContent, results); - dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); - resultFuture.complete(rowList); - }catch (Exception e){ - dealFillDataError(resultFuture, e, inputRow); + searchRequest.types(types); + } + + // add query condition + searchRequest.source(searchSourceBuilder); + + // 异步查询数据 + + rhlClient.searchAsync(searchRequest, RequestOptions.DEFAULT, new ActionListener() { + + //成功响应时 + @Override + public void onResponse(SearchResponse searchResponse) { + String scrollId = searchResponse.getScrollId(); + + List cacheContent = Lists.newArrayList(); + List results = Lists.newArrayList(); + loadDataToCache(searchResponse, key, results, cacheContent, copyCrow, resultFuture); + + if (!StringUtils.isEmpty(scrollId)) { + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + while (true) { + SearchResponse scrollResponse = null; + try { + scrollResponse = rhlClient.scroll(scrollRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + resultFuture.completeExceptionally(new RuntimeException("Request failed!")); } - }else{ - dealMissKey(inputRow, resultFuture); - dealCacheData(key, CacheMissVal.getMissKeyObj()); - } + if (scrollResponse.getHits().getHits() == null || scrollResponse.getHits().getHits().length < 1) { + break; + } + loadDataToCache(searchResponse, key, results, cacheContent, copyCrow, resultFuture); - } + scrollId = scrollResponse.getScrollId(); + scrollRequest.scrollId(scrollId); - // 响应失败处理 - @Override - public void onFailure(Exception e) { - resultFuture.completeExceptionally(new RuntimeException("Response timeout!")); + } + } + try { + clearScroll(scrollId); + } catch (IOException e) { + LOG.warn("Clear scroll response:{}: failure", e); } - }); + } + + private void loadDataToCache(SearchResponse searchResponse, String key, List results, List cacheContent, CRow copyCrow, ResultFuture resultFuture) { + SearchHit[] searchHits = searchResponse.getHits().getHits(); + if (searchHits.length > 0) { + try { + for (SearchHit searchHit : searchHits) { + Map object = searchHit.getSourceAsMap(); + results.add(object); + } + List rowList = getRows(copyCrow, cacheContent, results); + dealCacheData(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); + resultFuture.complete(rowList); + } catch (Exception e) { + dealFillDataError(resultFuture, e, copyCrow); + } + } else { + dealMissKey(copyCrow, resultFuture); + dealCacheData(key, CacheMissVal.getMissKeyObj()); + } + } + private void clearScroll(String scrollId) throws IOException { + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + ClearScrollResponse clearScrollResponse = rhlClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + boolean succeeded = clearScrollResponse.isSucceeded(); + LOG.info("Clear scroll response:{}", succeeded); + } - } catch (Exception e) { - LOG.error("", e); - } finally { - if (rhlClient != null) { - rhlClient.close(); + // 响应失败处理 + @Override + public void onFailure(Exception e) { + LOG.error("" + e); + resultFuture.completeExceptionally(new RuntimeException("Response failed!")); } - } + }); } - protected List getRows(Row inputRow, List cacheContent, List results) { - List rowList = Lists.newArrayList(); + protected List getRows(CRow inputRow, List cacheContent, List results) { + List rowList = Lists.newArrayList(); for (Object line : results) { - Row row = fillData(inputRow, line); + Row row = fillData(inputRow.row(), line); if (null != cacheContent && openCache()) { cacheContent.add(line); } - rowList.add(row); + rowList.add(new CRow(row, inputRow.change())); } return rowList; } @Override public Row fillData(Row input, Object line) { -// JsonArray jsonArray = (JsonArray) line; + Map cacheInfo = (Map) line; Row row = new Row(sideInfo.getOutFieldInfoList().size()); String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); @@ -265,11 +296,8 @@ public String buildCacheKey(List equalValList) { return sb.toString(); } -// public void setRdbSQLClient(SQLClient rdbSQLClient) { -// this.rdbSQLClient = rdbSQLClient; -// } - public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, String userName, String password, Integer timeout) { + public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, String userName, String password) { List httpHostList = new ArrayList<>(); String[] address = StringUtils.split(esAddress, ","); @@ -285,14 +313,10 @@ public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, Strin RestClientBuilder restClientBuilder = RestClient.builder(httpHostList.toArray(new HttpHost[httpHostList.size()])); - if (timeout != null) { - restClientBuilder.setMaxRetryTimeoutMillis(timeout * 1000); - } - if (isAuthMesh) { // 进行用户和密码认证 final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password)); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName.trim(), password.trim())); restClientBuilder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); } @@ -320,5 +344,96 @@ public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, Strin } + public int getFetchSize() { + return 1000; + } + + private SearchSourceBuilder getSelectFromStatement(List inputParams, List predicateInfoes) { + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + + + for (int i = 0; i < sqlJoinCompareOperate.size(); i++) { + String fieldName = sideInfo.getEqualFieldList().get(i); + String operatorKind = sqlJoinCompareOperate.get(sideInfo.getEqualFieldList().indexOf(fieldName)); + String condition = String.valueOf(inputParams.get(i)); + boolQueryBuilder = buildFilterCondition(boolQueryBuilder, new PredicateInfo(null, operatorKind, null, fieldName, condition)); + } + + if (predicateInfoes.size() > 0) { + for (PredicateInfo info : predicateInfoes) { + boolQueryBuilder = buildFilterCondition(boolQueryBuilder, info); + } + } + searchSourceBuilder.query(boolQueryBuilder); + return searchSourceBuilder; + } + + public BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info) { + switch (info.getOperatorKind()) { + case "IN": + return boolQueryBuilder.must(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName()), removeSpaces(info.getCondition()))); + case "NOT_IN": + return boolQueryBuilder.mustNot(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName()), removeSpaces(info.getCondition()))); + case ">=": + case "GREATER_THAN_OR_EQUAL": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gte(info.getCondition())); + case ">": + case "GREATER_THAN": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gt(info.getCondition())); + case "<=": + case "LESS_THAN_OR_EQUAL": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lte(info.getCondition())); + case "<": + case "LESS_THAN": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lt(info.getCondition())); + case "BETWEEN": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gte(StringUtils.split(info.getCondition().toUpperCase(), "AND")[0].trim()) + .lte(StringUtils.split(info.getCondition().toUpperCase(), "AND")[1].trim())); + case "IS_NULL": + return boolQueryBuilder.mustNot(QueryBuilders.existsQuery(info.getFieldName())); + case "IS_NOT_NULL": + return boolQueryBuilder.must(QueryBuilders.existsQuery(info.getFieldName())); + case "=": + case "EQUALS": + return boolQueryBuilder.must(QueryBuilders.termQuery(textConvertToKeyword(info.getFieldName()), info.getCondition())); + case "<>": + case "NOT_EQUALS": + return boolQueryBuilder.mustNot(QueryBuilders.termQuery(textConvertToKeyword(info.getFieldName()), info.getCondition())); + default: + try { + throw new Exception("elasticsearch6 does not support this operation: " + info.getOperatorName()); + } catch (Exception e) { + + e.printStackTrace(); + LOG.error(e.getMessage()); + } + return boolQueryBuilder; + } + + } + + public String[] removeSpaces(String str) { + String[] split = StringUtils.split(str, ","); + String[] result = new String[split.length]; + Arrays.asList(split).stream().map(f -> f.trim()).collect(Collectors.toList()).toArray(result); + return result; + } + + public String textConvertToKeyword(String fieldName) { + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); + String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); + int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); + String fieldType = sideFieldTypes[fieldIndex]; + switch (fieldType.toLowerCase()) { + case "varchar": + case "char": + case "text": + return fieldName + KEY_WORD_TYPE; + default: + return fieldName; + } + } } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java index dd5ba1e46..7b3a2f9e7 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java @@ -20,20 +20,17 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.dtstack.flink.sql.side.*; -import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import java.util.Arrays; import java.util.List; /** @@ -49,74 +46,18 @@ public Elasticsearch6AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, L @Override public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { - Elasticsearch6SideTableInfo elasticsearch6SideTableInfo = (Elasticsearch6SideTableInfo) sideTableInfo; String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); - List sqlNodeList = Lists.newArrayList(); - List sqlJoinCompareOperate= Lists.newArrayList(); - ParseUtils.parseAnd(conditionNode, sqlNodeList); - ParseUtils.parseJoinCompareOperate(conditionNode, sqlJoinCompareOperate); for (SqlNode sqlNode : sqlNodeList) { dealOneEqualCon(sqlNode, sideTableName); } - // set query condition - elasticsearch6SideTableInfo.setSearchSourceBuilder(getSelectFromStatement(sideTableInfo.getPredicateInfoes())); - } - private SearchSourceBuilder getSelectFromStatement(List predicateInfoes) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - - if (predicateInfoes.size() != 0) { - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - for (PredicateInfo info : sideTableInfo.getPredicateInfoes()) { - boolQueryBuilder = buildFilterCondition(boolQueryBuilder, info); - } - - searchSourceBuilder.query(boolQueryBuilder); - } - - return searchSourceBuilder; - } - - public BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info){ - switch (info.getOperatorKind()) { - case "GREATER_THAN_OR_EQUAL": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gte(info.getCondition())); - case "GREATER_THAN": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gt(info.getCondition())); - case "LESS_THAN_OR_EQUAL": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lte(info.getCondition())); - case "LESS_THAN": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lt(info.getCondition())); - case "EQUALS": - return boolQueryBuilder.must(QueryBuilders.termQuery(info.getFieldName(), info.getCondition())); - case "NOT_EQUALS": - return boolQueryBuilder.mustNot(QueryBuilders.termQuery(info.getFieldName(), info.getCondition())); - case "LIKE": - return boolQueryBuilder.must(QueryBuilders.fuzzyQuery(info.getFieldName(), info.getCondition())); - case "IN": - return boolQueryBuilder.must(QueryBuilders.termsQuery(info.getFieldName(), Arrays.asList(StringUtils.split(info.getCondition().trim(), ",")))); - case "NOT_IN": - return boolQueryBuilder.mustNot(QueryBuilders.termsQuery(info.getFieldName(), Arrays.asList(StringUtils.split(info.getCondition().trim(), ",")))); - default: - try { - throw new Exception("Predicate does not match!"); - } catch (Exception e) { - e.printStackTrace(); - } - - return boolQueryBuilder; - } - - } - - @Override public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { if (!SqlKind.COMPARISON.contains(sqlNode.getKind())) { diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java deleted file mode 100644 index 7765ddf17..000000000 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.side.elasticsearch6.util; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.sql.Date; -import java.sql.Timestamp; -import java.text.ParseException; -import java.text.SimpleDateFormat; - -/** - * Date: 2017/4/21 - * Company: www.dtstack.com - * - * @author xuchao - */ - -public class MathUtil { - - public static Long getLongVal(Object obj) { - if (obj == null) { - return null; - } - - if (obj instanceof String) { - return Long.valueOf((String) obj); - } else if (obj instanceof Long) { - return (Long) obj; - } else if (obj instanceof Integer) { - return Long.valueOf(obj.toString()); - } else if (obj instanceof BigDecimal) { - return ((BigDecimal) obj).longValue(); - } else if (obj instanceof BigInteger) { - return ((BigInteger) obj).longValue(); - } - throw new RuntimeException("not support type of " + obj.getClass() + " convert to Long."); - } - - public static Long getLongVal(Object obj, long defaultVal) { - if (obj == null) { - return defaultVal; - } - - return getLongVal(obj); - } - - public static Integer getIntegerVal(Object obj) { - if (obj == null) { - return null; - } - - if (obj instanceof String) { - return Integer.valueOf((String) obj); - } else if (obj instanceof Integer) { - return (Integer) obj; - } else if (obj instanceof Long) { - return ((Long) obj).intValue(); - } else if (obj instanceof Double) { - return ((Double) obj).intValue(); - } else if (obj instanceof BigDecimal) { - return ((BigDecimal) obj).intValue(); - } else if (obj instanceof BigInteger) { - return ((BigInteger) obj).intValue(); - } - - throw new RuntimeException("not support type of " + obj.getClass() + " convert to Integer."); - } - - public static Integer getIntegerVal(Object obj, int defaultVal) { - if (obj == null) { - return defaultVal; - } - - return getIntegerVal(obj); - } - - public static Float getFloatVal(Object obj) { - if (obj == null) { - return null; - } - - if (obj instanceof String) { - return Float.valueOf((String) obj); - } else if (obj instanceof Float) { - return (Float) obj; - } else if (obj instanceof BigDecimal) { - return ((BigDecimal) obj).floatValue(); - } - - throw new RuntimeException("not support type of " + obj.getClass() + " convert to Float."); - } - - public static Float getFloatVal(Object obj, float defaultVal) { - if (obj == null) { - return defaultVal; - } - - return getFloatVal(obj); - } - - public static Double getDoubleVal(Object obj) { - if (obj == null) { - return null; - } - - if (obj instanceof String) { - return Double.valueOf((String) obj); - } else if (obj instanceof Float) { - return (Double) obj; - } else if (obj instanceof BigDecimal) { - return ((BigDecimal) obj).doubleValue(); - } - - throw new RuntimeException("not support type of " + obj.getClass() + " convert to Double."); - } - - public static Double getDoubleVal(Object obj, double defaultVal) { - if (obj == null) { - return defaultVal; - } - - return getDoubleVal(obj); - } - - - public static Boolean getBoolean(Object obj) { - if (obj == null) { - return null; - } - - if (obj instanceof String) { - return Boolean.valueOf((String) obj); - } else if (obj instanceof Boolean) { - return (Boolean) obj; - } - - throw new RuntimeException("not support type of " + obj.getClass() + " convert to Boolean."); - } - - public static Boolean getBoolean(Object obj, boolean defaultVal) { - if (obj == null) { - return defaultVal; - } - - return getBoolean(obj); - } - - public static String getString(Object obj) { - if (obj == null) { - return null; - } - - if (obj instanceof String) { - return (String) obj; - } - - return obj.toString(); - } - - public static Byte getByte(Object obj) { - if (obj == null) { - return null; - } - - if (obj instanceof String) { - return Byte.valueOf((String) obj); - } else if (obj instanceof Byte) { - return (Byte) obj; - } - - throw new RuntimeException("not support type of " + obj.getClass() + " convert to Byte."); - } - - public static Short getShort(Object obj) { - if (obj == null) { - return null; - } - - if (obj instanceof String) { - return Short.valueOf((String) obj); - } else if (obj instanceof Short) { - return (Short) obj; - } - - throw new RuntimeException("not support type of " + obj.getClass() + " convert to Short."); - } - - public static BigDecimal getBigDecimal(Object obj) { - if (obj == null) { - return null; - } - if (obj instanceof String) { - return new BigDecimal((String) obj); - } else if (obj instanceof BigDecimal) { - return (BigDecimal) obj; - } else if (obj instanceof BigInteger) { - return new BigDecimal((BigInteger) obj); - } else if (obj instanceof Number) { - return new BigDecimal(((Number) obj).doubleValue()); - } - throw new RuntimeException("not support type of " + obj.getClass() + " convert to BigDecimal."); - } - - public static Date getDate(Object obj) { - if (obj == null) { - return null; - } - if (obj instanceof String) { - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - try { - return new Date(format.parse((String) obj).getTime()); - } catch (ParseException e) { - throw new RuntimeException("String convert to Date fail."); - } - } else if (obj instanceof Timestamp) { - return new Date(((Timestamp) obj).getTime()); - } else if (obj instanceof Date) { - return (Date) obj; - } - throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); - } - - public static Timestamp getTimestamp(Object obj) { - if (obj == null) { - return null; - } - if (obj instanceof Timestamp) { - return (Timestamp) obj; - } else if (obj instanceof Date) { - return new Timestamp(((Date) obj).getTime()); - } else if (obj instanceof String) { - return new Timestamp(getDate(obj).getTime()); - } - throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); - } - -} \ No newline at end of file diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java deleted file mode 100644 index 5f2e08651..000000000 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.dtstack.flink.sql.side.elasticsearch6.util; - -/** - * Reason: - * Date: 2018/12/3 - * Company: www.dtstack.com - * - * @author maqi - */ -public class SwitchUtil { - public static Object getTarget(Object obj, String targetType) { - targetType = targetType.toLowerCase(); - switch (targetType) { - - case "smallint": - case "smallintunsigned": - case "tinyint": - case "tinyintunsigned": - case "mediumint": - case "mediumintunsigned": - case "integer": - case "int": - return MathUtil.getIntegerVal(obj); - - case "bigint": - case "bigintunsigned": - case "intunsigned": - case "integerunsigned": - return MathUtil.getLongVal(obj); - - case "boolean": - return MathUtil.getBoolean(obj); - - case "blob": - return MathUtil.getByte(obj); - - case "varchar": - case "char": - case "text": - return MathUtil.getString(obj); - - case "real": - case "float": - case "realunsigned": - case "floatunsigned": - return MathUtil.getFloatVal(obj); - - case "double": - case "doubleunsigned": - return MathUtil.getDoubleVal(obj); - - case "decimal": - case "decimalunsigned": - return MathUtil.getBigDecimal(obj); - - case "date": - return MathUtil.getDate(obj); - - case "timestamp": - case "datetime": - return MathUtil.getTimestamp(obj); - } - return obj; - } -} diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java index fee2d2d8e..b866e2cd8 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java @@ -1,5 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.dtstack.flink.sql.side.elasticsearch6.table; +import com.dtstack.flink.sql.side.elasticsearch6.util.ClassUtil; import com.dtstack.flink.sql.table.AbsSideTableParser; import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; @@ -27,7 +46,6 @@ public class Elasticsearch6SideParser extends AbsSideTableParser { private static final String KEY_ES6_PASSWORD = "password"; - private static final String KEY_ES6_TIMEOUT = "timeout"; @Override protected boolean fieldNameNeedsUpperCase() { @@ -39,11 +57,11 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map stringConvertClass(String str) { + switch (str.toLowerCase()) { + case "boolean": + case "bit": + return Boolean.class; + + case "smallint": + case "smallintunsigned": + case "tinyint": + case "tinyintunsigned": + case "mediumint": + case "mediumintunsigned": + case "integer": + case "int": + return Integer.class; + + case "blob": + return Byte.class; + + case "long": + case "bigint": + case "intunsigned": + case "integerunsigned": + case "bigintunsigned": + return Long.class; + + case "varchar": + case "char": + case "text": + return String.class; + + case "real": + case "float": + case "realunsigned": + case "floatunsigned": + return Float.class; + + case "double": + case "doubleunsigned": + return Double.class; + + case "date": + return Date.class; + + case "datetime": + case "timestamp": + return Timestamp.class; + + case "time": + return Time.class; + + case "decimal": + case "decimalunsigned": + return BigDecimal.class; + + } + + throw new RuntimeException("不支持 " + str + " 类型"); + } + + public static Object convertType(Object field, String fromType, String toType) { + fromType = fromType.toUpperCase(); + toType = toType.toUpperCase(); + String rowData = field.toString(); + + switch(toType) { + case "TINYINT": + return Byte.valueOf(rowData); + case "SMALLINT": + return Short.valueOf(rowData); + case "INT": + return Integer.valueOf(rowData); + case "BIGINT": + return Long.valueOf(rowData); + case "FLOAT": + return Float.valueOf(rowData); + case "DOUBLE": + return Double.valueOf(rowData); + case "STRING": + return rowData; + case "BOOLEAN": + return Boolean.valueOf(rowData); + case "DATE": + return DateUtil.columnToDate(field); + case "TIMESTAMP": + Date d = DateUtil.columnToDate(field); + return new Timestamp(d.getTime()); + default: + throw new RuntimeException("Can't convert from " + fromType + " to " + toType); + } + + } + + public static String getTypeFromClass(Class clz) { + + if(clz == Byte.class){ + return "TINYINT"; + } + else if(clz == Short.class){ + return "SMALLINT"; + } + else if(clz == Integer.class){ + return "INT"; + } + else if(clz == Long.class){ + return "BIGINT"; + } + else if(clz == String.class){ + return "STRING"; + } + else if(clz == Float.class){ + return "FLOAT"; + } + else if(clz == Double.class){ + return "DOUBLE"; + } + else if(clz == Date.class){ + return "DATE"; + } + else if(clz == Timestamp.class){ + return "TIMESTAMP"; + } + else if(clz == Boolean.class){ + return "BOOLEAN"; + } + throw new IllegalArgumentException("Unsupported data type: " + clz.getName()); + + } + + public static String getTypeFromClassName(String clzName) { + + if(clzName.equals(Byte.class.getName())){ + return "TINYINT"; + } + else if(clzName.equals(Short.class.getName())){ + return "SMALLINT"; + } + else if(clzName.equals(Integer.class.getName())){ + return "INT"; + } + else if(clzName.equals(Long.class.getName())){ + return "BIGINT"; + } + else if(clzName.equals(String.class.getName())){ + return "STRING"; + } + else if(clzName.equals(Float.class.getName())){ + return "FLOAT"; + } + else if(clzName.equals(Double.class.getName())){ + return "DOUBLE"; + } + else if(clzName.equals(Date.class.getName())){ + return "DATE"; + } + else if(clzName.equals(Timestamp.class.getName())){ + return "TIMESTAMP"; + } + else if(clzName.equals(Boolean.class.getName())){ + return "BOOLEAN"; + } + throw new IllegalArgumentException("Unsupported data type: " + clzName); + } + +} diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java similarity index 98% rename from elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java rename to elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java index 7765ddf17..d6af042f4 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java @@ -103,6 +103,8 @@ public static Float getFloatVal(Object obj) { return (Float) obj; } else if (obj instanceof BigDecimal) { return ((BigDecimal) obj).floatValue(); + } else if (obj instanceof Double) { + return ((Double) obj).floatValue(); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Float."); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java similarity index 99% rename from elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java rename to elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java index 5f2e08651..ef3e1f50f 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java @@ -39,6 +39,7 @@ public static Object getTarget(Object obj, String targetType) { case "int": return MathUtil.getIntegerVal(obj); + case "long": case "bigint": case "bigintunsigned": case "intunsigned": From 54d4459f51b41177bee43b3ea112c2aba469329a Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Fri, 21 Feb 2020 11:39:26 +0800 Subject: [PATCH 026/523] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 451a983ac..d54c87c17 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -872,7 +872,7 @@ private void joinFun(Object pollObj, Map localTableCache, replaceInfoList.add(replaceInfo); if (!tableEnv.isRegistered(joinInfo.getNewTableName())){ - Table joinTable = tableEnv.fromDataStream(dsOut, String.join(",", sideOutTypeInfo.getFieldNames())); + Table joinTable = tableEnv.fromDataStream(dsOut); tableEnv.registerTable(joinInfo.getNewTableName(), joinTable); localTableCache.put(joinInfo.getNewTableName(), joinTable); } From e0e4316d0f8037cd38d063f3170401b37aa4902e Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Sun, 23 Feb 2020 20:00:23 +0800 Subject: [PATCH 027/523] =?UTF-8?q?elasticsearch6=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E9=83=A8=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Elasticsearch6AllReqRow.java | 260 +++------------- .../Elasticsearch6AsyncReqRow.java | 292 ++++-------------- .../elasticsearch6-side-core/pom.xml | 6 + .../sql/side/elasticsearch6/util/Es6Util.java | 211 +++++++++++++ 4 files changed, 331 insertions(+), 438 deletions(-) create mode 100644 elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index 8fdd02e87..d9d9e9487 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -24,30 +24,26 @@ import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; +import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; import com.dtstack.flink.sql.side.elasticsearch6.util.SwitchUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.action.search.*; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,9 +51,10 @@ import java.io.Serializable; import java.sql.SQLException; import java.sql.Timestamp; -import java.util.*; +import java.util.Calendar; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; /** * @author yinxi @@ -67,19 +64,16 @@ public class Elasticsearch6AllReqRow extends AllReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AllReqRow.class); - private static final Integer SCROLL_TIME = 1; private static final int CONN_RETRY_NUM = 3; - private static final String KEY_WORD_TYPE = ".keyword"; private AtomicReference>>> cacheRef = new AtomicReference<>(); - private String scrollId; - private Scroll scroll; private transient RestHighLevelClient rhlClient; + private SearchRequest searchRequest; + private BoolQueryBuilder boolQueryBuilder; public Elasticsearch6AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new Elasticsearch6AllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } - @Override public void flatMap(CRow value, Collector out) throws Exception { List inputParams = Lists.newArrayList(); @@ -165,6 +159,8 @@ protected void initCache() throws SQLException { Map>> newCache = Maps.newConcurrentMap(); cacheRef.set(newCache); try { + searchRequest = Es6Util.setSearchRequest(sideInfo); + boolQueryBuilder = Es6Util.setPredicateclause(sideInfo); loadData(newCache); } catch (Exception e) { LOG.error("", e); @@ -191,7 +187,7 @@ private void loadData(Map>> tmpCache) throws IO try { for (int i = 0; i < CONN_RETRY_NUM; i++) { try { - rhlClient = getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword()); + rhlClient = Es6Util.getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword()); break; } catch (Exception e) { if (i == CONN_RETRY_NUM - 1) { @@ -208,81 +204,56 @@ private void loadData(Map>> tmpCache) throws IO } } - - - // load data from tableA - SearchSourceBuilder searchSourceBuilder = getSelectFromStatement(sideInfo.getSideTableInfo().getPredicateInfoes()); - searchSourceBuilder.size(getFetchSize()); - SearchRequest searchRequest = new SearchRequest(); - scroll = new Scroll(TimeValue.timeValueMinutes(SCROLL_TIME)); - searchRequest.scroll(scroll); - - // determine existence of index - String index = tableInfo.getIndex().trim(); - if (!StringUtils.isEmpty(index)) { - // strip leading and trailing spaces from a string - String[] indexes = StringUtils.split(index, ","); - for (int i = 0; i < indexes.length; i++) { - indexes[i] = indexes[i].trim(); - } - - searchRequest.indices(indexes); - - } - - // determine existence of type - String type = tableInfo.getEsType().trim(); - if (!StringUtils.isEmpty(type)) { - // strip leading and trailing spaces from a string - String[] types = StringUtils.split(type, ","); - for (int i = 0; i < types.length; i++) { - types[i] = types[i].trim(); - } - - searchRequest.types(types); - } - - // add query condition - searchRequest.source(searchSourceBuilder); - - // get query reults - searchScroll(searchRequest, tmpCache); - + SearchSourceBuilder searchSourceBuilder = initConfiguration(boolQueryBuilder); + searchData(searchSourceBuilder, tmpCache); } catch (Exception e) { LOG.error("", e); } finally { - if (!StringUtils.isEmpty(scrollId)) { - clearScroll(); - } if (rhlClient != null) { rhlClient.close(); } } + } + + private SearchSourceBuilder initConfiguration(BoolQueryBuilder boolQueryBuilder){ + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + if (boolQueryBuilder != null) { + searchSourceBuilder.query(boolQueryBuilder); + } + searchSourceBuilder.size(getFetchSize()); + searchSourceBuilder.sort("_id", SortOrder.DESC); + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); + searchSourceBuilder.fetchSource(sideFieldNames, null); + return searchSourceBuilder; } - public void searchScroll(SearchRequest searchRequest, Map>> tmpCache) throws IOException { - SearchResponse searchResponse = rhlClient.search(searchRequest, RequestOptions.DEFAULT); - scrollId = searchResponse.getScrollId(); - SearchHit[] searchHits = searchResponse.getHits().getHits(); - loadToCache(searchHits, tmpCache); - - if (!StringUtils.isEmpty(scrollId)) { - SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); - scrollRequest.scroll(scroll); - while (true) { - SearchResponse scrollResponse = rhlClient.scroll(scrollRequest, RequestOptions.DEFAULT); - if (scrollResponse.getHits().getHits() == null || scrollResponse.getHits().getHits().length < 1) { - break; + + private void searchData(SearchSourceBuilder searchSourceBuilder, Map>> tmpCache) { + + Object[] searchAfterParameter = null; + SearchResponse searchResponse = null; + SearchHit[] searchHits = null; + + while (true) { + try { + if (searchAfterParameter != null) { + searchSourceBuilder.searchAfter(searchAfterParameter); } + searchRequest.source(searchSourceBuilder); + searchResponse = rhlClient.search(searchRequest, RequestOptions.DEFAULT); searchHits = searchResponse.getHits().getHits(); loadToCache(searchHits, tmpCache); + + if (searchHits.length < getFetchSize()) { + break; + } - scrollId = scrollResponse.getScrollId(); - scrollRequest.scrollId(scrollId); - + searchAfterParameter = searchHits[searchHits.length - 1].getSortValues(); + } catch (IOException e) { + LOG.error("Query failed!", e); } } } @@ -305,138 +276,7 @@ private void loadToCache(SearchHit[] searchHits, Map httpHostList = new ArrayList<>(); - String[] address = StringUtils.split(esAddress, ","); - for (String addr : address) { - String[] infoArray = StringUtils.split(addr, ":"); - int port = 9200; - String host = infoArray[0].trim(); - if (infoArray.length > 1) { - port = Integer.valueOf(infoArray[1].trim()); - } - httpHostList.add(new HttpHost(host, port, "http")); - } - - RestClientBuilder restClientBuilder = RestClient.builder(httpHostList.toArray(new HttpHost[httpHostList.size()])); - - if (isAuthMesh) { - // 进行用户和密码认证 - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName.trim(), password.trim())); - restClientBuilder.setHttpClientConfigCallback(httpAsyncClientBuilder -> - httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); - } - - RestHighLevelClient rhlClient = new RestHighLevelClient(restClientBuilder); - - if (LOG.isInfoEnabled()) { - LOG.info("Pinging Elasticsearch cluster via hosts {} ...", httpHostList); - } - - try { - if (!rhlClient.ping()) { - throw new RuntimeException("There are no reachable Elasticsearch nodes!"); - } - } catch (IOException e) { - LOG.warn("", e); - } - - - if (LOG.isInfoEnabled()) { - LOG.info("Created Elasticsearch RestHighLevelClient connected to {}", httpHostList.toString()); - } - - return rhlClient; - - } - public int getFetchSize() { return 1000; } - - private SearchSourceBuilder getSelectFromStatement(List predicateInfoes) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - - if (predicateInfoes.size() > 0) { - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - for (PredicateInfo info : predicateInfoes) { - boolQueryBuilder = buildFilterCondition(boolQueryBuilder, info); - } - - searchSourceBuilder.query(boolQueryBuilder); - } - - return searchSourceBuilder; - } - - public BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info) { - switch (info.getOperatorKind()) { - case "IN": - return boolQueryBuilder.must(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName()), removeSpaces(info.getCondition()))); - case "NOT_IN": - return boolQueryBuilder.mustNot(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName()), removeSpaces(info.getCondition()))); - case "GREATER_THAN_OR_EQUAL": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gte(info.getCondition())); - case "GREATER_THAN": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gt(info.getCondition())); - case "LESS_THAN_OR_EQUAL": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lte(info.getCondition())); - case "LESS_THAN": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lt(info.getCondition())); - case "BETWEEN": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gte(StringUtils.split(info.getCondition().toUpperCase(), "AND")[0].trim()) - .lte(StringUtils.split(info.getCondition().toUpperCase(), "AND")[1].trim())); - case "IS_NULL": - return boolQueryBuilder.mustNot(QueryBuilders.existsQuery(info.getFieldName())); - case "IS_NOT_NULL": - return boolQueryBuilder.must(QueryBuilders.existsQuery(info.getFieldName())); - case "EQUALS": - return boolQueryBuilder.must(QueryBuilders.termQuery(textConvertToKeyword(info.getFieldName()), info.getCondition())); - case "NOT_EQUALS": - return boolQueryBuilder.mustNot(QueryBuilders.termQuery(textConvertToKeyword(info.getFieldName()), info.getCondition())); - default: - try { - throw new Exception("elasticsearch6 does not support this operation: " + info.getOperatorName()); - } catch (Exception e) { - - e.printStackTrace(); - LOG.error(e.getMessage()); - } - return boolQueryBuilder; - } - - } - - public String[] removeSpaces(String str) { - String[] split = StringUtils.split(str, ","); - String[] result = new String[split.length]; - Arrays.asList(split).stream().map(f -> f.trim()).collect(Collectors.toList()).toArray(result); - return result; - } - - public String textConvertToKeyword(String fieldName) { - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); - String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); - int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); - String fieldType = sideFieldTypes[fieldIndex]; - switch (fieldType.toLowerCase()) { - case "varchar": - case "char": - case "text": - return fieldName + KEY_WORD_TYPE; - default: - return fieldName; - } - } - - } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index 16458d56e..411b79d26 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.elasticsearch6; import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -28,39 +29,28 @@ import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; +import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; import com.dtstack.flink.sql.side.elasticsearch6.util.SwitchUtil; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.impl.client.BasicCredentialsProvider; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.search.*; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.io.Serializable; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; /** * @author yinxi @@ -69,11 +59,9 @@ public class Elasticsearch6AsyncReqRow extends AsyncReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AsyncReqRow.class); - private static final Integer SCROLL_TIME = 1 ; - private static final String KEY_WORD_TYPE = ".keyword"; private transient RestHighLevelClient rhlClient; - private transient Scroll scroll; - private List sqlJoinCompareOperate= Lists.newArrayList(); + private SearchRequest searchRequest; + private List sqlJoinCompareOperate = Lists.newArrayList(); public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new Elasticsearch6AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); @@ -81,6 +69,15 @@ public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, Lis ParseUtils.parseJoinCompareOperate(conditionNode, sqlJoinCompareOperate); } + @Override + public void open(Configuration parameters) throws Exception{ + super.open(parameters); + Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); + rhlClient = Es6Util.getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword()); + searchRequest = Es6Util.setSearchRequest(sideInfo); + + } + @Override public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { @@ -116,98 +113,42 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce } } - Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); - - rhlClient = getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword()); - - // load data from tableA - SearchSourceBuilder searchSourceBuilder = getSelectFromStatement(inputParams, sideInfo.getSideTableInfo().getPredicateInfoes()); - - searchSourceBuilder.size(getFetchSize()); - SearchRequest searchRequest = new SearchRequest(); - scroll = new Scroll(TimeValue.timeValueMinutes(SCROLL_TIME)); - searchRequest.scroll(scroll); - - // determine existence of index - String index = tableInfo.getIndex().trim(); - if (!StringUtils.isEmpty(index)) { - // strip leading and trailing spaces from a string - String[] indexes = StringUtils.split(index, ","); - for (int i = 0; i < indexes.length; i++) { - indexes[i] = indexes[i].trim(); - } - - searchRequest.indices(indexes); - - } - - // determine existence of type - String type = tableInfo.getEsType().trim(); - if (!StringUtils.isEmpty(type)) { - // strip leading and trailing spaces from a string - String[] types = StringUtils.split(type, ","); - for (int i = 0; i < types.length; i++) { - types[i] = types[i].trim(); - } + BoolQueryBuilder boolQueryBuilder = Es6Util.setPredicateclause(sideInfo); + boolQueryBuilder = setInputParams(inputParams, boolQueryBuilder); + SearchSourceBuilder searchSourceBuilder = initConfiguration(); - searchRequest.types(types); - } + searchSourceBuilder.query(boolQueryBuilder); - // add query condition searchRequest.source(searchSourceBuilder); // 异步查询数据 - rhlClient.searchAsync(searchRequest, RequestOptions.DEFAULT, new ActionListener() { //成功响应时 @Override public void onResponse(SearchResponse searchResponse) { - String scrollId = searchResponse.getScrollId(); List cacheContent = Lists.newArrayList(); - List results = Lists.newArrayList(); - loadDataToCache(searchResponse, key, results, cacheContent, copyCrow, resultFuture); - - if (!StringUtils.isEmpty(scrollId)) { - SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); - scrollRequest.scroll(scroll); - while (true) { - SearchResponse scrollResponse = null; - try { - scrollResponse = rhlClient.scroll(scrollRequest, RequestOptions.DEFAULT); - } catch (IOException e) { - resultFuture.completeExceptionally(new RuntimeException("Request failed!")); - } - - if (scrollResponse.getHits().getHits() == null || scrollResponse.getHits().getHits().length < 1) { - break; - } - loadDataToCache(searchResponse, key, results, cacheContent, copyCrow, resultFuture); - - scrollId = scrollResponse.getScrollId(); - scrollRequest.scrollId(scrollId); - - } - } - try { - clearScroll(scrollId); - } catch (IOException e) { - LOG.warn("Clear scroll response:{}: failure", e); - } - - } - - private void loadDataToCache(SearchResponse searchResponse, String key, List results, List cacheContent, CRow copyCrow, ResultFuture resultFuture) { + List rowList = Lists.newArrayList(); SearchHit[] searchHits = searchResponse.getHits().getHits(); if (searchHits.length > 0) { - try { - for (SearchHit searchHit : searchHits) { - Map object = searchHit.getSourceAsMap(); - results.add(object); + try{ + loadDataToCache(searchHits, rowList, cacheContent, copyCrow); + while (true) { + if (searchHits.length < getFetchSize()) { + break; + } + BoolQueryBuilder newBoolQueryBuilder = Es6Util.setPredicateclause(sideInfo); + newBoolQueryBuilder = setInputParams(inputParams, newBoolQueryBuilder); + Object[] searchAfterParameter = searchHits[searchHits.length - 1].getSortValues(); + SearchSourceBuilder newSearchSourceBuilder = initConfiguration(); + newSearchSourceBuilder.searchAfter(searchAfterParameter); + newSearchSourceBuilder.query(newBoolQueryBuilder); + searchRequest.source(newSearchSourceBuilder); + searchResponse = rhlClient.search(searchRequest, RequestOptions.DEFAULT); + searchHits = searchResponse.getHits().getHits(); + loadDataToCache(searchHits, rowList, cacheContent, copyCrow); } - - List rowList = getRows(copyCrow, cacheContent, results); dealCacheData(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); resultFuture.complete(rowList); } catch (Exception e) { @@ -219,13 +160,7 @@ private void loadDataToCache(SearchResponse searchResponse, String key, List rowList, List cacheContent, CRow copyCrow){ + List results = Lists.newArrayList(); + for (SearchHit searchHit : searchHits) { + Map object = searchHit.getSourceAsMap(); + results.add(object); + } + rowList.addAll(getRows(copyCrow, cacheContent, results)); + } + + protected List getRows(CRow inputRow, List cacheContent, List results) { List rowList = Lists.newArrayList(); for (Object line : results) { @@ -269,7 +214,7 @@ public Row fillData(Row input, Object line) { if (cacheInfo == null) { row.setField(entry.getKey(), null); } else { - Object object = SwitchUtil.getTarget(cacheInfo.get(entry.getValue()), fields[entry.getValue()]); + Object object = SwitchUtil.getTarget(cacheInfo.get(sideInfo.getSideFieldNameIndex().get(entry.getKey())), fields[entry.getValue()]); row.setField(entry.getKey(), object); } } @@ -296,144 +241,35 @@ public String buildCacheKey(List equalValList) { return sb.toString(); } + private SearchSourceBuilder initConfiguration() { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(getFetchSize()); + searchSourceBuilder.sort("_id", SortOrder.DESC); + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); + searchSourceBuilder.fetchSource(sideFieldNames, null); - public RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, String userName, String password) { - - List httpHostList = new ArrayList<>(); - String[] address = StringUtils.split(esAddress, ","); - for (String addr : address) { - String[] infoArray = StringUtils.split(addr, ":"); - int port = 9200; - String host = infoArray[0].trim(); - if (infoArray.length > 1) { - port = Integer.valueOf(infoArray[1].trim()); - } - httpHostList.add(new HttpHost(host, port, "http")); - } - - RestClientBuilder restClientBuilder = RestClient.builder(httpHostList.toArray(new HttpHost[httpHostList.size()])); - - if (isAuthMesh) { - // 进行用户和密码认证 - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName.trim(), password.trim())); - restClientBuilder.setHttpClientConfigCallback(httpAsyncClientBuilder -> - httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); - } - - RestHighLevelClient rhlClient = new RestHighLevelClient(restClientBuilder); - - if (LOG.isInfoEnabled()) { - LOG.info("Pinging Elasticsearch cluster via hosts {} ...", httpHostList); - } - - try { - if (!rhlClient.ping()) { - throw new RuntimeException("There are no reachable Elasticsearch nodes!"); - } - } catch (IOException e) { - LOG.warn("", e); - } - - - if (LOG.isInfoEnabled()) { - LOG.info("Created Elasticsearch RestHighLevelClient connected to {}", httpHostList.toString()); - } - - return rhlClient; - - } - - public int getFetchSize() { - return 1000; + return searchSourceBuilder; } - private SearchSourceBuilder getSelectFromStatement(List inputParams, List predicateInfoes) { - - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + private BoolQueryBuilder setInputParams(List inputParams, BoolQueryBuilder boolQueryBuilder){ + if(boolQueryBuilder == null){ + boolQueryBuilder = new BoolQueryBuilder(); + } for (int i = 0; i < sqlJoinCompareOperate.size(); i++) { String fieldName = sideInfo.getEqualFieldList().get(i); String operatorKind = sqlJoinCompareOperate.get(sideInfo.getEqualFieldList().indexOf(fieldName)); String condition = String.valueOf(inputParams.get(i)); - boolQueryBuilder = buildFilterCondition(boolQueryBuilder, new PredicateInfo(null, operatorKind, null, fieldName, condition)); - } - - if (predicateInfoes.size() > 0) { - for (PredicateInfo info : predicateInfoes) { - boolQueryBuilder = buildFilterCondition(boolQueryBuilder, info); - } - } - searchSourceBuilder.query(boolQueryBuilder); - return searchSourceBuilder; - } - - public BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info) { - switch (info.getOperatorKind()) { - case "IN": - return boolQueryBuilder.must(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName()), removeSpaces(info.getCondition()))); - case "NOT_IN": - return boolQueryBuilder.mustNot(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName()), removeSpaces(info.getCondition()))); - case ">=": - case "GREATER_THAN_OR_EQUAL": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gte(info.getCondition())); - case ">": - case "GREATER_THAN": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gt(info.getCondition())); - case "<=": - case "LESS_THAN_OR_EQUAL": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lte(info.getCondition())); - case "<": - case "LESS_THAN": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lt(info.getCondition())); - case "BETWEEN": - return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gte(StringUtils.split(info.getCondition().toUpperCase(), "AND")[0].trim()) - .lte(StringUtils.split(info.getCondition().toUpperCase(), "AND")[1].trim())); - case "IS_NULL": - return boolQueryBuilder.mustNot(QueryBuilders.existsQuery(info.getFieldName())); - case "IS_NOT_NULL": - return boolQueryBuilder.must(QueryBuilders.existsQuery(info.getFieldName())); - case "=": - case "EQUALS": - return boolQueryBuilder.must(QueryBuilders.termQuery(textConvertToKeyword(info.getFieldName()), info.getCondition())); - case "<>": - case "NOT_EQUALS": - return boolQueryBuilder.mustNot(QueryBuilders.termQuery(textConvertToKeyword(info.getFieldName()), info.getCondition())); - default: - try { - throw new Exception("elasticsearch6 does not support this operation: " + info.getOperatorName()); - } catch (Exception e) { - - e.printStackTrace(); - LOG.error(e.getMessage()); - } - return boolQueryBuilder; + boolQueryBuilder = Es6Util.buildFilterCondition(boolQueryBuilder, new PredicateInfo(null, operatorKind, null, fieldName, condition), sideInfo); } + return boolQueryBuilder; } - public String[] removeSpaces(String str) { - String[] split = StringUtils.split(str, ","); - String[] result = new String[split.length]; - Arrays.asList(split).stream().map(f -> f.trim()).collect(Collectors.toList()).toArray(result); - return result; - } - public String textConvertToKeyword(String fieldName) { - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); - String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); - int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); - String fieldType = sideFieldTypes[fieldIndex]; - switch (fieldType.toLowerCase()) { - case "varchar": - case "char": - case "text": - return fieldName + KEY_WORD_TYPE; - default: - return fieldName; - } + public int getFetchSize() { + return 1000; } } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/pom.xml b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/pom.xml index 73c46fbb5..9149bdc8f 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/pom.xml +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/pom.xml @@ -25,6 +25,12 @@ elasticsearch-rest-high-level-client ${elasticsearch.version} + + junit + junit + 4.12 + test + diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java new file mode 100644 index 000000000..21bb7717c --- /dev/null +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.elasticsearch6.util; + +import com.dtstack.flink.sql.side.PredicateInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * @author yinxi + * @date 2020/2/21 - 20:51 + */ +public class Es6Util { + + private static final Logger LOG = LoggerFactory.getLogger(Es6Util.class); + private static final String KEY_WORD_TYPE = ".keyword"; + private static final String APOSTROPHE = "\'"; + + public static RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, String userName, String password) { + List httpHostList = new ArrayList<>(); + String[] address = StringUtils.split(esAddress, ","); + for (String addr : address) { + String[] infoArray = StringUtils.split(addr, ":"); + int port = 9200; + String host = infoArray[0].trim(); + if (infoArray.length > 1) { + port = Integer.valueOf(infoArray[1].trim()); + } + httpHostList.add(new HttpHost(host, port, "http")); + } + + RestClientBuilder restClientBuilder = RestClient.builder(httpHostList.toArray(new HttpHost[httpHostList.size()])); + + if (isAuthMesh) { + // 进行用户和密码认证 + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName.trim(), password.trim())); + restClientBuilder.setHttpClientConfigCallback(httpAsyncClientBuilder -> + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + + RestHighLevelClient rhlClient = new RestHighLevelClient(restClientBuilder); + + if (LOG.isInfoEnabled()) { + LOG.info("Pinging Elasticsearch cluster via hosts {} ...", httpHostList); + } + + try { + if (!rhlClient.ping()) { + throw new RuntimeException("There are no reachable Elasticsearch nodes!"); + } + } catch (IOException e) { + LOG.warn("", e); + } + + + if (LOG.isInfoEnabled()) { + LOG.info("Created Elasticsearch RestHighLevelClient connected to {}", httpHostList.toString()); + } + + return rhlClient; + } + + public static SearchRequest setSearchRequest(SideInfo sideInfo) { + SearchRequest searchRequest = new SearchRequest(); + Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); + // determine existence of index + String index = tableInfo.getIndex(); + if (!StringUtils.isEmpty(index) || index.trim().length() > 0) { + // strip leading and trailing spaces from a string + String[] indexes = StringUtils.split(index, ","); + for (int i = 0; i < indexes.length; i++) { + indexes[i] = indexes[i].trim(); + } + + searchRequest.indices(indexes); + } + + // determine existence of type + String type = tableInfo.getEsType(); + if (!StringUtils.isEmpty(type) || type.trim().length() > 0) { + // strip leading and trailing spaces from a string + String[] types = StringUtils.split(type, ","); + for (int i = 0; i < types.length; i++) { + types[i] = types[i].trim(); + } + + searchRequest.types(types); + } + + return searchRequest; + } + + public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { + + BoolQueryBuilder boolQueryBuilder = null; + List predicateInfoes = sideInfo.getSideTableInfo().getPredicateInfoes(); + if (predicateInfoes.size() > 0) { + boolQueryBuilder = new BoolQueryBuilder(); + for (PredicateInfo info : predicateInfoes) { + boolQueryBuilder = Es6Util.buildFilterCondition(boolQueryBuilder, info, sideInfo); + } + } + + return boolQueryBuilder; + } + + public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, SideInfo sideInfo) { + switch (info.getOperatorKind()) { + case "IN": + return boolQueryBuilder.must(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName(), sideInfo), removeSpaceAndApostrophe(info.getCondition()))); + case "NOT_IN": + return boolQueryBuilder.mustNot(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName(), sideInfo), removeSpaceAndApostrophe(info.getCondition()))); + case ">=": + case "GREATER_THAN_OR_EQUAL": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gte(info.getCondition())); + case ">": + case "GREATER_THAN": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gt(info.getCondition())); + case "<=": + case "LESS_THAN_OR_EQUAL": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lte(info.getCondition())); + case "<": + case "LESS_THAN": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).lt(info.getCondition())); + case "BETWEEN": + return boolQueryBuilder.must(QueryBuilders.rangeQuery(info.getFieldName()).gte(StringUtils.split(info.getCondition().toUpperCase(), "AND")[0].trim()) + .lte(StringUtils.split(info.getCondition().toUpperCase(), "AND")[1].trim())); + case "IS_NULL": + return boolQueryBuilder.mustNot(QueryBuilders.existsQuery(info.getFieldName())); + case "IS_NOT_NULL": + return boolQueryBuilder.must(QueryBuilders.existsQuery(info.getFieldName())); + case "=": + case "EQUALS": + return boolQueryBuilder.must(QueryBuilders.termQuery(textConvertToKeyword(info.getFieldName(), sideInfo), removeSpaceAndApostrophe(info.getCondition())[0])); + case "<>": + case "NOT_EQUALS": + return boolQueryBuilder.mustNot(QueryBuilders.termQuery(textConvertToKeyword(info.getFieldName(), sideInfo), removeSpaceAndApostrophe(info.getCondition())[0])); + default: + try { + throw new Exception("elasticsearch6 does not support this operation: " + info.getOperatorKind()); + } catch (Exception e) { + + e.printStackTrace(); + LOG.error(e.getMessage()); + } + return boolQueryBuilder; + } + + } + + public static String[] removeSpaceAndApostrophe(String str) { + String[] split = StringUtils.split(str, ","); + for (int i = 0; i < split.length; i++) { + split[i] = StringUtils.trim(split[i]); + if (StringUtils.startsWith(split[i], APOSTROPHE)) { + split[i] = StringUtils.substring(split[i], 1, split[i].length() - 1); + } + } + + return split; + } + + public static String textConvertToKeyword(String fieldName, SideInfo sideInfo) { + String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); + int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); + String fieldType = sideFieldTypes[fieldIndex]; + switch (fieldType.toLowerCase()) { + case "varchar": + case "char": + case "text": + return fieldName + KEY_WORD_TYPE; + default: + return fieldName; + } + } +} From 18f0c6267e1e04aa5e9b3835ec9f53c4036a42b0 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Sun, 23 Feb 2020 21:40:32 +0800 Subject: [PATCH 028/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E8=A7=84=E8=8C=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/elasticsearch6/Elasticsearch6Sink.java | 4 ++-- .../flink/sql/sink/elasticsearch6/Es6Util.java | 7 +++++-- ...iCallBridge.java => ExtendEs6ApiCallBridge.java} | 13 +++++++------ .../elasticsearch6/MetricElasticsearch6Sink.java | 2 +- .../table/ElasticsearchSinkParser.java | 5 ++++- 5 files changed, 19 insertions(+), 12 deletions(-) rename elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/{ExtendES6ApiCallBridge.java => ExtendEs6ApiCallBridge.java} (93%) diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Elasticsearch6Sink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Elasticsearch6Sink.java index 08f334f79..5ac50bc8d 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Elasticsearch6Sink.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Elasticsearch6Sink.java @@ -32,6 +32,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.elasticsearch6.table.ElasticsearchTableInfo; import com.dtstack.flink.sql.table.TargetTableInfo; +import com.google.common.collect.Maps; import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; import org.slf4j.Logger; @@ -39,7 +40,6 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -105,7 +105,7 @@ public TypeInformation[] getFieldTypes() { private RichSinkFunction createEsSinkFunction() { - Map userConfig = new HashMap<>(); + Map userConfig = Maps.newHashMap(); userConfig.put("cluster.name", clusterName); // This instructs the sink to emit after every element, otherwise they would be buffered userConfig.put(org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "" + bulkFlushMaxActions); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Es6Util.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Es6Util.java index ecd07b746..df892381d 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Es6Util.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Es6Util.java @@ -21,6 +21,8 @@ import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; +import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; + import com.dtstack.flink.sql.util.DtStringUtil; import java.util.HashMap; @@ -35,7 +37,7 @@ public class Es6Util { public static Map rowToJsonMap(Row row, List fields, List types) { Preconditions.checkArgument(row.getArity() == fields.size()); - Map jsonMap = new HashMap<>(); + Map jsonMap = Maps.newHashMap(); int i = 0; for(; i < fields.size(); ++i) { String field = fields.get(i); @@ -44,7 +46,8 @@ public static Map rowToJsonMap(Row row, List fields, Lis for(int j = 0; j < parts.length - 1; ++j) { String key = parts[j]; if(currMap.get(key) == null) { - currMap.put(key, new HashMap()); + HashMap hashMap = Maps.newHashMap(); + currMap.put(key, hashMap); } currMap = (Map) currMap.get(key); } diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendES6ApiCallBridge.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendEs6ApiCallBridge.java similarity index 93% rename from elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendES6ApiCallBridge.java rename to elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendEs6ApiCallBridge.java index 4ab596fc9..2eafb8a2b 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendES6ApiCallBridge.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendEs6ApiCallBridge.java @@ -31,6 +31,7 @@ import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkProcessor; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; @@ -48,17 +49,17 @@ * @author yinxi * @date 2020/1/9 - 15:09 */ -public class ExtendES6ApiCallBridge implements ElasticsearchApiCallBridge { +public class ExtendEs6ApiCallBridge implements ElasticsearchApiCallBridge { - private static final Logger LOG = LoggerFactory.getLogger(ExtendES6ApiCallBridge.class); + private static final Logger LOG = LoggerFactory.getLogger(ExtendEs6ApiCallBridge.class); private final List HttpAddresses; protected ElasticsearchTableInfo es6TableInfo; - public ExtendES6ApiCallBridge(List HttpAddresses, ElasticsearchTableInfo es6TableInfo) { - Preconditions.checkArgument(HttpAddresses != null && !HttpAddresses.isEmpty()); - this.HttpAddresses = HttpAddresses; + public ExtendEs6ApiCallBridge(List httpAddresses, ElasticsearchTableInfo es6TableInfo) { + Preconditions.checkArgument(httpAddresses != null && !httpAddresses.isEmpty()); + this.HttpAddresses = httpAddresses; this.es6TableInfo = es6TableInfo; } @@ -81,7 +82,7 @@ public RestHighLevelClient createClient(Map clientConfig) { } try{ - if (!rhlClient.ping()) { + if (!rhlClient.ping(RequestOptions.DEFAULT)) { throw new RuntimeException("There are no reachable Elasticsearch nodes!"); } } catch (IOException e){ diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java index ffd14bf43..4415e7675 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java @@ -49,7 +49,7 @@ public class MetricElasticsearch6Sink extends ElasticsearchSinkBase Date: Mon, 24 Feb 2020 02:27:11 +0800 Subject: [PATCH 029/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E8=A7=84=E8=8C=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/elasticsearch6Side | 2 +- docs/elasticsearch6Sink.md | 4 +- .../Elasticsearch6AsyncReqRow.java | 61 ++++-- .../table/Elasticsearch6SideParser.java | 4 +- .../side/elasticsearch6/util/ClassUtil.java | 6 +- .../sql/side/elasticsearch6/util/Es6Util.java | 3 +- .../side/elasticsearch6/util/SwitchUtil.java | 3 +- elasticsearch6/elasticsearch6-sink/pom.xml | 126 ------------- .../sink/elasticsearch6/CustomerSinkFunc.java | 119 ------------ .../elasticsearch6/Elasticsearch6Sink.java | 173 ------------------ .../sql/sink/elasticsearch6/Es6Util.java | 64 ------- .../ExtendES6ApiCallBridge.java | 139 -------------- .../MetricElasticsearch6Sink.java | 70 ------- .../table/ElasticsearchSinkParser.java | 74 -------- .../table/ElasticsearchTableInfo.java | 139 -------------- elasticsearch6/pom.xml | 1 - 16 files changed, 62 insertions(+), 926 deletions(-) delete mode 100644 elasticsearch6/elasticsearch6-sink/pom.xml delete mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java delete mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Elasticsearch6Sink.java delete mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Es6Util.java delete mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendES6ApiCallBridge.java delete mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java delete mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchSinkParser.java delete mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchTableInfo.java diff --git a/docs/elasticsearch6Side b/docs/elasticsearch6Side index 936ac7295..279db888d 100644 --- a/docs/elasticsearch6Side +++ b/docs/elasticsearch6Side @@ -40,7 +40,7 @@ |参数名称|含义|是否必填|默认值| |----|---|---|----| - type|表明 输出表类型[mysq|hbase|elasticsearch6]|是|| + type|表明 输出表类型[elasticsearch6]|是|| |address | 连接ES Transport地址(tcp地址)|是|| |cluster | ES 集群名称 |是|| |index | 选择的ES上的index名称|否|| diff --git a/docs/elasticsearch6Sink.md b/docs/elasticsearch6Sink.md index f8a398217..a708eb40a 100644 --- a/docs/elasticsearch6Sink.md +++ b/docs/elasticsearch6Sink.md @@ -27,11 +27,11 @@ CREATE TABLE tableName( ## 4.参数： |参数名称|含义|是否必填|默认值| |----|---|---|----| -|type|表明 输出表类型[mysq|hbase|elasticsearch]|是|| +|type|表明 输出表类型[elasticsearch6]|是|| |address | 连接ES Transport地址(tcp地址)|是|| |cluster | ES 集群名称 |是|| |index | 选择的ES上的index名称|是|| -|estype | 选择ES上的type名称|是|| +|esType | 选择ES上的type名称|是|| |id | 生成id的规则(当前是根据指定的字段pos获取字段信息,拼接生成id;|是|| | |若id为空字符串或索引都超出范围，则随机生成id值)||| |authMesh | 是否进行用户名密码认证 | 否 | false| diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index 411b79d26..7c0f782a5 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -47,6 +47,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.io.Serializable; import java.sql.Timestamp; import java.util.List; @@ -121,6 +122,35 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce searchRequest.source(searchSourceBuilder); + +// List cacheContent = Lists.newArrayList(); +// List rowList = Lists.newArrayList(); +// ActionListener searchActionListener = new ActionListener() { +// @Override +// public void onResponse(SearchResponse searchResponse) { +// +// SearchHit[] searchHits = searchResponse.getHits().getHits(); +// while (searchHits != null && searchHits.length > 0){ +// loadDataToCache(searchHits, rowList, cacheContent, copyCrow); +// if (searchHits.length < getFetchSize()) { +// break; +// } +// +// Object[] searchAfterParameter = searchHits[searchHits.length - 1].getSortValues(); +// searchSourceBuilder.searchAfter(searchAfterParameter); +// searchRequest.source(searchSourceBuilder); +// SearchResponse newSearchResponse = rhlClient.search(searchRequest, RequestOptions.DEFAULT); +// searchHits = newSearchResponse.getHits().getHits(); +// +// } +// } +// +// @Override +// public void onFailure(Exception e) { +// +// } +// }; + // 异步查询数据 rhlClient.searchAsync(searchRequest, RequestOptions.DEFAULT, new ActionListener() { @@ -132,27 +162,36 @@ public void onResponse(SearchResponse searchResponse) { List rowList = Lists.newArrayList(); SearchHit[] searchHits = searchResponse.getHits().getHits(); if (searchHits.length > 0) { - try{ - loadDataToCache(searchHits, rowList, cacheContent, copyCrow); + Elasticsearch6SideTableInfo tableInfo = null; + RestHighLevelClient tmpRhlClient = null; + try { while (true) { + loadDataToCache(searchHits, rowList, cacheContent, copyCrow); if (searchHits.length < getFetchSize()) { break; } - BoolQueryBuilder newBoolQueryBuilder = Es6Util.setPredicateclause(sideInfo); - newBoolQueryBuilder = setInputParams(inputParams, newBoolQueryBuilder); + if (tableInfo == null && tmpRhlClient == null) { + tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); + tmpRhlClient = Es6Util.getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword()); + } Object[] searchAfterParameter = searchHits[searchHits.length - 1].getSortValues(); - SearchSourceBuilder newSearchSourceBuilder = initConfiguration(); - newSearchSourceBuilder.searchAfter(searchAfterParameter); - newSearchSourceBuilder.query(newBoolQueryBuilder); - searchRequest.source(newSearchSourceBuilder); - searchResponse = rhlClient.search(searchRequest, RequestOptions.DEFAULT); + searchSourceBuilder.searchAfter(searchAfterParameter); + searchRequest.source(searchSourceBuilder); + searchResponse = tmpRhlClient.search(searchRequest, RequestOptions.DEFAULT); searchHits = searchResponse.getHits().getHits(); - loadDataToCache(searchHits, rowList, cacheContent, copyCrow); } dealCacheData(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); resultFuture.complete(rowList); } catch (Exception e) { dealFillDataError(resultFuture, e, copyCrow); + } finally { + if (tmpRhlClient != null) { + try { + tmpRhlClient.close(); + } catch (IOException e) { + LOG.warn("Failed to shut down tmpRhlClient.", e); + } + } } } else { dealMissKey(copyCrow, resultFuture); @@ -160,8 +199,6 @@ public void onResponse(SearchResponse searchResponse) { } } - - // 响应失败处理 @Override public void onFailure(Exception e) { diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java index b866e2cd8..13dfe2995 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java @@ -46,6 +46,8 @@ public class Elasticsearch6SideParser extends AbsSideTableParser { private static final String KEY_ES6_PASSWORD = "password"; + private static final String KEY_TRUE = "true"; + @Override protected boolean fieldNameNeedsUpperCase() { @@ -64,7 +66,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map stringConvertClass(String str) { case "decimal": case "decimalunsigned": return BigDecimal.class; - + default: + throw new RuntimeException("不支持 " + str + " 类型"); } - throw new RuntimeException("不支持 " + str + " 类型"); } public static Object convertType(Object field, String fromType, String toType) { diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java index 21bb7717c..0956cebf6 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java @@ -28,6 +28,7 @@ import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; @@ -80,7 +81,7 @@ public static RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh } try { - if (!rhlClient.ping()) { + if (!rhlClient.ping(RequestOptions.DEFAULT)) { throw new RuntimeException("There are no reachable Elasticsearch nodes!"); } } catch (IOException e) { diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java index ef3e1f50f..e0aaa123e 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java @@ -77,7 +77,8 @@ public static Object getTarget(Object obj, String targetType) { case "timestamp": case "datetime": return MathUtil.getTimestamp(obj); + default: + return obj; } - return obj; } } diff --git a/elasticsearch6/elasticsearch6-sink/pom.xml b/elasticsearch6/elasticsearch6-sink/pom.xml deleted file mode 100644 index cb5fb753c..000000000 --- a/elasticsearch6/elasticsearch6-sink/pom.xml +++ /dev/null @@ -1,126 +0,0 @@ - - - - sql.elasticsearch6 - com.dtstack.flink - 1.0-SNAPSHOT - - 4.0.0 - - sql.sink.elasticsearch6 - elasticsearch6-sink - - - 6.8.6 - - - - - org.apache.flink - flink-streaming-java_2.11 - ${flink.version} - provided - - - - org.elasticsearch.client - elasticsearch-rest-high-level-client - ${elasticsearch.version} - - - - ch.qos.logback - logback-core - 1.1.7 - - - - ch.qos.logback - logback-classic - 1.1.7 - - - - org.apache.logging.log4j - log4j-to-slf4j - 2.7 - - - - org.apache.flink - flink-connector-elasticsearch6_2.11 - ${flink.version} - - - - org.apache.flink - flink-connector-elasticsearch-base_2.11 - ${flink.version} - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 1.4 - - - package - - shade - - - - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - maven-antrun-plugin - 1.2 - - - copy-resources - - package - - run - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java deleted file mode 100644 index 091e18ede..000000000 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.elasticsearch6; - -import org.apache.flink.api.common.functions.RuntimeContext; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.metrics.Counter; -import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction; -import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer; -import org.apache.flink.types.Row; - -import org.apache.commons.lang3.StringUtils; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.client.Requests; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * @author yinxi - * @date 2020/1/9 - 15:10 - */ -public class CustomerSinkFunc implements ElasticsearchSinkFunction { - - private final Logger logger = LoggerFactory.getLogger(CustomerSinkFunc.class); - - private String index; - - private String type; - - private List idFieldIndexList; - - private List fieldNames; - - private List fieldTypes; - - public transient Counter outRecords; - - /** 默认分隔符为'_' */ - private char sp = '_'; - - public CustomerSinkFunc(String index, String type, List fieldNames, List fieldTypes, List idFieldIndexes){ - this.index = index; - this.type = type; - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - this.idFieldIndexList = idFieldIndexes; - } - - @Override - public void process(Tuple2 tuple2, RuntimeContext ctx, RequestIndexer indexer) { - try{ - Tuple2 tupleTrans = tuple2; - Boolean retract = tupleTrans.getField(0); - Row element = tupleTrans.getField(1); - if(!retract){ - return; - } - - - indexer.add(createIndexRequest(element)); - outRecords.inc(); - }catch (Throwable e){ - logger.error("", e); - } - } - - public void setOutRecords(Counter outRecords) { - this.outRecords = outRecords; - } - - private IndexRequest createIndexRequest(Row element) { - - List idFieldList = new ArrayList<>(); - for(int index : idFieldIndexList){ - if(index >= element.getArity()){ - continue; - } - - idFieldList.add(element.getField(index).toString()); - } - - Map dataMap = Es6Util.rowToJsonMap(element,fieldNames,fieldTypes); - int length = Math.min(element.getArity(), fieldNames.size()); - for(int i=0; i, IStreamSinkGener { - - private final Logger logger = LoggerFactory.getLogger(Elasticsearch6Sink.class); - - private String clusterName; - - private int bulkFlushMaxActions = 1; - - private List esAddressList; - - private String index = ""; - - private String type = ""; - - private List idIndexList; - - protected String[] fieldNames; - - protected String[] columnTypes; - - private TypeInformation[] fieldTypes; - - private int parallelism = -1; - - private ElasticsearchTableInfo esTableInfo; - - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), getRecordType()); - } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - - private RichSinkFunction createEsSinkFunction() { - - - Map userConfig = new HashMap<>(); - userConfig.put("cluster.name", clusterName); - // This instructs the sink to emit after every element, otherwise they would be buffered - userConfig.put(org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "" + bulkFlushMaxActions); - List transports = new ArrayList<>(); - - for (String address : esAddressList) { - String[] infoArray = address.split(":"); - int port = 9200; - String host = infoArray[0]; - if (infoArray.length > 1) { - port = Integer.valueOf(infoArray[1].trim()); - } - - try { - transports.add(new HttpHost(host.trim(), port, "http")); - } catch (Exception e) { - logger.error("", e); - throw new RuntimeException(e); - } - } - - CustomerSinkFunc customerSinkFunc = new CustomerSinkFunc(index, type, Arrays.asList(fieldNames), Arrays.asList(columnTypes), idIndexList); - - return new MetricElasticsearch6Sink(userConfig, transports, customerSinkFunc, esTableInfo); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - RichSinkFunction richSinkFunction = createEsSinkFunction(); - DataStreamSink streamSink = dataStream.addSink(richSinkFunction); - if (parallelism > 0) { - streamSink.setParallelism(parallelism); - } - } - - public void setParallelism(int parallelism) { - this.parallelism = parallelism; - } - - public void setBulkFlushMaxActions(int bulkFlushMaxActions) { - this.bulkFlushMaxActions = bulkFlushMaxActions; - } - - @Override - public Elasticsearch6Sink genStreamSink(TargetTableInfo targetTableInfo) { - ElasticsearchTableInfo elasticsearchTableInfo = (ElasticsearchTableInfo) targetTableInfo; - esTableInfo = elasticsearchTableInfo; - clusterName = elasticsearchTableInfo.getClusterName(); - String address = elasticsearchTableInfo.getAddress(); - String[] addr = address.split(","); - esAddressList = Arrays.asList(addr); - index = elasticsearchTableInfo.getIndex(); - type = elasticsearchTableInfo.getEsType(); - String id = elasticsearchTableInfo.getId(); - String[] idField = StringUtils.split(id, ","); - idIndexList = new ArrayList<>(); - - for (int i = 0; i < idField.length; ++i) { - idIndexList.add(Integer.valueOf(idField[i])); - } - - columnTypes = elasticsearchTableInfo.getFieldTypes(); - return this; - } -} diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Es6Util.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Es6Util.java deleted file mode 100644 index ecd07b746..000000000 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Es6Util.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.elasticsearch6; - -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; - -import com.dtstack.flink.sql.util.DtStringUtil; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author yinxi - * @date 2020/1/9 - 15:08 - */ -public class Es6Util { - - public static Map rowToJsonMap(Row row, List fields, List types) { - Preconditions.checkArgument(row.getArity() == fields.size()); - Map jsonMap = new HashMap<>(); - int i = 0; - for(; i < fields.size(); ++i) { - String field = fields.get(i); - String[] parts = field.split("\\."); - Map currMap = jsonMap; - for(int j = 0; j < parts.length - 1; ++j) { - String key = parts[j]; - if(currMap.get(key) == null) { - currMap.put(key, new HashMap()); - } - currMap = (Map) currMap.get(key); - } - String key = parts[parts.length - 1]; - Object col = row.getField(i); - if(col != null) { - Object value = DtStringUtil.col2string(col, types.get(i)); - currMap.put(key, value); - } - - } - - return jsonMap; - } - - -} diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendES6ApiCallBridge.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendES6ApiCallBridge.java deleted file mode 100644 index 4ab596fc9..000000000 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendES6ApiCallBridge.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.elasticsearch6; - -import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchApiCallBridge; -import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase; -import org.apache.flink.util.Preconditions; - -import com.dtstack.flink.sql.sink.elasticsearch6.table.ElasticsearchTableInfo; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.action.bulk.BackoffPolicy; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkProcessor; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.unit.TimeValue; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nullable; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -/** - * @author yinxi - * @date 2020/1/9 - 15:09 - */ -public class ExtendES6ApiCallBridge implements ElasticsearchApiCallBridge { - - private static final Logger LOG = LoggerFactory.getLogger(ExtendES6ApiCallBridge.class); - - private final List HttpAddresses; - - protected ElasticsearchTableInfo es6TableInfo; - - public ExtendES6ApiCallBridge(List HttpAddresses, ElasticsearchTableInfo es6TableInfo) { - Preconditions.checkArgument(HttpAddresses != null && !HttpAddresses.isEmpty()); - this.HttpAddresses = HttpAddresses; - this.es6TableInfo = es6TableInfo; - } - - @Override - public RestHighLevelClient createClient(Map clientConfig) { - - RestClientBuilder restClientBuilder = RestClient.builder(HttpAddresses.toArray(new HttpHost[HttpAddresses.size()])); - if (es6TableInfo.isAuthMesh()) { - // 进行用户和密码认证 - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(es6TableInfo.getUserName(), es6TableInfo.getPassword())); - restClientBuilder.setHttpClientConfigCallback(httpAsyncClientBuilder -> - httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); - } - - RestHighLevelClient rhlClient = new RestHighLevelClient(restClientBuilder); - - if (LOG.isInfoEnabled()) { - LOG.info("Pinging Elasticsearch cluster via hosts {} ...", HttpAddresses); - } - - try{ - if (!rhlClient.ping()) { - throw new RuntimeException("There are no reachable Elasticsearch nodes!"); - } - } catch (IOException e){ - LOG.warn("", e); - } - - - if (LOG.isInfoEnabled()) { - LOG.info("Created Elasticsearch RestHighLevelClient connected to {}", HttpAddresses.toString()); - } - - return rhlClient; - } - - @Override - public BulkProcessor.Builder createBulkProcessorBuilder(RestHighLevelClient client, BulkProcessor.Listener listener) { - return BulkProcessor.builder(client::bulkAsync, listener); - } - - @Override - public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) { - if (!bulkItemResponse.isFailed()) { - return null; - } else { - return bulkItemResponse.getFailure().getCause(); - } - } - - @Override - public void configureBulkProcessorBackoff( - BulkProcessor.Builder builder, - @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { - - BackoffPolicy backoffPolicy; - if (flushBackoffPolicy != null) { - switch (flushBackoffPolicy.getBackoffType()) { - case CONSTANT: - backoffPolicy = BackoffPolicy.constantBackoff( - new TimeValue(flushBackoffPolicy.getDelayMillis()), - flushBackoffPolicy.getMaxRetryCount()); - break; - case EXPONENTIAL: - default: - backoffPolicy = BackoffPolicy.exponentialBackoff( - new TimeValue(flushBackoffPolicy.getDelayMillis()), - flushBackoffPolicy.getMaxRetryCount()); - } - } else { - backoffPolicy = BackoffPolicy.noBackoff(); - } - - builder.setBackoffPolicy(backoffPolicy); - } - -} diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java deleted file mode 100644 index ffd14bf43..000000000 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.elasticsearch6; - -import org.apache.flink.configuration.Configuration; -import org.apache.flink.metrics.Counter; -import org.apache.flink.metrics.Meter; -import org.apache.flink.metrics.MeterView; -import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase; -import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction; -import org.apache.flink.streaming.connectors.elasticsearch.util.NoOpFailureHandler; - -import com.dtstack.flink.sql.metric.MetricConstant; -import com.dtstack.flink.sql.sink.elasticsearch6.table.ElasticsearchTableInfo; -import org.elasticsearch.client.RestHighLevelClient; - -import java.util.List; -import java.util.Map; - -/** - * @author yinxi - * @date 2020/1/9 - 15:05 - */ -public class MetricElasticsearch6Sink extends ElasticsearchSinkBase { - - protected CustomerSinkFunc customerSinkFunc; - - protected transient Meter outRecordsRate; - - protected Map userConfig; - - - public MetricElasticsearch6Sink(Map userConfig, List transportAddresses, - ElasticsearchSinkFunction elasticsearchSinkFunction, - ElasticsearchTableInfo es6TableInfo) { - super(new ExtendES6ApiCallBridge(transportAddresses, es6TableInfo), userConfig, elasticsearchSinkFunction, new NoOpFailureHandler()); - this.customerSinkFunc = (CustomerSinkFunc) elasticsearchSinkFunction; - this.userConfig = userConfig; - } - - @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - initMetric(); - } - - - public void initMetric() { - Counter counter = getRuntimeContext().getMetricGroup().counter(MetricConstant.DT_NUM_RECORDS_OUT); - customerSinkFunc.setOutRecords(counter); - outRecordsRate = getRuntimeContext().getMetricGroup().meter(MetricConstant.DT_NUM_RECORDS_OUT_RATE, new MeterView(counter, 20)); - } -} - diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchSinkParser.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchSinkParser.java deleted file mode 100644 index e662a54f3..000000000 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchSinkParser.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.elasticsearch6.table; - -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.util.MathUtil; - -import java.util.Map; - -/** - * @author yinxi - * @date 2020/1/9 - 15:06 - */ -public class ElasticsearchSinkParser extends AbsTableParser { - - private static final String KEY_ES6_ADDRESS = "address"; - - private static final String KEY_ES6_CLUSTER = "cluster"; - - private static final String KEY_ES6_INDEX = "index"; - - private static final String KEY_ES6_TYPE = "estype"; - - private static final String KEY_ES6_ID_FIELD_INDEX_LIST = "id"; - - private static final String KEY_ES6_AUTHMESH = "authMesh"; - - private static final String KEY_ES6_USERNAME = "userName"; - - private static final String KEY_ES6_PASSWORD = "password"; - - @Override - protected boolean fieldNameNeedsUpperCase() { - return false; - } - - @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - ElasticsearchTableInfo elasticsearchTableInfo = new ElasticsearchTableInfo(); - elasticsearchTableInfo.setName(tableName); - parseFieldsInfo(fieldsInfo, elasticsearchTableInfo); - elasticsearchTableInfo.setAddress((String) props.get(KEY_ES6_ADDRESS.toLowerCase())); - elasticsearchTableInfo.setClusterName((String) props.get(KEY_ES6_CLUSTER.toLowerCase())); - elasticsearchTableInfo.setId((String) props.get(KEY_ES6_ID_FIELD_INDEX_LIST.toLowerCase())); - elasticsearchTableInfo.setIndex((String) props.get(KEY_ES6_INDEX.toLowerCase())); - elasticsearchTableInfo.setEsType((String) props.get(KEY_ES6_TYPE.toLowerCase())); - - String authMeshStr = (String) props.get(KEY_ES6_AUTHMESH.toLowerCase()); - if (authMeshStr != null && "true".equalsIgnoreCase(authMeshStr)) { - elasticsearchTableInfo.setAuthMesh(MathUtil.getBoolean(authMeshStr)); - elasticsearchTableInfo.setUserName(MathUtil.getString(props.get(KEY_ES6_USERNAME.toLowerCase()))); - elasticsearchTableInfo.setPassword(MathUtil.getString(props.get(KEY_ES6_PASSWORD.toLowerCase()))); - } - elasticsearchTableInfo.check(); - return elasticsearchTableInfo; - } -} diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchTableInfo.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchTableInfo.java deleted file mode 100644 index eb7dc1c4f..000000000 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchTableInfo.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.elasticsearch6.table; - -import com.dtstack.flink.sql.table.TargetTableInfo; -import com.google.common.base.Preconditions; - -/** - * @author yinxi - * @date 2020/1/9 - 15:06 - */ -public class ElasticsearchTableInfo extends TargetTableInfo { - - private static final String CURR_TYPE = "elasticsearch6"; - - private String address; - - private String index; - - private String id; - - private String clusterName; - - private String esType; - - private boolean authMesh = false; - - private String userName; - - private String password; - - public String getEsType() { - return esType; - } - - public void setEsType(String esType) { - this.esType = esType; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getAddress() { - return address; - } - - public void setAddress(String address) { - this.address = address; - } - - public String getIndex() { - return index; - } - - public void setIndex(String index) { - this.index = index; - } - - @Override - public String getType() { - //return super.getType().toLowerCase() + TARGET_SUFFIX; - return super.getType().toLowerCase(); - } - - public String getClusterName() { - return clusterName; - } - - public void setClusterName(String clusterName) { - this.clusterName = clusterName; - } - - public boolean isAuthMesh() { - return authMesh; - } - - public void setAuthMesh(boolean authMesh) { - this.authMesh = authMesh; - } - - public String getUserName() { - return userName; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - - public ElasticsearchTableInfo() { - setType(CURR_TYPE); - } - - @Override - public boolean check() { - Preconditions.checkNotNull(address, "elasticsearch6 type of address is required"); - Preconditions.checkNotNull(index, "elasticsearch6 type of index is required"); - Preconditions.checkNotNull(esType, "elasticsearch6 type of type is required"); - Preconditions.checkNotNull(id, "elasticsearch6 type of id is required"); - Preconditions.checkNotNull(clusterName, "elasticsearch6 type of clusterName is required"); - - if (isAuthMesh()) { - Preconditions.checkNotNull(userName, "elasticsearch6 type of userName is required"); - Preconditions.checkNotNull(password, "elasticsearch6 type of password is required"); - } - - return true; - } - -} - diff --git a/elasticsearch6/pom.xml b/elasticsearch6/pom.xml index 38472cd4e..9bdd108b7 100644 --- a/elasticsearch6/pom.xml +++ b/elasticsearch6/pom.xml @@ -12,7 +12,6 @@ sql.elasticsearch6 pom - elasticsearch6-sink elasticsearch6-side From 2df63ae8960ab0be0b8380fe26c5198c3dc57c91 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Mon, 24 Feb 2020 02:38:28 +0800 Subject: [PATCH 030/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9elasticsearch6-sink?= =?UTF-8?q?=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/elasticsearch6Sink.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/elasticsearch6Sink.md b/docs/elasticsearch6Sink.md index f8a398217..981693735 100644 --- a/docs/elasticsearch6Sink.md +++ b/docs/elasticsearch6Sink.md @@ -7,9 +7,12 @@ CREATE TABLE tableName( type ='elasticsearch6', address ='ip:port[,ip:port]', cluster='clusterName', - estype ='esType', + esType ='esType', index ='index', id ='num[,num]', + authMesh = 'true', + userName = 'userName', + password = 'password', parallelism ='1' ) ``` @@ -27,11 +30,11 @@ CREATE TABLE tableName( ## 4.参数： |参数名称|含义|是否必填|默认值| |----|---|---|----| -|type|表明 输出表类型[mysq|hbase|elasticsearch]|是|| +|type|表明 输出表类型[elasticsearch6]|是|| |address | 连接ES Transport地址(tcp地址)|是|| |cluster | ES 集群名称 |是|| |index | 选择的ES上的index名称|是|| -|estype | 选择ES上的type名称|是|| +|esType | 选择ES上的type名称|是|| |id | 生成id的规则(当前是根据指定的字段pos获取字段信息,拼接生成id;|是|| | |若id为空字符串或索引都超出范围，则随机生成id值)||| |authMesh | 是否进行用户名密码认证 | 否 | false| @@ -48,8 +51,11 @@ CREATE TABLE MyResult( type ='elasticsearch', address ='172.16.10.47:9500', cluster='es_47_menghan', - estype ='type1', + esType ='type1', index ='xc_es_test', + authMesh = 'true', + userName = 'elastic', + password = 'abc123', id ='0,1', parallelism ='1' ) From 87643e52b6f69def3d785be123f00886b74e0146 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Mon, 24 Feb 2020 02:40:54 +0800 Subject: [PATCH 031/523] =?UTF-8?q?=E5=88=A0=E9=99=A4elasticsearch6-sink?= =?UTF-8?q?=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/elasticsearch6Sink.md | 56 -------------------------------------- 1 file changed, 56 deletions(-) delete mode 100644 docs/elasticsearch6Sink.md diff --git a/docs/elasticsearch6Sink.md b/docs/elasticsearch6Sink.md deleted file mode 100644 index a708eb40a..000000000 --- a/docs/elasticsearch6Sink.md +++ /dev/null @@ -1,56 +0,0 @@ -## 1.格式： -``` -CREATE TABLE tableName( - colName colType, - bb INT - )WITH( - type ='elasticsearch6', - address ='ip:port[,ip:port]', - cluster='clusterName', - estype ='esType', - index ='index', - id ='num[,num]', - parallelism ='1' - ) -``` -## 2.支持的版本 - elasticsearch 6.8.6 - -## 3.表结构定义 - -|参数名称|含义| -|----|---| -|tableName|在 sql 中使用的名称;即注册到flink-table-env上的名称| -|colName|列名称| -|colType|列类型 [colType支持的类型](colType.md)| - -## 4.参数： -|参数名称|含义|是否必填|默认值| -|----|---|---|----| -|type|表明 输出表类型[elasticsearch6]|是|| -|address | 连接ES Transport地址(tcp地址)|是|| -|cluster | ES 集群名称 |是|| -|index | 选择的ES上的index名称|是|| -|esType | 选择ES上的type名称|是|| -|id | 生成id的规则(当前是根据指定的字段pos获取字段信息,拼接生成id;|是|| -| |若id为空字符串或索引都超出范围，则随机生成id值)||| -|authMesh | 是否进行用户名密码认证 | 否 | false| -|userName | 用户名 | 否，authMesh='true'时为必填 || -|password | 密码 | 否，authMesh='true'时为必填 || -|parallelism | 并行度设置|否|1| - -## 5.样例： -``` -CREATE TABLE MyResult( - aa INT, - bb INT - )WITH( - type ='elasticsearch', - address ='172.16.10.47:9500', - cluster='es_47_menghan', - estype ='type1', - index ='xc_es_test', - id ='0,1', - parallelism ='1' - ) - ``` \ No newline at end of file From 2a70c5fe8a15bc6dc154eac348a37d6eccd79405 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Mon, 24 Feb 2020 04:00:38 +0800 Subject: [PATCH 032/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Elasticsearch6AsyncReqRow.java | 39 ++----------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index 7c0f782a5..e2351860c 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -71,7 +71,7 @@ public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, Lis } @Override - public void open(Configuration parameters) throws Exception{ + public void open(Configuration parameters) throws Exception { super.open(parameters); Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); rhlClient = Es6Util.getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword()); @@ -117,40 +117,9 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce BoolQueryBuilder boolQueryBuilder = Es6Util.setPredicateclause(sideInfo); boolQueryBuilder = setInputParams(inputParams, boolQueryBuilder); SearchSourceBuilder searchSourceBuilder = initConfiguration(); - searchSourceBuilder.query(boolQueryBuilder); - searchRequest.source(searchSourceBuilder); - -// List cacheContent = Lists.newArrayList(); -// List rowList = Lists.newArrayList(); -// ActionListener searchActionListener = new ActionListener() { -// @Override -// public void onResponse(SearchResponse searchResponse) { -// -// SearchHit[] searchHits = searchResponse.getHits().getHits(); -// while (searchHits != null && searchHits.length > 0){ -// loadDataToCache(searchHits, rowList, cacheContent, copyCrow); -// if (searchHits.length < getFetchSize()) { -// break; -// } -// -// Object[] searchAfterParameter = searchHits[searchHits.length - 1].getSortValues(); -// searchSourceBuilder.searchAfter(searchAfterParameter); -// searchRequest.source(searchSourceBuilder); -// SearchResponse newSearchResponse = rhlClient.search(searchRequest, RequestOptions.DEFAULT); -// searchHits = newSearchResponse.getHits().getHits(); -// -// } -// } -// -// @Override -// public void onFailure(Exception e) { -// -// } -// }; - // 异步查询数据 rhlClient.searchAsync(searchRequest, RequestOptions.DEFAULT, new ActionListener() { @@ -209,7 +178,7 @@ public void onFailure(Exception e) { } - private void loadDataToCache(SearchHit[] searchHits, List rowList, List cacheContent, CRow copyCrow){ + private void loadDataToCache(SearchHit[] searchHits, List rowList, List cacheContent, CRow copyCrow) { List results = Lists.newArrayList(); for (SearchHit searchHit : searchHits) { Map object = searchHit.getSourceAsMap(); @@ -289,8 +258,8 @@ private SearchSourceBuilder initConfiguration() { } - private BoolQueryBuilder setInputParams(List inputParams, BoolQueryBuilder boolQueryBuilder){ - if(boolQueryBuilder == null){ + private BoolQueryBuilder setInputParams(List inputParams, BoolQueryBuilder boolQueryBuilder) { + if (boolQueryBuilder == null) { boolQueryBuilder = new BoolQueryBuilder(); } From 5693662fd5654de6f470e1e5bd95f8440d0ca3b3 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Mon, 24 Feb 2020 14:36:46 +0800 Subject: [PATCH 033/523] =?UTF-8?q?string.split=E6=94=B9=E7=94=A8StringUti?= =?UTF-8?q?ls.split()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../side/cassandra/CassandraAllReqRow.java | 33 +- .../side/cassandra/CassandraAsyncReqRow.java | 38 +-- .../sink/cassandra/CassandraOutputFormat.java | 24 +- .../flink/sql/parser/CreateTableParser.java | 3 +- .../dtstack/flink/sql/side/SideSqlExec.java | 288 +++++++++--------- .../flink/sql/table/AbsTableParser.java | 2 +- .../flink/sql/table/TableInfoParser.java | 9 +- .../dtstack/flink/sql/util/DtStringUtil.java | 6 +- .../sink/elasticsearch/ElasticsearchSink.java | 20 +- .../flink/sql/sink/elasticsearch/EsUtil.java | 6 +- .../flink/sql/side/hbase/RowKeyBuilder.java | 5 +- .../sql/side/hbase/table/HbaseSideParser.java | 3 +- .../sql/sink/hbase/HbaseOutputFormat.java | 17 +- .../sql/sink/hbase/table/HbaseSinkParser.java | 7 +- .../side/impala/table/ImpalaSideParser.java | 6 +- .../sink/impala/table/ImpalaTableInfo.java | 3 +- .../flink/sql/source/kafka/KafkaConsumer.java | 6 +- .../sql/source/kafka/KafkaConsumer09.java | 6 +- .../sql/source/kafka/KafkaConsumer010.java | 6 +- .../sql/source/kafka/KafkaConsumer011.java | 6 +- .../flink/sql/side/kudu/KuduAllReqRow.java | 41 +-- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 24 +- .../flink/sql/side/kudu/utils/KuduUtil.java | 3 +- .../sql/launcher/perjob/PerJobSubmitter.java | 12 +- .../flink/sql/side/mongo/MongoAllReqRow.java | 27 +- .../sql/side/mongo/MongoAsyncReqRow.java | 31 +- .../flink/sql/side/mongo/utils/MongoUtil.java | 6 +- .../sql/sink/mongo/MongoOutputFormat.java | 15 +- .../flink/sql/side/rdb/all/RdbAllReqRow.java | 22 +- .../sql/side/rdb/all/RdbAllSideInfo.java | 13 +- .../sql/side/rdb/async/RdbAsyncSideInfo.java | 16 +- .../flink/sql/side/redis/RedisAllReqRow.java | 37 ++- .../sql/side/redis/RedisAsyncReqRow.java | 18 +- .../sql/sink/redis/RedisOutputFormat.java | 13 +- .../sql/sink/redis/table/RedisSinkParser.java | 3 +- 35 files changed, 369 insertions(+), 406 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index e30b6dfc8..8d9510e40 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -18,14 +18,13 @@ package com.dtstack.flink.sql.side.cassandra; -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.ConsistencyLevel; -import com.datastax.driver.core.HostDistance; -import com.datastax.driver.core.PoolingOptions; -import com.datastax.driver.core.QueryOptions; -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.SocketOptions; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; + +import com.datastax.driver.core.*; import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; import com.dtstack.flink.sql.side.AllReqRow; @@ -33,15 +32,11 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; -import org.apache.calcite.sql.JoinType; -import org.apache.commons.collections.CollectionUtils; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; +import org.apache.calcite.sql.JoinType; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -222,9 +217,9 @@ private Session getConn(CassandraSideTableInfo tableInfo) { //重试策略 RetryPolicy retryPolicy = DowngradingConsistencyRetryPolicy.INSTANCE; - for (String server : address.split(",")) { - cassandraPort = Integer.parseInt(server.split(":")[1]); - serversList.add(InetAddress.getByName(server.split(":")[0])); + for (String server : StringUtils.split(address, ",")) { + cassandraPort = Integer.parseInt(StringUtils.split(server, ":")[1]); + serversList.add(InetAddress.getByName(StringUtils.split(server, ":")[0])); } if (userName == null || userName.isEmpty() || password == null || password.isEmpty()) { @@ -278,7 +273,7 @@ private void loadData(Map>> tmpCache) throws SQ //load data from table String sql = sideInfo.getSqlCondition() + " limit " + FETCH_SIZE; ResultSet resultSet = session.execute(sql); - String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); for (com.datastax.driver.core.Row row : resultSet) { Map oneRow = Maps.newHashMap(); for (String fieldName : sideFieldNames) { diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index a60116e34..922821f51 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -19,44 +19,34 @@ package com.dtstack.flink.sql.side.cassandra; -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.ConsistencyLevel; -import com.datastax.driver.core.HostDistance; -import com.datastax.driver.core.PoolingOptions; -import com.datastax.driver.core.QueryOptions; -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.SocketOptions; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; + +import com.datastax.driver.core.*; import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; -import com.dtstack.flink.sql.side.CacheMissVal; -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.google.common.base.Function; +import com.google.common.collect.Lists; import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import io.vertx.core.json.JsonArray; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import com.google.common.collect.Lists; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.InetAddress; import java.sql.Timestamp; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -134,9 +124,9 @@ private void connCassandraDB(CassandraSideTableInfo tableInfo) { //重试策略 RetryPolicy retryPolicy = DowngradingConsistencyRetryPolicy.INSTANCE; - for (String server : address.split(",")) { - cassandraPort = Integer.parseInt(server.split(":")[1]); - serversList.add(InetAddress.getByName(server.split(":")[0])); + for (String server : StringUtils.split(address, ",")) { + cassandraPort = Integer.parseInt(StringUtils.split(server, ":")[1]); + serversList.add(InetAddress.getByName(StringUtils.split(server, ":")[0])); } if (userName == null || userName.isEmpty() || password == null || password.isEmpty()) { diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java index 59d849426..88ad81fcf 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java @@ -38,24 +38,20 @@ package com.dtstack.flink.sql.sink.cassandra; -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.ConsistencyLevel; -import com.datastax.driver.core.HostDistance; -import com.datastax.driver.core.PoolingOptions; -import com.datastax.driver.core.QueryOptions; -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.SocketOptions; -import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; -import com.datastax.driver.core.policies.RetryPolicy; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; + +import com.datastax.driver.core.*; +import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; +import com.datastax.driver.core.policies.RetryPolicy; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.IOException; import java.net.InetAddress; import java.sql.DriverManager; @@ -145,9 +141,9 @@ public void open(int taskNumber, int numTasks) { //重试策略 RetryPolicy retryPolicy = DowngradingConsistencyRetryPolicy.INSTANCE; - for (String server : address.split(",")) { - cassandraPort = Integer.parseInt(server.split(":")[1]); - serversList.add(InetAddress.getByName(server.split(":")[0])); + for (String server : StringUtils.split(address, ",")) { + cassandraPort = Integer.parseInt(StringUtils.split(server, ":")[1]); + serversList.add(InetAddress.getByName(StringUtils.split(server, ":")[0])); } if (userName == null || userName.isEmpty() || password == null || password.isEmpty()) { diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java index ae6e1f708..9a2d064d9 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java @@ -22,6 +22,7 @@ import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Map; @@ -69,7 +70,7 @@ public void parseSql(String sql, SqlTree sqlTree) { } private Map parseProp(String propsStr){ - String[] strs = propsStr.trim().split("'\\s*,"); + String[] strs = StringUtils.split(propsStr.trim(), "'\\s*,"); Map propMap = Maps.newHashMap(); for(int i=0; i ss = DtStringUtil.splitIgnoreQuota(strs[i], '='); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index d54c87c17..7db118c45 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -17,9 +17,22 @@ */ - package com.dtstack.flink.sql.side; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.table.api.StreamQueryConfig; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.runtime.CRowKeySelector; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; + import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.exec.FlinkSQLExec; import com.dtstack.flink.sql.parser.CreateTmpTableParser; @@ -31,47 +44,17 @@ import com.google.common.collect.HashBasedTable; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.calcite.sql.SqlAsOperator; -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlDataTypeSpec; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlInsert; -import org.apache.calcite.sql.SqlJoin; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlLiteral; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlOrderBy; -import org.apache.calcite.sql.SqlSelect; -import org.apache.calcite.sql.SqlWithItem; +import org.apache.calcite.sql.*; import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.collections.CollectionUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.table.api.StreamQueryConfig; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.table.runtime.CRowKeySelector; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.runtime.types.CRowTypeInfo; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.Arrays; -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; +import java.util.*; import static org.apache.calcite.sql.SqlKind.*; @@ -79,6 +62,7 @@ * Reason: * Date: 2018/7/24 * Company: www.dtstack.com + * * @author xuchao */ @@ -94,11 +78,11 @@ public class SideSqlExec { private SidePredicatesParser sidePredicatesParser = new SidePredicatesParser(); private Map localTableCache = Maps.newHashMap(); - private StreamTableEnvironment tableEnv ; + private StreamTableEnvironment tableEnv; public void exec(String sql, Map sideTableMap, StreamTableEnvironment tableEnv, Map tableCache, StreamQueryConfig queryConfig) throws Exception { - if(localSqlPluginPath == null){ + if (localSqlPluginPath == null) { throw new RuntimeException("need to set localSqlPluginPath"); } @@ -117,36 +101,36 @@ public void exec(String sql, Map sideTableMap, StreamTabl boolean preIsSideJoin = false; List replaceInfoList = Lists.newArrayList(); - while((pollObj = exeQueue.poll()) != null){ + while ((pollObj = exeQueue.poll()) != null) { - if(pollObj instanceof SqlNode){ + if (pollObj instanceof SqlNode) { SqlNode pollSqlNode = (SqlNode) pollObj; - if(preIsSideJoin){ + if (preIsSideJoin) { preIsSideJoin = false; List fieldNames = null; - for(FieldReplaceInfo replaceInfo : replaceInfoList){ + for (FieldReplaceInfo replaceInfo : replaceInfoList) { fieldNames = Lists.newArrayList(); replaceFieldName(pollSqlNode, replaceInfo); addAliasForFieldNode(pollSqlNode, fieldNames, replaceInfo.getMappingTable()); } } - if(pollSqlNode.getKind() == INSERT){ - System.out.println("----------real exec sql-----------" ); + if (pollSqlNode.getKind() == INSERT) { + System.out.println("----------real exec sql-----------"); System.out.println(pollSqlNode.toString()); FlinkSQLExec.sqlUpdate(tableEnv, pollSqlNode.toString(), queryConfig); - if(LOG.isInfoEnabled()){ + if (LOG.isInfoEnabled()) { LOG.info("exec sql: " + pollSqlNode.toString()); } - }else if(pollSqlNode.getKind() == AS){ + } else if (pollSqlNode.getKind() == AS) { AliasInfo aliasInfo = parseASNode(pollSqlNode); Table table = tableEnv.sqlQuery(aliasInfo.getName()); tableEnv.registerTable(aliasInfo.getAlias(), table); localTableCache.put(aliasInfo.getAlias(), table); FieldReplaceInfo fieldReplaceInfo = parseAsQuery((SqlBasicCall) pollSqlNode, tableCache); - if(fieldReplaceInfo != null){ + if (fieldReplaceInfo != null) { replaceInfoList.add(fieldReplaceInfo); } } else if (pollSqlNode.getKind() == WITH_ITEM) { @@ -156,7 +140,7 @@ public void exec(String sql, Map sideTableMap, StreamTabl tableEnv.registerTable(TableAlias, table); } - }else if (pollObj instanceof JoinInfo){ + } else if (pollObj instanceof JoinInfo) { preIsSideJoin = true; joinFun(pollObj, localTableCache, sideTableMap, tableEnv, replaceInfoList); } @@ -166,16 +150,17 @@ public void exec(String sql, Map sideTableMap, StreamTabl /** * 解析出as查询的表和字段的关系 + * * @param asSqlNode * @param tableCache * @return */ - private FieldReplaceInfo parseAsQuery(SqlBasicCall asSqlNode, Map tableCache){ + private FieldReplaceInfo parseAsQuery(SqlBasicCall asSqlNode, Map tableCache) { SqlNode info = asSqlNode.getOperands()[0]; SqlNode alias = asSqlNode.getOperands()[1]; SqlKind infoKind = info.getKind(); - if(infoKind != SELECT){ + if (infoKind != SELECT) { return null; } @@ -199,6 +184,7 @@ private FieldReplaceInfo parseAsQuery(SqlBasicCall asSqlNode, Map /** * 添加字段别名 + * * @param pollSqlNode * @param fieldList * @param mappingTable @@ -238,7 +224,7 @@ private void addAliasForFieldNode(SqlNode pollSqlNode, List fieldList, H } String name = sqlIdentifier.names.get(1); // avoid real field pv0 convert pv - if (name.endsWith("0") && !fieldList.contains(name) && !fieldList.contains(name.substring(0, name.length() - 1))) { + if (name.endsWith("0") && !fieldList.contains(name) && !fieldList.contains(name.substring(0, name.length() - 1))) { SqlOperator operator = new SqlAsOperator(); SqlParserPos sqlParserPos = new SqlParserPos(0, 0); @@ -259,11 +245,11 @@ private void addAliasForFieldNode(SqlNode pollSqlNode, List fieldList, H public AliasInfo parseASNode(SqlNode sqlNode) throws SqlParseException { SqlKind sqlKind = sqlNode.getKind(); - if(sqlKind != AS){ + if (sqlKind != AS) { throw new RuntimeException(sqlNode + " is not 'as' operator"); } - SqlNode info = ((SqlBasicCall)sqlNode).getOperands()[0]; + SqlNode info = ((SqlBasicCall) sqlNode).getOperands()[0]; SqlNode alias = ((SqlBasicCall) sqlNode).getOperands()[1]; AliasInfo aliasInfo = new AliasInfo(); @@ -290,9 +276,9 @@ public RowTypeInfo buildOutRowTypeInfo(List sideJoinFieldInfo, HashBa } - /** - * 对时间类型进行类型转换 + * 对时间类型进行类型转换 + * * @param leftTypeInfo * @return */ @@ -327,14 +313,14 @@ private void replaceFieldName(SqlNode sqlNode, FieldReplaceInfo replaceInfo) { break; case SELECT: SqlSelect sqlSelect = (SqlSelect) filterNodeWithTargetName(sqlNode, replaceInfo.getTargetTableName()); - if(sqlSelect == null){ + if (sqlSelect == null) { return; } SqlNode sqlSource1 = sqlSelect.getFrom(); - if(sqlSource1.getKind() == AS){ - String tableName = ((SqlBasicCall)sqlSource1).getOperands()[0].toString(); - if(tableName.equalsIgnoreCase(replaceInfo.getTargetTableName())){ + if (sqlSource1.getKind() == AS) { + String tableName = ((SqlBasicCall) sqlSource1).getOperands()[0].toString(); + if (tableName.equalsIgnoreCase(replaceInfo.getTargetTableName())) { SqlNodeList sqlSelectList = sqlSelect.getSelectList(); SqlNode whereNode = sqlSelect.getWhere(); SqlNodeList sqlGroup = sqlSelect.getGroup(); @@ -343,17 +329,17 @@ private void replaceFieldName(SqlNode sqlNode, FieldReplaceInfo replaceInfo) { SqlNode sqlHaving = sqlSelect.getHaving(); List newSelectNodeList = Lists.newArrayList(); - for( int i=0; i replaceNodeList = replaceSelectStarFieldName(selectNode, replaceInfo); newSelectNodeList.addAll(replaceNodeList); continue; } SqlNode replaceNode = replaceSelectFieldName(selectNode, replaceInfo); - if(replaceNode == null){ + if (replaceNode == null) { continue; } @@ -365,23 +351,23 @@ private void replaceFieldName(SqlNode sqlNode, FieldReplaceInfo replaceInfo) { sqlSelect.setSelectList(newSelectList); //where - if(whereNode != null){ - SqlNode[] sqlNodeList = ((SqlBasicCall)whereNode).getOperands(); - for(int i =0; i localTableCache, String tableAlias, String tableName){ + private Table getTableFromCache(Map localTableCache, String tableAlias, String tableName) { Table table = localTableCache.get(tableAlias); - if(table == null){ + if (table == null) { table = localTableCache.get(tableName); } - if(table == null){ + if (table == null) { throw new RuntimeException("not register table " + tableName); } return table; } - private List replaceSelectStarFieldName(SqlNode selectNode, FieldReplaceInfo replaceInfo){ + private List replaceSelectStarFieldName(SqlNode selectNode, FieldReplaceInfo replaceInfo) { SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; List sqlNodes = Lists.newArrayList(); - if(sqlIdentifier.isStar()){//处理 [* or table.*] + if (sqlIdentifier.isStar()) {//处理 [* or table.*] int identifierSize = sqlIdentifier.names.size(); Collection columns = null; - if(identifierSize == 1){ + if (identifierSize == 1) { columns = replaceInfo.getMappingTable().values(); - }else{ + } else { columns = replaceInfo.getMappingTable().row(sqlIdentifier.names.get(0)).values(); } - for(String colAlias : columns){ + for (String colAlias : columns) { SqlParserPos sqlParserPos = new SqlParserPos(0, 0); List columnInfo = Lists.newArrayList(); columnInfo.add(replaceInfo.getTargetTableAlias()); @@ -532,7 +518,7 @@ private List replaceSelectStarFieldName(SqlNode selectNode, FieldReplac } return sqlNodes; - }else{ + } else { throw new RuntimeException("is not a star select field." + selectNode); } } @@ -546,25 +532,25 @@ private SqlNode replaceSelectFieldName(SqlNode selectNode, FieldReplaceInfo repl } return selectNode; - }else if(selectNode.getKind() == IDENTIFIER){ + } else if (selectNode.getKind() == IDENTIFIER) { SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; - if(sqlIdentifier.names.size() == 1){ + if (sqlIdentifier.names.size() == 1) { return selectNode; } //Same level mappingTable String mappingFieldName = replaceInfo.getTargetFieldName(sqlIdentifier.getComponent(0).getSimple(), sqlIdentifier.getComponent(1).getSimple()); if (mappingFieldName == null) { - throw new RuntimeException("can't find mapping fieldName:" + selectNode.toString() ); + throw new RuntimeException("can't find mapping fieldName:" + selectNode.toString()); } sqlIdentifier = sqlIdentifier.setName(0, replaceInfo.getTargetTableAlias()); sqlIdentifier = sqlIdentifier.setName(1, mappingFieldName); return sqlIdentifier; - }else if(selectNode.getKind() == LITERAL || selectNode.getKind() == LITERAL_CHAIN){//字面含义 + } else if (selectNode.getKind() == LITERAL || selectNode.getKind() == LITERAL_CHAIN) {//字面含义 return selectNode; - }else if( AGGREGATE.contains(selectNode.getKind()) + } else if (AGGREGATE.contains(selectNode.getKind()) || AVG_AGG_FUNCTIONS.contains(selectNode.getKind()) || COMPARISON.contains(selectNode.getKind()) || selectNode.getKind() == OTHER_FUNCTION @@ -594,20 +580,20 @@ private SqlNode replaceSelectFieldName(SqlNode selectNode, FieldReplaceInfo repl || selectNode.getKind() == TIMESTAMP_DIFF || selectNode.getKind() == LIKE - ){ + ) { SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; - for(int i=0; i conditionFields = getConditionFields(conditionNode, sideTableAlias, sideTableInfo); - if(CollectionUtils.isEqualCollection(conditionFields, convertPrimaryAlias(sideTableInfo))){ + if (CollectionUtils.isEqualCollection(conditionFields, convertPrimaryAlias(sideTableInfo))) { return true; } return false; @@ -669,27 +655,27 @@ private List convertPrimaryAlias(SideTableInfo sideTableInfo) { return res; } - public List getConditionFields(SqlNode conditionNode, String specifyTableName, SideTableInfo sideTableInfo){ + public List getConditionFields(SqlNode conditionNode, String specifyTableName, SideTableInfo sideTableInfo) { List sqlNodeList = Lists.newArrayList(); ParseUtils.parseAnd(conditionNode, sqlNodeList); List conditionFields = Lists.newArrayList(); - for(SqlNode sqlNode : sqlNodeList){ + for (SqlNode sqlNode : sqlNodeList) { if (!SqlKind.COMPARISON.contains(sqlNode.getKind())) { throw new RuntimeException("not compare operator."); } - SqlIdentifier left = (SqlIdentifier)((SqlBasicCall)sqlNode).getOperands()[0]; - SqlIdentifier right = (SqlIdentifier)((SqlBasicCall)sqlNode).getOperands()[1]; + SqlIdentifier left = (SqlIdentifier) ((SqlBasicCall) sqlNode).getOperands()[0]; + SqlIdentifier right = (SqlIdentifier) ((SqlBasicCall) sqlNode).getOperands()[1]; String leftTableName = left.getComponent(0).getSimple(); String rightTableName = right.getComponent(0).getSimple(); String tableCol = ""; - if(leftTableName.equalsIgnoreCase(specifyTableName)){ + if (leftTableName.equalsIgnoreCase(specifyTableName)) { tableCol = left.getComponent(1).getSimple(); - }else if(rightTableName.equalsIgnoreCase(specifyTableName)){ + } else if (rightTableName.equalsIgnoreCase(specifyTableName)) { tableCol = right.getComponent(1).getSimple(); - }else{ + } else { throw new RuntimeException(String.format("side table:%s join condition is wrong", specifyTableName)); } tableCol = sideTableInfo.getPhysicalFields().getOrDefault(tableCol, tableCol); @@ -704,7 +690,7 @@ public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, Map tableCache) throws Exception { - if(localSqlPluginPath == null){ + if (localSqlPluginPath == null) { throw new RuntimeException("need to set localSqlPluginPath"); } @@ -716,12 +702,12 @@ public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, boolean preIsSideJoin = false; List replaceInfoList = Lists.newArrayList(); - while((pollObj = exeQueue.poll()) != null){ + while ((pollObj = exeQueue.poll()) != null) { - if(pollObj instanceof SqlNode){ + if (pollObj instanceof SqlNode) { SqlNode pollSqlNode = (SqlNode) pollObj; - if(preIsSideJoin){ + if (preIsSideJoin) { preIsSideJoin = false; List fieldNames = null; for (FieldReplaceInfo replaceInfo : replaceInfoList) { @@ -731,16 +717,16 @@ public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, } } - if(pollSqlNode.getKind() == INSERT){ + if (pollSqlNode.getKind() == INSERT) { tableEnv.sqlUpdate(pollSqlNode.toString()); - }else if(pollSqlNode.getKind() == AS){ + } else if (pollSqlNode.getKind() == AS) { dealAsSourceTable(tableEnv, pollSqlNode); - } else if (pollSqlNode.getKind() == SELECT){ + } else if (pollSqlNode.getKind() == SELECT) { Table table = tableEnv.sqlQuery(pollObj.toString()); - if (result.getFieldsInfoStr() == null){ + if (result.getFieldsInfoStr() == null) { tableEnv.registerTable(result.getTableName(), table); } else { - if (checkFieldsInfo(result, table)){ + if (checkFieldsInfo(result, table)) { table = table.as(tmpFields); tableEnv.registerTable(result.getTableName(), table); } else { @@ -751,7 +737,7 @@ public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, } - }else if (pollObj instanceof JoinInfo){ + } else if (pollObj instanceof JoinInfo) { preIsSideJoin = true; joinFun(pollObj, localTableCache, sideTableMap, tableEnv, replaceInfoList); } @@ -775,7 +761,7 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, SqlNode pollSq private void joinFun(Object pollObj, Map localTableCache, Map sideTableMap, StreamTableEnvironment tableEnv, - List replaceInfoList) throws Exception{ + List replaceInfoList) throws Exception { JoinInfo joinInfo = (JoinInfo) pollObj; JoinScope joinScope = new JoinScope(); @@ -784,7 +770,7 @@ private void joinFun(Object pollObj, Map localTableCache, leftScopeChild.setTableName(joinInfo.getLeftTableName()); SqlKind sqlKind = joinInfo.getLeftNode().getKind(); - if(sqlKind == AS){ + if (sqlKind == AS) { dealAsSourceTable(tableEnv, joinInfo.getLeftNode()); } @@ -796,15 +782,15 @@ private void joinFun(Object pollObj, Map localTableCache, rightScopeChild.setAlias(joinInfo.getRightTableAlias()); rightScopeChild.setTableName(joinInfo.getRightTableName()); SideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); - if(sideTableInfo == null){ + if (sideTableInfo == null) { sideTableInfo = sideTableMap.get(joinInfo.getRightTableAlias()); } - if(sideTableInfo == null){ + if (sideTableInfo == null) { throw new RuntimeException("can't not find side table:" + joinInfo.getRightTableName()); } - if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ + if (!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)) { throw new RuntimeException("ON condition must contain all equal fields!!!"); } @@ -818,7 +804,7 @@ private void joinFun(Object pollObj, Map localTableCache, String leftTableAlias = joinInfo.getLeftTableAlias(); Table targetTable = localTableCache.get(leftTableAlias); - if(targetTable == null){ + if (targetTable == null) { targetTable = localTableCache.get(joinInfo.getLeftTableName()); } @@ -839,9 +825,9 @@ private void joinFun(Object pollObj, Map localTableCache, } DataStream dsOut = null; - if(ECacheType.ALL.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ + if (ECacheType.ALL.name().equalsIgnoreCase(sideTableInfo.getCacheType())) { dsOut = SideWithAllCacheOperator.getSideJoinDataStream(adaptStream, sideTableInfo.getType(), localSqlPluginPath, typeInfo, joinInfo, sideJoinFieldInfo, sideTableInfo); - }else{ + } else { dsOut = SideAsyncOperator.getSideJoinDataStream(adaptStream, sideTableInfo.getType(), localSqlPluginPath, typeInfo, joinInfo, sideJoinFieldInfo, sideTableInfo); } @@ -861,9 +847,9 @@ private void joinFun(Object pollObj, Map localTableCache, replaceInfo.setTargetTableAlias(targetTableAlias); //判断之前是不是被替换过,被替换过则设置之前的替换信息作为上一个节点 - for(FieldReplaceInfo tmp : replaceInfoList){ - if(tmp.getTargetTableName().equalsIgnoreCase(joinInfo.getLeftTableName()) - ||tmp.getTargetTableName().equalsIgnoreCase(joinInfo.getLeftTableAlias())){ + for (FieldReplaceInfo tmp : replaceInfoList) { + if (tmp.getTargetTableName().equalsIgnoreCase(joinInfo.getLeftTableName()) + || tmp.getTargetTableName().equalsIgnoreCase(joinInfo.getLeftTableAlias())) { replaceInfo.setPreNode(tmp); break; } @@ -871,7 +857,7 @@ private void joinFun(Object pollObj, Map localTableCache, replaceInfoList.add(replaceInfo); - if (!tableEnv.isRegistered(joinInfo.getNewTableName())){ + if (!tableEnv.isRegistered(joinInfo.getNewTableName())) { Table joinTable = tableEnv.fromDataStream(dsOut); tableEnv.registerTable(joinInfo.getNewTableName(), joinTable); localTableCache.put(joinInfo.getNewTableName(), joinTable); @@ -891,20 +877,20 @@ private TypeInformation projectedTypeInfo(int[] fields, TableSchema schema) private boolean checkFieldsInfo(CreateTmpTableParser.SqlParserResult result, Table table) { List fieldNames = new LinkedList<>(); String fieldsInfo = result.getFieldsInfoStr(); - String[] fields = fieldsInfo.split(","); + String[] fields = StringUtils.split(fieldsInfo, ","); for (int i = 0; i < fields.length; i++) { - String[] filed = fields[i].split("\\s"); - if (filed.length < 2 || fields.length != table.getSchema().getColumnNames().length){ + String[] filed = StringUtils.split(fields[i], "\\s"); + if (filed.length < 2 || fields.length != table.getSchema().getColumnNames().length) { return false; } else { String[] filedNameArr = new String[filed.length - 1]; System.arraycopy(filed, 0, filedNameArr, 0, filed.length - 1); String fieldName = String.join(" ", filedNameArr); fieldNames.add(fieldName); - String fieldType = filed[filed.length - 1 ].trim(); + String fieldType = filed[filed.length - 1].trim(); Class fieldClass = ClassUtil.stringConvertClass(fieldType); Class tableField = table.getSchema().getFieldType(i).get().getTypeClass(); - if (fieldClass == tableField){ + if (fieldClass == tableField) { continue; } else { return false; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java index 31e70caa2..2895f2b2e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java @@ -92,7 +92,7 @@ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ throw new RuntimeException(String.format("table [%s],exists field empty.", tableInfo.getName())); } - String[] filedInfoArr = fieldRow.split("\\s+"); + String[] filedInfoArr = StringUtils.split(fieldRow, "\\s+"); if(filedInfoArr.length < 2 ){ throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow)); } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java index ae98d90ae..46ad3691a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java @@ -29,6 +29,7 @@ import com.dtstack.flink.sql.util.MathUtil; import com.google.common.base.Strings; import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; import java.util.Map; import java.util.regex.Matcher; @@ -109,11 +110,11 @@ public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserRe * @param tableField * @return */ - private static boolean checkIsSideTable(String tableField){ - String[] fieldInfos = tableField.split(","); - for(String field : fieldInfos){ + private static boolean checkIsSideTable(String tableField) { + String[] fieldInfos = StringUtils.split(tableField, ","); + for (String field : fieldInfos) { Matcher matcher = SIDE_PATTERN.matcher(field.trim()); - if(matcher.find()){ + if (matcher.find()) { return true; } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index 862b0700e..84fe49fd3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -180,14 +180,14 @@ public static String addJdbcParam(String dbUrl, Map addParams, b return dbUrl; } - String[] splits = dbUrl.split("\\?"); + String[] splits = StringUtils.split(dbUrl, "\\?"); String preStr = splits[0]; Map params = Maps.newHashMap(); if(splits.length > 1){ String existsParamStr = splits[1]; - String[] existsParams = existsParamStr.split("&"); + String[] existsParams = StringUtils.split(existsParamStr, "&"); for(String oneParam : existsParams){ - String[] kv = oneParam.split("="); + String[] kv = StringUtils.split(oneParam, "="); if(kv.length != 2){ throw new RuntimeException("illegal dbUrl:" + dbUrl); } diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 7ae2fb1ae..5933ab7ec 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -20,10 +20,6 @@ package com.dtstack.flink.sql.sink.elasticsearch; -import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -34,15 +30,17 @@ import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; + +import com.dtstack.flink.sql.sink.IStreamSinkGener; +import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.net.InetAddress; import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; /** * table output elastic5plugin @@ -116,7 +114,7 @@ private RichSinkFunction createEsSinkFunction(){ List transports = new ArrayList<>(); for(String address : esAddressList){ - String[] infoArray = address.split(":"); + String[] infoArray = StringUtils.split(address, ":"); int port = 9300; String host = infoArray[0]; if(infoArray.length > 1){ @@ -167,7 +165,7 @@ public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { esTableInfo = elasticsearchTableInfo; clusterName = elasticsearchTableInfo.getClusterName(); String address = elasticsearchTableInfo.getAddress(); - String[] addr = address.split(","); + String[] addr = StringUtils.split(address, ","); esAddressList = Arrays.asList(addr); index = elasticsearchTableInfo.getIndex(); type = elasticsearchTableInfo.getEsType(); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java index 26d7827da..24c5d3ab9 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java @@ -18,10 +18,12 @@ package com.dtstack.flink.sql.sink.elasticsearch; -import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; +import com.dtstack.flink.sql.util.DtStringUtil; +import org.apache.commons.lang3.StringUtils; + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,7 +42,7 @@ public static Map rowToJsonMap(Row row, List fields, Lis int i = 0; for(; i < fields.size(); ++i) { String field = fields.get(i); - String[] parts = field.split("\\."); + String[] parts = StringUtils.split(field, "\\."); Map currMap = jsonMap; for(int j = 0; j < parts.length - 1; ++j) { String key = parts[j]; diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java index 114b7fa6a..6ad5544ab 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java @@ -22,6 +22,7 @@ import com.dtstack.flink.sql.side.hbase.enums.EReplaceType; import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; import java.io.Serializable; import java.util.List; @@ -58,7 +59,7 @@ public void init(String rowKeyTempl){ fieldCols = infoAlias; } - String[] fieldArr = fieldCols.split("\\+"); + String[] fieldArr = StringUtils.split(fieldCols, "\\+"); List fieldList = Lists.newArrayList(); for(String oneField : fieldArr){ ReplaceInfo replaceInfo = getReplaceInfo(oneField); @@ -108,7 +109,7 @@ private String buildStr(List fieldList, Map refData public static String[] splitIgnoreQuotaBrackets(String str, String delimiter){ String splitPatternStr = delimiter + "(?![^()]*+\\))(?![^{}]*+})(?![^\\[\\]]*+\\])"; - return str.split(splitPatternStr); + return StringUtils.split(str, splitPatternStr); } /** diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 7b627da4d..1ba9a9907 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -23,6 +23,7 @@ import com.dtstack.flink.sql.table.AbsSideTableParser; import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; +import org.apache.commons.lang3.StringUtils; import java.util.Map; import java.util.regex.Matcher; @@ -83,7 +84,7 @@ private void dealField(Matcher matcher, TableInfo tableInfo){ String filedDefineStr = matcher.group(1); String aliasStr = matcher.group(2); - String[] filedInfoArr = filedDefineStr.split("\\s+"); + String[] filedInfoArr = StringUtils.split(filedDefineStr, "\\s+"); if(filedInfoArr.length < 2){ throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), matcher.group(0))); } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 409cf312f..990686afc 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -20,21 +20,18 @@ package com.dtstack.flink.sql.sink.hbase; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; -import com.dtstack.flink.sql.enums.EUpdateMode; -import com.google.common.collect.Lists; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; + +import com.dtstack.flink.sql.enums.EUpdateMode; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -280,7 +277,7 @@ public HbaseOutputFormat finish() { String[] columns = keySet.toArray(new String[keySet.size()]); for (int i = 0; i < columns.length; ++i) { String col = columns[i]; - String[] part = col.split(":"); + String[] part = StringUtils.split(col, ":"); families[i] = part[0]; qualifiers[i] = part[1]; } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index f1ae72b6b..e93e8e612 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -26,6 +26,7 @@ import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.MathUtil; +import org.apache.commons.lang3.StringUtils; import java.util.LinkedHashMap; import java.util.List; @@ -68,7 +69,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); + super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java index 1d6c941f2..78eabde09 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java @@ -18,7 +18,6 @@ package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.format.DeserializationMetricWrapper; import org.apache.flink.metrics.MetricGroup; import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks; import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks; @@ -31,6 +30,9 @@ import org.apache.flink.types.Row; import org.apache.flink.util.SerializedValue; +import com.dtstack.flink.sql.format.DeserializationMetricWrapper; +import org.apache.commons.lang3.StringUtils; + import java.util.Arrays; import java.util.Map; import java.util.Properties; @@ -49,7 +51,7 @@ public class KafkaConsumer09 extends FlinkKafkaConsumer09 { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer09(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); + super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java index 765658325..b78f47cd5 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java @@ -18,7 +18,6 @@ package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.format.DeserializationMetricWrapper; import org.apache.flink.metrics.MetricGroup; import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks; import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks; @@ -30,6 +29,9 @@ import org.apache.flink.types.Row; import org.apache.flink.util.SerializedValue; +import com.dtstack.flink.sql.format.DeserializationMetricWrapper; +import org.apache.commons.lang3.StringUtils; + import java.util.Arrays; import java.util.Map; import java.util.Properties; @@ -48,7 +50,7 @@ public class KafkaConsumer010 extends FlinkKafkaConsumer010 { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer010(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); + super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java index 1614b470a..276ce211f 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java @@ -18,7 +18,6 @@ package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.format.DeserializationMetricWrapper; import org.apache.flink.metrics.MetricGroup; import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks; import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks; @@ -30,6 +29,9 @@ import org.apache.flink.types.Row; import org.apache.flink.util.SerializedValue; +import com.dtstack.flink.sql.format.DeserializationMetricWrapper; +import org.apache.commons.lang3.StringUtils; + import java.util.Arrays; import java.util.Map; import java.util.Properties; @@ -48,7 +50,7 @@ public class KafkaConsumer011 extends FlinkKafkaConsumer011 { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer011(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); + super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 047665928..072d7f630 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -1,42 +1,29 @@ package com.dtstack.flink.sql.side.kudu; -import com.dtstack.flink.sql.side.AllReqRow; -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; -import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; -import org.apache.calcite.sql.JoinType; -import org.apache.commons.collections.CollectionUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; import org.apache.flink.util.Preconditions; + +import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; +import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.calcite.sql.JoinType; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; -import org.apache.kudu.client.KuduClient; -import org.apache.kudu.client.KuduException; -import org.apache.kudu.client.KuduPredicate; -import org.apache.kudu.client.KuduScanner; -import org.apache.kudu.client.KuduTable; -import org.apache.kudu.client.PartialRow; -import org.apache.kudu.client.RowResult; -import org.apache.kudu.client.RowResultIterator; +import org.apache.kudu.client.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.SQLException; import java.sql.Timestamp; -import java.util.Arrays; -import java.util.Calendar; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicReference; public class KuduAllReqRow extends AllReqRow { @@ -159,7 +146,7 @@ private void loadData(Map>> tmpCache) { } //load data from table assert scanner != null; - String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); while (scanner.hasMoreRows()) { @@ -262,7 +249,7 @@ private KuduScanner buildScanner(KuduScanner.KuduScannerBuilder builder, Schema Long limitNum = tableInfo.getLimitNum(); Boolean isFaultTolerant = tableInfo.getFaultTolerant(); //查询需要的字段 - String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); //主键过滤条件 主键最小值 String lowerBoundPrimaryKey = tableInfo.getLowerBoundPrimaryKey(); //主键过滤条件 主键最大值 @@ -316,7 +303,7 @@ private KuduScanner buildScanner(KuduScanner.KuduScannerBuilder builder, Schema } private String[] splitString(String data) { - return data.split(","); + return StringUtils.split(data, ","); } @Override diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index 4a7457418..4b47f9e92 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -1,22 +1,24 @@ package com.dtstack.flink.sql.side.kudu; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; import io.vertx.core.json.JsonArray; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import org.apache.flink.configuration.Configuration; -import com.google.common.collect.Maps; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; +import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.*; @@ -101,7 +103,7 @@ private void connKuDu() throws KuduException { Long limitNum = kuduSideTableInfo.getLimitNum(); Boolean isFaultTolerant = kuduSideTableInfo.getFaultTolerant(); //查询需要的字段 - String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); if (null == limitNum || limitNum <= 0) { scannerBuilder.limit(FETCH_SIZE); @@ -268,7 +270,7 @@ public GetListRowCB() { public Deferred> call(RowResultIterator results) throws Exception { for (RowResult result : results) { Map oneRow = Maps.newHashMap(); - for (String sideFieldName1 : sideInfo.getSideSelectFields().split(",")) { + for (String sideFieldName1 : StringUtils.split(sideInfo.getSideSelectFields(), ",")) { String sideFieldName = sideFieldName1.trim(); ColumnSchema columnSchema = table.getSchema().getColumn(sideFieldName); if (null != columnSchema) { diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java index c5cd8cdb2..b750e2d4a 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java @@ -20,6 +20,7 @@ import com.dtstack.flink.sql.side.PredicateInfo; import org.apache.commons.lang.math.NumberUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.Type; @@ -163,7 +164,7 @@ public static KuduPredicate buildKuduPredicate(Schema schema, PredicateInfo info case "IN": case "NOT_IN": case "BETWEEN": - value = Arrays.asList(info.getCondition().split(",")).stream() + value = Arrays.asList(StringUtils.split(info.getCondition(), ",")).stream() .map(val -> KuduUtil.getValue(val.trim(), column.getType())).collect(Collectors.toList()); break; case "IS_NOT_NULL": diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index 6a525a7e7..17ec3591e 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -18,19 +18,21 @@ package com.dtstack.flink.sql.launcher.perjob; -import com.dtstack.flink.sql.option.Options; -import com.dtstack.flink.sql.util.PluginUtil; -import org.apache.commons.io.Charsets; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.client.deployment.ClusterSpecification; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.Configuration; import org.apache.flink.core.fs.Path; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.yarn.AbstractYarnClusterDescriptor; + +import com.dtstack.flink.sql.option.Options; +import com.dtstack.flink.sql.util.PluginUtil; +import org.apache.commons.io.Charsets; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.net.URLDecoder; import java.util.Arrays; import java.util.List; @@ -82,7 +84,7 @@ private static List getJarPaths(String addjarPath) { if (addjarPath.length() > 2) { addjarPath = addjarPath.substring(1,addjarPath.length()-1).replace("\"",""); } - List paths = Arrays.asList(addjarPath.split(",")); + List paths = Arrays.asList(StringUtils.split(addjarPath, ",")); return paths; } diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index b416e4820..bcde9431e 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -18,17 +18,21 @@ package com.dtstack.flink.sql.side.mongo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; + import com.dtstack.flink.sql.side.AllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; -import com.mongodb.BasicDBObject; -import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; -import com.mongodb.MongoCredential; -import com.mongodb.ServerAddress; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.mongodb.*; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; @@ -36,13 +40,6 @@ import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -184,13 +181,13 @@ private MongoCollection getConn(String address, String userName, String password MongoCollection dbCollection; try { MongoCredential credential; - String[] servers = address.split(","); + String[] servers = StringUtils.split(address, ","); String host; Integer port; String[] hostAndPort; List lists = new ArrayList<>(); for (String server : servers) { - hostAndPort = server.split(":"); + hostAndPort = StringUtils.split(server, ":"); host = hostAndPort[0]; port = Integer.parseInt(hostAndPort[1]); lists.add(new ServerAddress(host, port)); @@ -236,7 +233,7 @@ private void loadData(Map>> tmpCache) throws SQ } //load data from table - String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); BasicDBObject basicDBObject = new BasicDBObject(); for (String selectField : sideFieldNames) { basicDBObject.append(selectField, 1); diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 8ac7d83ee..ed238cc9b 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -19,6 +19,13 @@ package com.dtstack.flink.sql.side.mongo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; + import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.AsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; @@ -27,36 +34,22 @@ import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; +import com.google.common.collect.Lists; import com.mongodb.BasicDBObject; import com.mongodb.Block; import com.mongodb.MongoCredential; import com.mongodb.ServerAddress; import com.mongodb.async.SingleResultCallback; -import com.mongodb.async.client.MongoClient; -import com.mongodb.async.client.MongoClientSettings; -import com.mongodb.async.client.MongoClients; -import com.mongodb.async.client.MongoCollection; -import com.mongodb.async.client.MongoDatabase; +import com.mongodb.async.client.*; import com.mongodb.connection.ClusterSettings; import com.mongodb.connection.ConnectionPoolSettings; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicInteger; /** @@ -91,13 +84,13 @@ public void open(Configuration parameters) throws Exception { public void connMongoDB() throws Exception { MongoCredential mongoCredential; - String[] servers = MongoSideTableInfo.getAddress().split(","); + String[] servers = StringUtils.split(MongoSideTableInfo.getAddress(), ","); String host; Integer port; String[] hostAndPort; List lists = new ArrayList<>(); for (String server : servers) { - hostAndPort = server.split(":"); + hostAndPort = StringUtils.split(server, ":"); host = hostAndPort[0]; port = Integer.parseInt(hostAndPort[1]); lists.add(new ServerAddress(host, port)); diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java index 6e0177759..2f1a2092d 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java @@ -20,9 +20,9 @@ import com.dtstack.flink.sql.side.PredicateInfo; import com.mongodb.BasicDBObject; +import org.apache.commons.lang3.StringUtils; import java.util.Arrays; -import java.util.List; import java.util.stream.Collectors; /** @@ -46,11 +46,11 @@ public static BasicDBObject buildFilterObject(PredicateInfo info) { case "<>": return new BasicDBObject("$ne", info.getCondition()); case "IN": - Object[] values = Arrays.stream(info.getCondition().split(",")).map(String::trim) + Object[] values = Arrays.stream(StringUtils.split(info.getCondition(), ",")).map(String::trim) .collect(Collectors.toList()).toArray(); return new BasicDBObject("$in", values); case "NOT IN": - return new BasicDBObject("$nin", info.getCondition().split(",")); + return new BasicDBObject("$nin", StringUtils.split(info.getCondition(), ",")); case "IS NOT NULL": return new BasicDBObject("$exists", true); case "IS NULL": diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index 82bc9c15f..337fbad59 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -19,6 +19,11 @@ package com.dtstack.flink.sql.sink.mongo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.types.Row; + import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import com.mongodb.MongoClient; import com.mongodb.MongoClientOptions; @@ -28,16 +33,12 @@ import com.mongodb.client.MongoDatabase; import com.mongodb.client.result.UpdateResult; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.types.Row; import org.bson.Document; import org.bson.types.ObjectId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.IOException; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; @@ -126,13 +127,13 @@ public void close() { private void establishConnection() { try { MongoCredential credential; - String[] servers = address.split(","); + String[] servers = StringUtils.split(address, ","); String host; Integer port; String[] hostAndPort; List lists = new ArrayList<>(); for (String server : servers) { - hostAndPort = server.split(":"); + hostAndPort = StringUtils.split(server, ":"); host = hostAndPort[0]; port = Integer.parseInt(hostAndPort[1]); lists.add(new ServerAddress(host, port)); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java index 48d2407aa..72338205f 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java @@ -18,26 +18,24 @@ package com.dtstack.flink.sql.side.rdb.all; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; + import com.dtstack.flink.sql.side.AllReqRow; import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; -import org.apache.calcite.sql.JoinType; -import org.apache.commons.collections.CollectionUtils; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; +import org.apache.calcite.sql.JoinType; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Timestamp; +import java.sql.*; import java.util.Calendar; import java.util.List; import java.util.Map; @@ -197,7 +195,7 @@ private void loadData(Map>> tmpCache) throws SQ Statement statement = connection.createStatement(); statement.setFetchSize(getFetchSize()); ResultSet resultSet = statement.executeQuery(sql); - String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); while (resultSet.next()) { Map oneRow = Maps.newHashMap(); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 619d08529..a6e8501da 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -18,18 +18,15 @@ package com.dtstack.flink.sql.side.rdb.all; -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; + +import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; +import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; import java.util.Arrays; import java.util.List; @@ -55,7 +52,7 @@ public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List> tmpCache) throws SQLExcep Set keys = ((Jedis) jedis).keys(perKey); List newPerKeys = new LinkedList<>(); for (String key : keys){ - String[] splitKey = key.split(":"); + String[] splitKey = StringUtils.split(key, ":"); String newKey = splitKey[0] + ":" + splitKey[1] + ":" + splitKey[2]; newPerKeys.add(newKey); } List list = newPerKeys.stream().distinct().collect(Collectors.toList()); for(String key : list){ Map kv = Maps.newHashMap(); - String[] primaryKv = key.split(":"); + String[] primaryKv = StringUtils.split(key, ":"); kv.put(primaryKv[1], primaryKv[2]); String pattern = key + "*"; Set realKeys = ((Jedis) jedis).keys(pattern); - for (String realKey : realKeys){ - kv.put(realKey.split(":")[3], jedis.get(realKey)); + for (String realKey : realKeys) { + kv.put(StringUtils.split(realKey, ":")[3], jedis.get(realKey)); } tmpCache.put(key, kv); } @@ -183,19 +188,19 @@ private void loadData(Map> tmpCache) throws SQLExcep Set keys = keys((JedisCluster) jedis, perKey); List newPerKeys = new LinkedList<>(); for (String key : keys){ - String[] splitKey = key.split(":"); + String[] splitKey = StringUtils.split(key, ":"); String newKey = splitKey[0] + ":" + splitKey[1] + ":" + splitKey[2]; newPerKeys.add(newKey); } List list = newPerKeys.stream().distinct().collect(Collectors.toList()); for(String key : list){ Map kv = Maps.newHashMap(); - String[] primaryKv = key.split(":"); + String[] primaryKv = StringUtils.split(key, ":"); kv.put(primaryKv[1], primaryKv[2]); String pattern = key + "*"; Set realKeys = keys((JedisCluster) jedis, pattern); for (String realKey : realKeys){ - kv.put(realKey.split(":")[3], jedis.get(realKey)); + kv.put(StringUtils.split(key, ":")[3], jedis.get(realKey)); } tmpCache.put(key, kv); } @@ -230,15 +235,15 @@ private JedisCommands getJedis(RedisSideTableInfo tableInfo) { timeout = 1000; } - String[] nodes = url.split(","); - String[] firstIpPort = nodes[0].split(":"); + String[] nodes = StringUtils.split(url, ","); + String[] firstIpPort = StringUtils.split(nodes[0], ":"); String firstIp = firstIpPort[0]; String firstPort = firstIpPort[1]; Set addresses = new HashSet<>(); Set ipPorts = new HashSet<>(); for (String ipPort : nodes) { ipPorts.add(ipPort); - String[] ipPortPair = ipPort.split(":"); + String[] ipPortPair = StringUtils.split(ipPort, ":"); addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.valueOf(ipPortPair[1].trim()))); } if (timeout == 0){ diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index d20875e01..903092790 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -18,11 +18,19 @@ package com.dtstack.flink.sql.side.redis; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; + import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import io.lettuce.core.KeyValue; import io.lettuce.core.RedisClient; import io.lettuce.core.RedisFuture; @@ -31,13 +39,7 @@ import io.lettuce.core.api.async.RedisStringAsyncCommands; import io.lettuce.core.cluster.RedisClusterClient; import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; +import org.apache.commons.lang3.StringUtils; import java.util.Collections; import java.util.List; @@ -167,7 +169,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce public void accept(List> keyValues) { if (keyValues.size() != 0) { for (int i = 0; i < keyValues.size(); i++) { - String[] splitKeys = keyValues.get(i).getKey().split(":"); + String[] splitKeys = StringUtils.split(keyValues.get(i).getKey(), ":"); keyValue.put(splitKeys[1], splitKeys[2]); keyValue.put(splitKeys[3], keyValues.get(i).getValue()); } diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index 62d14cabd..4e2d54bde 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -18,15 +18,18 @@ package com.dtstack.flink.sql.sink.redis; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; -import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; + +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.*; + import java.io.Closeable; import java.io.IOException; import java.util.*; @@ -97,15 +100,15 @@ private GenericObjectPoolConfig setPoolConfig(String maxTotal, String maxIdle, S private void establishConnection() { poolConfig = setPoolConfig(maxTotal, maxIdle, minIdle); - String[] nodes = url.split(","); - String[] firstIpPort = nodes[0].split(":"); + String[] nodes = StringUtils.split(url, ","); + String[] firstIpPort = StringUtils.split(nodes[0], ":"); String firstIp = firstIpPort[0]; String firstPort = firstIpPort[1]; Set addresses = new HashSet<>(); Set ipPorts = new HashSet<>(); for (String ipPort : nodes) { ipPorts.add(ipPort); - String[] ipPortPair = ipPort.split(":"); + String[] ipPortPair = StringUtils.split(ipPort, ":"); addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.valueOf(ipPortPair[1].trim()))); } if (timeout == 0){ diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 49861bb9a..0c9b0a64a 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -24,7 +24,6 @@ import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -51,7 +50,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map primaryKeysList = Lists.newArrayList(); if (!StringUtils.isEmpty(primaryKeysStr)) { - primaryKeysList = Arrays.asList(primaryKeysStr.split(",")); + primaryKeysList = Arrays.asList(StringUtils.split(primaryKeysStr, ",")); } redisTableInfo.setPrimaryKeys(primaryKeysList); From ccc0ccde2b9f10749cdb93dd67027483856118d0 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 24 Feb 2020 14:41:23 +0800 Subject: [PATCH 034/523] extract common method --- .../com/dtstack/flink/sql/CommonProcess.java | 234 ------------------ .../java/com/dtstack/flink/sql/GetPlan.java | 80 +----- .../main/java/com/dtstack/flink/sql/Main.java | 85 +------ .../dtstack/flink/sql/exec/FlinkSQLExec.java | 6 +- .../dtstack/flink/sql/exec/ParamsInfo.java | 165 ++++++++++++ .../flink/sql/option/OptionParser.java | 4 +- 6 files changed, 183 insertions(+), 391 deletions(-) delete mode 100644 core/src/main/java/com/dtstack/flink/sql/CommonProcess.java create mode 100644 core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java diff --git a/core/src/main/java/com/dtstack/flink/sql/CommonProcess.java b/core/src/main/java/com/dtstack/flink/sql/CommonProcess.java deleted file mode 100644 index 61bc983d1..000000000 --- a/core/src/main/java/com/dtstack/flink/sql/CommonProcess.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql; - -import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.config.CalciteConfig; -import com.dtstack.flink.sql.enums.ClusterMode; -import com.dtstack.flink.sql.enums.ECacheType; -import com.dtstack.flink.sql.environment.MyLocalStreamEnvironment; -import com.dtstack.flink.sql.environment.StreamEnvConfigManager; -import com.dtstack.flink.sql.exec.FlinkSQLExec; -import com.dtstack.flink.sql.function.FunctionManager; -import com.dtstack.flink.sql.parser.CreateFuncParser; -import com.dtstack.flink.sql.parser.CreateTmpTableParser; -import com.dtstack.flink.sql.parser.InsertSqlParser; -import com.dtstack.flink.sql.parser.SqlTree; -import com.dtstack.flink.sql.side.SideSqlExec; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.sink.StreamSinkFactory; -import com.dtstack.flink.sql.source.StreamSourceFactory; -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; -import com.dtstack.flink.sql.watermarker.WaterMarkerAssigner; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Sets; -import org.apache.calcite.sql.SqlInsert; -import org.apache.calcite.sql.SqlNode; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.StreamQueryConfig; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.types.Row; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.InvocationTargetException; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - -/** - * 提取任务执行时共同的流程方法 - * Date: 2020/2/17 - * Company: www.dtstack.com - * @author maqi - */ -public class CommonProcess { - - private static final String CLASS_FILE_NAME_FMT = "class_path_%d"; - private static final Logger LOG = LoggerFactory.getLogger(Main.class); - - public static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironment tableEnv, SqlTree sqlTree, Map sideTableMap, Map registerTableCache, StreamQueryConfig queryConfig) throws Exception { - SideSqlExec sideSqlExec = new SideSqlExec(); - sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); - for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { - sideSqlExec.registerTmpTable(result, sideTableMap, tableEnv, registerTableCache); - } - - for (InsertSqlParser.SqlParseResult result : sqlTree.getExecSqlList()) { - if (LOG.isInfoEnabled()) { - LOG.info("exe-sql:\n" + result.getExecSql()); - } - boolean isSide = false; - for (String tableName : result.getTargetTableList()) { - if (sqlTree.getTmpTableMap().containsKey(tableName)) { - CreateTmpTableParser.SqlParserResult tmp = sqlTree.getTmpTableMap().get(tableName); - String realSql = DtStringUtil.replaceIgnoreQuota(result.getExecSql(), "`", ""); - - SqlNode sqlNode = org.apache.calcite.sql.parser.SqlParser.create(realSql, CalciteConfig.MYSQL_LEX_CONFIG).parseStmt(); - String tmpSql = ((SqlInsert) sqlNode).getSource().toString(); - tmp.setExecSql(tmpSql); - sideSqlExec.registerTmpTable(tmp, sideTableMap, tableEnv, registerTableCache); - } else { - for (String sourceTable : result.getSourceTableList()) { - if (sideTableMap.containsKey(sourceTable)) { - isSide = true; - break; - } - } - if (isSide) { - //sql-dimensional table contains the dimension table of execution - sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig); - } else { - FlinkSQLExec.sqlUpdate(tableEnv, result.getExecSql(), queryConfig); - if (LOG.isInfoEnabled()) { - LOG.info("exec sql: " + result.getExecSql()); - } - } - } - } - } - } - - public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrlList, TableEnvironment tableEnv) - throws IllegalAccessException, InvocationTargetException { - // udf和tableEnv须由同一个类加载器加载 - ClassLoader levelClassLoader = tableEnv.getClass().getClassLoader(); - URLClassLoader classLoader = null; - List funcList = sqlTree.getFunctionList(); - for (CreateFuncParser.SqlParserResult funcInfo : funcList) { - //classloader - if (classLoader == null) { - classLoader = ClassLoaderManager.loadExtraJar(jarUrlList, (URLClassLoader) levelClassLoader); - } - FunctionManager.registerUDF(funcInfo.getType(), funcInfo.getClassName(), funcInfo.getName(), tableEnv, classLoader); - } - } - - /** - * 向Flink注册源表和结果表，返回执行时插件包的全路径 - * @param sqlTree - * @param env - * @param tableEnv - * @param localSqlPluginPath - * @param remoteSqlPluginPath - * @param pluginLoadMode 插件加载模式 classpath or shipfile - * @param sideTableMap - * @param registerTableCache - * @return - * @throws Exception - */ - public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { - Set pluginClassPatshSets = Sets.newHashSet(); - WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); - for (TableInfo tableInfo : sqlTree.getTableInfoMap().values()) { - - if (tableInfo instanceof SourceTableInfo) { - - SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; - Table table = StreamSourceFactory.getStreamSource(sourceTableInfo, env, tableEnv, localSqlPluginPath); - tableEnv.registerTable(sourceTableInfo.getAdaptName(), table); - //Note --- parameter conversion function can not be used inside a function of the type of polymerization - //Create table in which the function is arranged only need adaptation sql - String adaptSql = sourceTableInfo.getAdaptSelectSql(); - Table adaptTable = adaptSql == null ? table : tableEnv.sqlQuery(adaptSql); - - RowTypeInfo typeInfo = new RowTypeInfo(adaptTable.getSchema().getFieldTypes(), adaptTable.getSchema().getFieldNames()); - DataStream adaptStream = tableEnv.toRetractStream(adaptTable, typeInfo) - .map((Tuple2 f0) -> { - return f0.f1; - }) - .returns(typeInfo); - - String fields = String.join(",", typeInfo.getFieldNames()); - - if (waterMarkerAssigner.checkNeedAssignWaterMarker(sourceTableInfo)) { - adaptStream = waterMarkerAssigner.assignWaterMarker(adaptStream, typeInfo, sourceTableInfo); - fields += ",ROWTIME.ROWTIME"; - } else { - fields += ",PROCTIME.PROCTIME"; - } - - Table regTable = tableEnv.fromDataStream(adaptStream, fields); - tableEnv.registerTable(tableInfo.getName(), regTable); - if (LOG.isInfoEnabled()) { - LOG.info("registe table {} success.", tableInfo.getName()); - } - registerTableCache.put(tableInfo.getName(), regTable); - - URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), SourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); - pluginClassPatshSets.add(sourceTablePathUrl); - } else if (tableInfo instanceof TargetTableInfo) { - - TableSink tableSink = StreamSinkFactory.getTableSink((TargetTableInfo) tableInfo, localSqlPluginPath); - TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); - tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); - - URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), TargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); - pluginClassPatshSets.add(sinkTablePathUrl); - } else if (tableInfo instanceof SideTableInfo) { - String sideOperator = ECacheType.ALL.name().equals(((SideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; - sideTableMap.put(tableInfo.getName(), (SideTableInfo) tableInfo); - - URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, SideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); - pluginClassPatshSets.add(sideTablePathUrl); - } else { - throw new RuntimeException("not support table type:" + tableInfo.getType()); - } - } - return pluginClassPatshSets; - } - - /** - * perjob模式将job依赖的插件包路径存储到cacheFile，在外围将插件包路径传递给jobgraph - * @param env - * @param classPathSet - */ - public static void registerPluginUrlToCachedFile(StreamExecutionEnvironment env, Set classPathSet) { - int i = 0; - for (URL url : classPathSet) { - String classFileName = String.format(CLASS_FILE_NAME_FMT, i); - env.registerCachedFile(url.getPath(), classFileName, true); - i++; - } - } - - public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperties, String deployMode) throws Exception { - StreamExecutionEnvironment env = !ClusterMode.local.name().equals(deployMode) ? - StreamExecutionEnvironment.getExecutionEnvironment() : - new MyLocalStreamEnvironment(); - - StreamEnvConfigManager.streamExecutionEnvironmentConfig(env, confProperties); - return env; - } -} diff --git a/core/src/main/java/com/dtstack/flink/sql/GetPlan.java b/core/src/main/java/com/dtstack/flink/sql/GetPlan.java index 4f3ab71d4..a9ed37a32 100644 --- a/core/src/main/java/com/dtstack/flink/sql/GetPlan.java +++ b/core/src/main/java/com/dtstack/flink/sql/GetPlan.java @@ -18,39 +18,15 @@ package com.dtstack.flink.sql; -import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.enums.ClusterMode; -import com.dtstack.flink.sql.environment.MyLocalStreamEnvironment; -import com.dtstack.flink.sql.environment.StreamEnvConfigManager; -import com.dtstack.flink.sql.option.OptionParser; -import com.dtstack.flink.sql.option.Options; -import com.dtstack.flink.sql.parser.SqlParser; -import com.dtstack.flink.sql.parser.SqlTree; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.util.PluginUtil; +import com.dtstack.flink.sql.exec.BuildProcess; +import com.dtstack.flink.sql.exec.ParamsInfo; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import org.apache.commons.io.Charsets; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.StreamQueryConfig; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.java.StreamTableEnvironment; - -import java.io.File; -import java.net.URL; -import java.net.URLDecoder; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; /** - * 获取sql任务的执行计划 + * local模式获取sql任务的执行计划 * Date: 2020/2/17 * Company: www.dtstack.com * @author maqi @@ -65,54 +41,8 @@ public class GetPlan { public static String getExecutionPlan(String[] args) { try { - Arrays.stream(args).forEach(System.out::println); - OptionParser optionParser = new OptionParser(args); - Options options = optionParser.getOptions(); - String sql = options.getSql(); - String addJarListStr = options.getAddjar(); - String localSqlPluginPath = options.getLocalSqlPluginPath(); - String deployMode = ClusterMode.local.name(); - String confProp = options.getConfProp(); - - sql = URLDecoder.decode(sql, Charsets.UTF_8.name()); - SqlParser.setLocalSqlPluginRoot(localSqlPluginPath); - - List addJarFileList = Lists.newArrayList(); - if (!Strings.isNullOrEmpty(addJarListStr)) { - addJarListStr = URLDecoder.decode(addJarListStr, Charsets.UTF_8.name()); - addJarFileList = OBJECT_MAPPER.readValue(addJarListStr, List.class); - } - - confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); - Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - StreamExecutionEnvironment env = CommonProcess.getStreamExeEnv(confProperties, deployMode); - - StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); - StreamQueryConfig streamQueryConfig = StreamEnvConfigManager.getStreamQueryConfig(tableEnv, confProperties); - - List jarUrlList = Lists.newArrayList(); - SqlTree sqlTree = SqlParser.parseSql(sql); - - //Get External jar to load - for (String addJarPath : addJarFileList) { - File tmpFile = new File(addJarPath); - jarUrlList.add(tmpFile.toURI().toURL()); - } - - Map sideTableMap = Maps.newHashMap(); - Map registerTableCache = Maps.newHashMap(); - - //register udf - CommonProcess.registerUserDefinedFunction(sqlTree, jarUrlList, tableEnv); - //register table schema - Set classPathSets = CommonProcess.registerTable(sqlTree, env, tableEnv, localSqlPluginPath, null, null, sideTableMap, registerTableCache); - // cache classPathSets - CommonProcess.registerPluginUrlToCachedFile(env, classPathSets); - - CommonProcess.sqlTranslation(localSqlPluginPath, tableEnv, sqlTree, sideTableMap, registerTableCache, streamQueryConfig); - - ((MyLocalStreamEnvironment)env).setClasspaths(ClassLoaderManager.getClassPath()); - + ParamsInfo paramsInfo = BuildProcess.parseParams(args); + StreamExecutionEnvironment env = BuildProcess.getStreamExecution(paramsInfo); String executionPlan = env.getExecutionPlan(); return getJsonStr(SUCCESS, executionPlan); } catch (Exception e) { diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index d0e9424d0..c598e869c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -19,33 +19,11 @@ package com.dtstack.flink.sql; -import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.environment.MyLocalStreamEnvironment; -import com.dtstack.flink.sql.environment.StreamEnvConfigManager; -import com.dtstack.flink.sql.option.OptionParser; -import com.dtstack.flink.sql.parser.SqlParser; -import com.dtstack.flink.sql.parser.SqlTree; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.util.PluginUtil; -import org.apache.commons.io.Charsets; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.StreamQueryConfig; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.java.StreamTableEnvironment; -import java.io.File; -import java.net.URL; -import java.net.URLDecoder; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; +import com.dtstack.flink.sql.exec.BuildProcess; +import com.dtstack.flink.sql.exec.ParamsInfo; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import com.dtstack.flink.sql.option.Options; /** * Date: 2018/6/26 @@ -55,60 +33,9 @@ public class Main { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public static void main(String[] args) throws Exception { - - OptionParser optionParser = new OptionParser(args); - Options options = optionParser.getOptions(); - String sql = options.getSql(); - String name = options.getName(); - String addJarListStr = options.getAddjar(); - String localSqlPluginPath = options.getLocalSqlPluginPath(); - String remoteSqlPluginPath = options.getRemoteSqlPluginPath(); - String pluginLoadMode = options.getPluginLoadMode(); - String deployMode = options.getMode(); - String confProp = options.getConfProp(); - - sql = URLDecoder.decode(sql, Charsets.UTF_8.name()); - SqlParser.setLocalSqlPluginRoot(localSqlPluginPath); - - List addJarFileList = Lists.newArrayList(); - if (!Strings.isNullOrEmpty(addJarListStr)) { - addJarListStr = URLDecoder.decode(addJarListStr, Charsets.UTF_8.name()); - addJarFileList = OBJECT_MAPPER.readValue(addJarListStr, List.class); - } - - confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); - Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - StreamExecutionEnvironment env = CommonProcess.getStreamExeEnv(confProperties, deployMode); - StreamTableEnvironment tableEnv = StreamTableEnvironment.getTableEnvironment(env); - StreamQueryConfig streamQueryConfig = StreamEnvConfigManager.getStreamQueryConfig(tableEnv, confProperties); - - List jarURList = Lists.newArrayList(); - SqlTree sqlTree = SqlParser.parseSql(sql); - - //Get External jar to load - for (String addJarPath : addJarFileList) { - File tmpFile = new File(addJarPath); - jarURList.add(tmpFile.toURI().toURL()); - } - - Map sideTableMap = Maps.newHashMap(); - Map registerTableCache = Maps.newHashMap(); - - //register udf - CommonProcess.registerUserDefinedFunction(sqlTree, jarURList, tableEnv); - //register table schema - Set classPathSets = CommonProcess.registerTable(sqlTree, env, tableEnv, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode, sideTableMap, registerTableCache); - // cache classPathSets - CommonProcess.registerPluginUrlToCachedFile(env, classPathSets); - - CommonProcess.sqlTranslation(localSqlPluginPath, tableEnv, sqlTree, sideTableMap, registerTableCache, streamQueryConfig); - - if (env instanceof MyLocalStreamEnvironment) { - ((MyLocalStreamEnvironment) env).setClasspaths(ClassLoaderManager.getClassPath()); - } - env.execute(name); + ParamsInfo paramsInfo = BuildProcess.parseParams(args); + StreamExecutionEnvironment env = BuildProcess.getStreamExecution(paramsInfo); + env.execute(paramsInfo.getName()); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/FlinkSQLExec.java b/core/src/main/java/com/dtstack/flink/sql/exec/FlinkSQLExec.java index 3834242bd..2b533fb5e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/FlinkSQLExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/FlinkSQLExec.java @@ -21,7 +21,11 @@ import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlInsert; import org.apache.calcite.sql.SqlNode; -import org.apache.flink.table.api.*; +import org.apache.flink.table.api.StreamQueryConfig; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableException; +import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.table.calcite.FlinkPlannerImpl; import org.apache.flink.table.plan.logical.LogicalRelNode; diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java new file mode 100644 index 000000000..32238b3e0 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.exec; + + +import java.net.URL; +import java.util.List; +import java.util.Properties; + +/** + * 解析传递的参数信息 + * Date: 2020/2/24 + * Company: www.dtstack.com + * @author maqi + */ +public class ParamsInfo { + + private String sql; + private String name; + private List jarUrlList; + private String localSqlPluginPath; + private String remoteSqlPluginPath; + private String pluginLoadMode; + private String deployMode; + private Properties confProp; + + public ParamsInfo(String sql, String name, List jarUrlList, String localSqlPluginPath, + String remoteSqlPluginPath, String pluginLoadMode, String deployMode, Properties confProp) { + this.sql = sql; + this.name = name; + this.jarUrlList = jarUrlList; + this.localSqlPluginPath = localSqlPluginPath; + this.remoteSqlPluginPath = remoteSqlPluginPath; + this.pluginLoadMode = pluginLoadMode; + this.deployMode = deployMode; + this.confProp = confProp; + } + + public String getSql() { + return sql; + } + + public String getName() { + return name; + } + + public List getJarUrlList() { + return jarUrlList; + } + + public String getLocalSqlPluginPath() { + return localSqlPluginPath; + } + + public String getRemoteSqlPluginPath() { + return remoteSqlPluginPath; + } + + public String getPluginLoadMode() { + return pluginLoadMode; + } + + public String getDeployMode() { + return deployMode; + } + + public Properties getConfProp() { + return confProp; + } + + @Override + public String toString() { + return "ParamsInfo{" + + "sql='" + sql + '\'' + + ", name='" + name + '\'' + + ", jarUrlList=" + convertJarUrlListToString(jarUrlList) + + ", localSqlPluginPath='" + localSqlPluginPath + '\'' + + ", remoteSqlPluginPath='" + remoteSqlPluginPath + '\'' + + ", pluginLoadMode='" + pluginLoadMode + '\'' + + ", deployMode='" + deployMode + '\'' + + ", confProp=" + confProp + + '}'; + } + + public String convertJarUrlListToString(List jarUrlList) { + return jarUrlList.stream().map(URL::toString).reduce((pre, last) -> pre + last).get(); + } + + public static ParamsInfo.Builder builder() { + return new ParamsInfo.Builder(); + } + public static class Builder { + + private String sql; + private String name; + private List jarUrlList; + private String localSqlPluginPath; + private String remoteSqlPluginPath; + private String pluginLoadMode; + private String deployMode; + private Properties confProp; + + + public ParamsInfo.Builder setSql(String sql) { + this.sql = sql; + return this; + } + + public ParamsInfo.Builder setName(String name) { + this.name = name; + return this; + } + + public ParamsInfo.Builder setJarUrlList(List jarUrlList) { + this.jarUrlList = jarUrlList; + return this; + } + + public ParamsInfo.Builder setLocalSqlPluginPath(String localSqlPluginPath) { + this.localSqlPluginPath = localSqlPluginPath; + return this; + } + + public ParamsInfo.Builder setRemoteSqlPluginPath(String remoteSqlPluginPath) { + this.remoteSqlPluginPath = remoteSqlPluginPath; + return this; + } + + public ParamsInfo.Builder setPluginLoadMode(String pluginLoadMode) { + this.pluginLoadMode = pluginLoadMode; + return this; + } + + public ParamsInfo.Builder setDeployMode(String deployMode) { + this.deployMode = deployMode; + return this; + } + + public ParamsInfo.Builder setConfProp(Properties confProp) { + this.confProp = confProp; + return this; + } + + public ParamsInfo build() { + return new ParamsInfo(sql, name, jarUrlList, localSqlPluginPath, + remoteSqlPluginPath, pluginLoadMode, deployMode, confProp); + } + } +} diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java index 07860b608..43f599d14 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java @@ -20,8 +20,8 @@ import com.google.common.collect.Lists; import com.dtstack.flink.sql.util.PluginUtil; -import org.apache.commons.cli.BasicParser; import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.ParseException; import org.apache.commons.lang.StringUtils; import java.lang.reflect.InvocationTargetException; @@ -46,7 +46,7 @@ public class OptionParser { private org.apache.commons.cli.Options options = new org.apache.commons.cli.Options(); - private BasicParser parser = new BasicParser(); + private DefaultParser parser = new DefaultParser(); private Options properties = new Options(); From bfa6ac00c96306417edeb0b3c33c6d3d5e496ecc Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 24 Feb 2020 14:53:44 +0800 Subject: [PATCH 035/523] add file --- .../dtstack/flink/sql/exec/BuildProcess.java | 323 ++++++++++++++++++ 1 file changed, 323 insertions(+) create mode 100644 core/src/main/java/com/dtstack/flink/sql/exec/BuildProcess.java diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/BuildProcess.java b/core/src/main/java/com/dtstack/flink/sql/exec/BuildProcess.java new file mode 100644 index 000000000..c12239f66 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/exec/BuildProcess.java @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.exec; + +import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.dtstack.flink.sql.config.CalciteConfig; +import com.dtstack.flink.sql.enums.ClusterMode; +import com.dtstack.flink.sql.enums.ECacheType; +import com.dtstack.flink.sql.environment.MyLocalStreamEnvironment; +import com.dtstack.flink.sql.environment.StreamEnvConfigManager; +import com.dtstack.flink.sql.function.FunctionManager; +import com.dtstack.flink.sql.option.OptionParser; +import com.dtstack.flink.sql.option.Options; +import com.dtstack.flink.sql.parser.CreateFuncParser; +import com.dtstack.flink.sql.parser.CreateTmpTableParser; +import com.dtstack.flink.sql.parser.InsertSqlParser; +import com.dtstack.flink.sql.parser.SqlParser; +import com.dtstack.flink.sql.parser.SqlTree; +import com.dtstack.flink.sql.side.SideSqlExec; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.sink.StreamSinkFactory; +import com.dtstack.flink.sql.source.StreamSourceFactory; +import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.util.PluginUtil; +import com.dtstack.flink.sql.watermarker.WaterMarkerAssigner; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.calcite.sql.SqlInsert; +import org.apache.calcite.sql.SqlNode; +import org.apache.commons.io.Charsets; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.StreamQueryConfig; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.types.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.lang.reflect.InvocationTargetException; +import java.net.URL; +import java.net.URLClassLoader; +import java.net.URLDecoder; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +/** + * 提取任务执行时共同的流程方法 + * Date: 2020/2/17 + * Company: www.dtstack.com + * @author maqi + */ +public class BuildProcess { + + private static final String CLASS_FILE_NAME_FMT = "class_path_%d"; + private static final Logger LOG = LoggerFactory.getLogger(BuildProcess.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + + public static ParamsInfo parseParams(String[] args) throws Exception { + LOG.info("------------program params-------------------------"); + System.out.println("------------program params-------------------------"); + Arrays.stream(args).forEach(arg -> LOG.info("{}", arg)); + Arrays.stream(args).forEach(System.out::println); + LOG.info("-------------------------------------------"); + System.out.println("----------------------------------------"); + + OptionParser optionParser = new OptionParser(args); + Options options = optionParser.getOptions(); + + String sql = URLDecoder.decode(options.getSql(), Charsets.UTF_8.name()); + String name = options.getName(); + String localSqlPluginPath = options.getLocalSqlPluginPath(); + String remoteSqlPluginPath = options.getRemoteSqlPluginPath(); + String pluginLoadMode = options.getPluginLoadMode(); + String deployMode = options.getMode(); + + String confProp = URLDecoder.decode(options.getConfProp(), Charsets.UTF_8.toString()); + Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); + + List jarURList = getExternalJarUrls(options.getAddjar()); + + return ParamsInfo.builder() + .setSql(sql) + .setName(name) + .setLocalSqlPluginPath(localSqlPluginPath) + .setRemoteSqlPluginPath(remoteSqlPluginPath) + .setPluginLoadMode(pluginLoadMode) + .setDeployMode(deployMode) + .setConfProp(confProperties) + .setJarUrlList(jarURList) + .build(); + + } + + public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInfo) throws Exception { + StreamExecutionEnvironment env = BuildProcess.getStreamExeEnv(paramsInfo.getConfProp(), paramsInfo.getDeployMode()); + StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); + StreamQueryConfig streamQueryConfig = StreamEnvConfigManager.getStreamQueryConfig(tableEnv, paramsInfo.getConfProp()); + + SqlParser.setLocalSqlPluginRoot(paramsInfo.getLocalSqlPluginPath()); + SqlTree sqlTree = SqlParser.parseSql(paramsInfo.getSql()); + + Map sideTableMap = Maps.newHashMap(); + Map registerTableCache = Maps.newHashMap(); + + //register udf + BuildProcess.registerUserDefinedFunction(sqlTree, paramsInfo.getJarUrlList(), tableEnv); + //register table schema + Set classPathSets = BuildProcess.registerTable(sqlTree, env, tableEnv, paramsInfo.getLocalSqlPluginPath(), + paramsInfo.getRemoteSqlPluginPath(), paramsInfo.getPluginLoadMode(), sideTableMap, registerTableCache); + // cache classPathSets + BuildProcess.registerPluginUrlToCachedFile(env, classPathSets); + + BuildProcess.sqlTranslation(paramsInfo.getLocalSqlPluginPath(), tableEnv, sqlTree, sideTableMap, registerTableCache, streamQueryConfig); + + if (env instanceof MyLocalStreamEnvironment) { + ((MyLocalStreamEnvironment) env).setClasspaths(ClassLoaderManager.getClassPath()); + } + return env; + } + + + public static List getExternalJarUrls(String addJarListStr) throws java.io.IOException { + List jarUrlList = Lists.newArrayList(); + if (Strings.isNullOrEmpty(addJarListStr)) { + return jarUrlList; + } + + List addJarFileList = OBJECT_MAPPER.readValue(URLDecoder.decode(addJarListStr, Charsets.UTF_8.name()), List.class); + //Get External jar to load + for (String addJarPath : addJarFileList) { + jarUrlList.add(new File(addJarPath).toURI().toURL()); + } + return jarUrlList; + } + + public static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironment tableEnv, SqlTree sqlTree, Map sideTableMap, Map registerTableCache, StreamQueryConfig queryConfig) throws Exception { + SideSqlExec sideSqlExec = new SideSqlExec(); + sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); + for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { + sideSqlExec.registerTmpTable(result, sideTableMap, tableEnv, registerTableCache); + } + + for (InsertSqlParser.SqlParseResult result : sqlTree.getExecSqlList()) { + if (LOG.isInfoEnabled()) { + LOG.info("exe-sql:\n" + result.getExecSql()); + } + boolean isSide = false; + for (String tableName : result.getTargetTableList()) { + if (sqlTree.getTmpTableMap().containsKey(tableName)) { + CreateTmpTableParser.SqlParserResult tmp = sqlTree.getTmpTableMap().get(tableName); + String realSql = DtStringUtil.replaceIgnoreQuota(result.getExecSql(), "`", ""); + + SqlNode sqlNode = org.apache.calcite.sql.parser.SqlParser.create(realSql, CalciteConfig.MYSQL_LEX_CONFIG).parseStmt(); + String tmpSql = ((SqlInsert) sqlNode).getSource().toString(); + tmp.setExecSql(tmpSql); + sideSqlExec.registerTmpTable(tmp, sideTableMap, tableEnv, registerTableCache); + } else { + for (String sourceTable : result.getSourceTableList()) { + if (sideTableMap.containsKey(sourceTable)) { + isSide = true; + break; + } + } + if (isSide) { + //sql-dimensional table contains the dimension table of execution + sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig); + } else { + FlinkSQLExec.sqlUpdate(tableEnv, result.getExecSql(), queryConfig); + if (LOG.isInfoEnabled()) { + LOG.info("exec sql: " + result.getExecSql()); + } + } + } + } + } + } + + public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrlList, TableEnvironment tableEnv) + throws IllegalAccessException, InvocationTargetException { + // udf和tableEnv须由同一个类加载器加载 + ClassLoader levelClassLoader = tableEnv.getClass().getClassLoader(); + URLClassLoader classLoader = null; + List funcList = sqlTree.getFunctionList(); + for (CreateFuncParser.SqlParserResult funcInfo : funcList) { + //classloader + if (classLoader == null) { + classLoader = ClassLoaderManager.loadExtraJar(jarUrlList, (URLClassLoader) levelClassLoader); + } + FunctionManager.registerUDF(funcInfo.getType(), funcInfo.getClassName(), funcInfo.getName(), tableEnv, classLoader); + } + } + + /** + * 向Flink注册源表和结果表，返回执行时插件包的全路径 + * @param sqlTree + * @param env + * @param tableEnv + * @param localSqlPluginPath + * @param remoteSqlPluginPath + * @param pluginLoadMode 插件加载模式 classpath or shipfile + * @param sideTableMap + * @param registerTableCache + * @return + * @throws Exception + */ + public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, + String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { + Set pluginClassPatshSets = Sets.newHashSet(); + WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); + for (TableInfo tableInfo : sqlTree.getTableInfoMap().values()) { + + if (tableInfo instanceof SourceTableInfo) { + + SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; + Table table = StreamSourceFactory.getStreamSource(sourceTableInfo, env, tableEnv, localSqlPluginPath); + tableEnv.registerTable(sourceTableInfo.getAdaptName(), table); + //Note --- parameter conversion function can not be used inside a function of the type of polymerization + //Create table in which the function is arranged only need adaptation sql + String adaptSql = sourceTableInfo.getAdaptSelectSql(); + Table adaptTable = adaptSql == null ? table : tableEnv.sqlQuery(adaptSql); + + RowTypeInfo typeInfo = new RowTypeInfo(adaptTable.getSchema().getFieldTypes(), adaptTable.getSchema().getFieldNames()); + DataStream adaptStream = tableEnv.toRetractStream(adaptTable, typeInfo) + .map((Tuple2 f0) -> { + return f0.f1; + }) + .returns(typeInfo); + + String fields = String.join(",", typeInfo.getFieldNames()); + + if (waterMarkerAssigner.checkNeedAssignWaterMarker(sourceTableInfo)) { + adaptStream = waterMarkerAssigner.assignWaterMarker(adaptStream, typeInfo, sourceTableInfo); + fields += ",ROWTIME.ROWTIME"; + } else { + fields += ",PROCTIME.PROCTIME"; + } + + Table regTable = tableEnv.fromDataStream(adaptStream, fields); + tableEnv.registerTable(tableInfo.getName(), regTable); + if (LOG.isInfoEnabled()) { + LOG.info("registe table {} success.", tableInfo.getName()); + } + registerTableCache.put(tableInfo.getName(), regTable); + + URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), SourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + pluginClassPatshSets.add(sourceTablePathUrl); + } else if (tableInfo instanceof TargetTableInfo) { + + TableSink tableSink = StreamSinkFactory.getTableSink((TargetTableInfo) tableInfo, localSqlPluginPath); + TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); + tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); + + URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), TargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + pluginClassPatshSets.add(sinkTablePathUrl); + } else if (tableInfo instanceof SideTableInfo) { + String sideOperator = ECacheType.ALL.name().equals(((SideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; + sideTableMap.put(tableInfo.getName(), (SideTableInfo) tableInfo); + + URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, SideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + pluginClassPatshSets.add(sideTablePathUrl); + } else { + throw new RuntimeException("not support table type:" + tableInfo.getType()); + } + } + return pluginClassPatshSets; + } + + /** + * perjob模式将job依赖的插件包路径存储到cacheFile，在外围将插件包路径传递给jobgraph + * @param env + * @param classPathSet + */ + public static void registerPluginUrlToCachedFile(StreamExecutionEnvironment env, Set classPathSet) { + int i = 0; + for (URL url : classPathSet) { + String classFileName = String.format(CLASS_FILE_NAME_FMT, i); + env.registerCachedFile(url.getPath(), classFileName, true); + i++; + } + } + + public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperties, String deployMode) throws Exception { + StreamExecutionEnvironment env = !ClusterMode.local.name().equals(deployMode) ? + StreamExecutionEnvironment.getExecutionEnvironment() : + new MyLocalStreamEnvironment(); + + StreamEnvConfigManager.streamExecutionEnvironmentConfig(env, confProperties); + return env; + } +} \ No newline at end of file From b24aa113885019496c75c1093098aa210cca36f3 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Mon, 24 Feb 2020 15:03:02 +0800 Subject: [PATCH 036/523] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E6=9B=BF=E6=8D=A2?= =?UTF-8?q?=E6=97=B6=E7=9A=84=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/side/impala/table/ImpalaSideParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java index 1ae837763..b6a54ca3f 100644 --- a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java +++ b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java @@ -96,7 +96,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Mon, 24 Feb 2020 16:32:27 +0800 Subject: [PATCH 037/523] rename --- .../java/com/dtstack/flink/sql/GetPlan.java | 26 ++---- .../main/java/com/dtstack/flink/sql/Main.java | 10 +- .../com/dtstack/flink/sql/exec/ApiResult.java | 91 +++++++++++++++++++ ...Process.java => ExecuteProcessHelper.java} | 16 ++-- .../dtstack/flink/sql/exec/ParamsInfo.java | 2 +- .../com/dtstack/flink/sql/TestGetPlan.java | 3 +- 6 files changed, 115 insertions(+), 33 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java rename core/src/main/java/com/dtstack/flink/sql/exec/{BuildProcess.java => ExecuteProcessHelper.java} (95%) diff --git a/core/src/main/java/com/dtstack/flink/sql/GetPlan.java b/core/src/main/java/com/dtstack/flink/sql/GetPlan.java index a9ed37a32..07c8b3c2f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/GetPlan.java +++ b/core/src/main/java/com/dtstack/flink/sql/GetPlan.java @@ -18,10 +18,9 @@ package com.dtstack.flink.sql; -import com.dtstack.flink.sql.exec.BuildProcess; +import com.dtstack.flink.sql.exec.ApiResult; +import com.dtstack.flink.sql.exec.ExecuteProcessHelper; import com.dtstack.flink.sql.exec.ParamsInfo; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -32,28 +31,15 @@ * @author maqi */ public class GetPlan { - public static final String STATUS_KEY = "status"; - public static final String MSG_KEY = "msg"; - public static final Integer FAIL = 0; - public static final Integer SUCCESS = 1; - - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); public static String getExecutionPlan(String[] args) { try { - ParamsInfo paramsInfo = BuildProcess.parseParams(args); - StreamExecutionEnvironment env = BuildProcess.getStreamExecution(paramsInfo); + ParamsInfo paramsInfo = ExecuteProcessHelper.parseParams(args); + StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExecution(paramsInfo); String executionPlan = env.getExecutionPlan(); - return getJsonStr(SUCCESS, executionPlan); + return ApiResult.createSuccessResultJsonStr(executionPlan); } catch (Exception e) { - return getJsonStr(FAIL, ExceptionUtils.getFullStackTrace(e)); + return ApiResult.createErrorResultJsonStr(ExceptionUtils.getFullStackTrace(e)); } } - - public static String getJsonStr(int status, String msg) { - ObjectNode objectNode = OBJECT_MAPPER.createObjectNode(); - objectNode.put(STATUS_KEY, status); - objectNode.put(MSG_KEY, msg); - return objectNode.toString(); - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index c598e869c..1131e97c6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -20,9 +20,11 @@ package com.dtstack.flink.sql; -import com.dtstack.flink.sql.exec.BuildProcess; +import com.dtstack.flink.sql.exec.ExecuteProcessHelper; import com.dtstack.flink.sql.exec.ParamsInfo; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** @@ -32,10 +34,12 @@ */ public class Main { + private static final Logger LOG = LoggerFactory.getLogger(Main.class); public static void main(String[] args) throws Exception { - ParamsInfo paramsInfo = BuildProcess.parseParams(args); - StreamExecutionEnvironment env = BuildProcess.getStreamExecution(paramsInfo); + ParamsInfo paramsInfo = ExecuteProcessHelper.parseParams(args); + StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExecution(paramsInfo); env.execute(paramsInfo.getName()); + LOG.info("program {} execution success", paramsInfo.getName()); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java b/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java new file mode 100644 index 000000000..f38bc37d1 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java @@ -0,0 +1,91 @@ +package com.dtstack.flink.sql.exec; + +import org.codehaus.jackson.map.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.UUID; + +/** + * API调用结果返回 + * Date: 2020/2/24 + * Company: www.dtstack.com + * @author maqi + */ +public class ApiResult { + + private static final Logger LOG = LoggerFactory.getLogger(ApiResult.class); + private static final ObjectMapper objectMapper = new ObjectMapper(); + + public static final Integer FAIL = 0; + public static final Integer SUCCESS = 1; + + private int code; + private Object data; + private String errorMsg; + + public ApiResult() { + } + + public static String createSuccessResultJsonStr(String message) { + ApiResult apiResult = createSuccessResult(SUCCESS, message); + String result; + try { + result = objectMapper.writeValueAsString(apiResult); + } catch (Exception e) { + LOG.error("", e); + result = "code:" + SUCCESS + ",message:" + message; + } + return result; + } + + public static ApiResult createSuccessResult(int code, String message) { + ApiResult apiResult = new ApiResult(); + apiResult.setCode(code); + apiResult.setData(message); + return apiResult; + } + + public static String createErrorResultJsonStr(String message) { + ApiResult apiResult = createErrorResult(message, FAIL); + String result; + try { + result = objectMapper.writeValueAsString(apiResult); + } catch (Exception e) { + LOG.error("", e); + result = "code:" + FAIL + ",message:" + message; + } + return result; + } + + public static ApiResult createErrorResult(String errMsg, int code) { + ApiResult apiResult = new ApiResult(); + apiResult.setCode(code); + apiResult.setErrorMsg(errMsg); + return apiResult; + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public Object getData() { + return data; + } + + public void setData(Object data) { + this.data = data; + } + + public String getErrorMsg() { + return errorMsg; + } + + public void setErrorMsg(String errorMsg) { + this.errorMsg = errorMsg; + } +} diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/BuildProcess.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java similarity index 95% rename from core/src/main/java/com/dtstack/flink/sql/exec/BuildProcess.java rename to core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index c12239f66..c1b8e39ef 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/BuildProcess.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -76,15 +76,15 @@ import java.util.Set; /** - * 提取任务执行时共同的流程方法 + * 任务执行时的流程方法 * Date: 2020/2/17 * Company: www.dtstack.com * @author maqi */ -public class BuildProcess { +public class ExecuteProcessHelper { private static final String CLASS_FILE_NAME_FMT = "class_path_%d"; - private static final Logger LOG = LoggerFactory.getLogger(BuildProcess.class); + private static final Logger LOG = LoggerFactory.getLogger(ExecuteProcessHelper.class); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -125,7 +125,7 @@ public static ParamsInfo parseParams(String[] args) throws Exception { } public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInfo) throws Exception { - StreamExecutionEnvironment env = BuildProcess.getStreamExeEnv(paramsInfo.getConfProp(), paramsInfo.getDeployMode()); + StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExeEnv(paramsInfo.getConfProp(), paramsInfo.getDeployMode()); StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); StreamQueryConfig streamQueryConfig = StreamEnvConfigManager.getStreamQueryConfig(tableEnv, paramsInfo.getConfProp()); @@ -136,14 +136,14 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf Map registerTableCache = Maps.newHashMap(); //register udf - BuildProcess.registerUserDefinedFunction(sqlTree, paramsInfo.getJarUrlList(), tableEnv); + ExecuteProcessHelper.registerUserDefinedFunction(sqlTree, paramsInfo.getJarUrlList(), tableEnv); //register table schema - Set classPathSets = BuildProcess.registerTable(sqlTree, env, tableEnv, paramsInfo.getLocalSqlPluginPath(), + Set classPathSets = ExecuteProcessHelper.registerTable(sqlTree, env, tableEnv, paramsInfo.getLocalSqlPluginPath(), paramsInfo.getRemoteSqlPluginPath(), paramsInfo.getPluginLoadMode(), sideTableMap, registerTableCache); // cache classPathSets - BuildProcess.registerPluginUrlToCachedFile(env, classPathSets); + ExecuteProcessHelper.registerPluginUrlToCachedFile(env, classPathSets); - BuildProcess.sqlTranslation(paramsInfo.getLocalSqlPluginPath(), tableEnv, sqlTree, sideTableMap, registerTableCache, streamQueryConfig); + ExecuteProcessHelper.sqlTranslation(paramsInfo.getLocalSqlPluginPath(), tableEnv, sqlTree, sideTableMap, registerTableCache, streamQueryConfig); if (env instanceof MyLocalStreamEnvironment) { ((MyLocalStreamEnvironment) env).setClasspaths(ClassLoaderManager.getClassPath()); diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java index 32238b3e0..0c98a4460 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java @@ -99,7 +99,7 @@ public String toString() { } public String convertJarUrlListToString(List jarUrlList) { - return jarUrlList.stream().map(URL::toString).reduce((pre, last) -> pre + last).get(); + return jarUrlList.stream().map(URL::toString).reduce((pre, last) -> pre + last).orElse(""); } public static ParamsInfo.Builder builder() { diff --git a/core/src/test/java/com/dtstack/flink/sql/TestGetPlan.java b/core/src/test/java/com/dtstack/flink/sql/TestGetPlan.java index ed8190887..2ae5ee812 100644 --- a/core/src/test/java/com/dtstack/flink/sql/TestGetPlan.java +++ b/core/src/test/java/com/dtstack/flink/sql/TestGetPlan.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql; import com.dtstack.flink.sql.classloader.DtClassLoader; +import com.dtstack.flink.sql.exec.ApiResult; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.Assert; @@ -69,7 +70,7 @@ public void testGetExecutionPlan() throws Exception { String jsonStr = (String) getExecutionPlan.invoke(aClass.newInstance(), (Object)params); ObjectNode jsonNodes = OBJECT_MAPPER.readValue(jsonStr, ObjectNode.class); - Assert.assertEquals(jsonNodes.get(GetPlan.STATUS_KEY).asLong(), GetPlan.SUCCESS.longValue()); + Assert.assertEquals(jsonNodes.get("code").asLong(), ApiResult.SUCCESS.longValue()); } From 22bcea576694b4fa47009b73d458760ce349f900 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 24 Feb 2020 16:43:47 +0800 Subject: [PATCH 038/523] kudu code scan --- .../java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java | 3 --- .../java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java | 3 --- .../com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java | 1 + .../java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java | 1 + .../com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java | 1 + 5 files changed, 3 insertions(+), 6 deletions(-) diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 047665928..cf2c70f1f 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -227,9 +227,6 @@ private KuduScanner getConn(KuduSideTableInfo tableInfo) { if (null != workerCount) { kuduClientBuilder.workerCount(workerCount); } - if (null != defaultSocketReadTimeoutMs) { - kuduClientBuilder.defaultSocketReadTimeoutMs(defaultSocketReadTimeoutMs); - } if (null != defaultOperationTimeoutMs) { kuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index 4a7457418..12689a07b 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -82,9 +82,6 @@ private void connKuDu() throws KuduException { if (null != workerCount) { asyncKuduClientBuilder.workerCount(workerCount); } - if (null != defaultSocketReadTimeoutMs) { - asyncKuduClientBuilder.defaultSocketReadTimeoutMs(defaultSocketReadTimeoutMs); - } if (null != defaultOperationTimeoutMs) { asyncKuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java index c03c470a7..83864fa22 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java @@ -107,6 +107,7 @@ public Class dbTypeConvertToJavaType(String fieldType) { return BigDecimal.class; case "binary": return byte[].class; + default: } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java index c5cd8cdb2..91c94a2b2 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java @@ -190,6 +190,7 @@ public static KuduPredicate buildKuduPredicate(Schema schema, PredicateInfo info return KuduPredicate.newIsNotNullPredicate(column); case "IS NULL": return KuduPredicate.newIsNullPredicate(column); + default: } return null; diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java index ea61bd325..e38a85390 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java @@ -86,6 +86,7 @@ public Class dbTypeConvertToJavaType(String fieldType) { return BigDecimal.class; case "binary": return byte[].class; + default: } throw new RuntimeException("不支持 " + fieldType + " 类型"); From af856295567061b25b48da4b978d5d10ea3be8e2 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 24 Feb 2020 18:21:26 +0800 Subject: [PATCH 039/523] remote plugin path not required --- .../java/com/dtstack/flink/sql/GetPlan.java | 4 +++- .../com/dtstack/flink/sql/exec/ApiResult.java | 18 ++++++++++++---- .../flink/sql/exec/ExecuteProcessHelper.java | 21 +++++++++++++++++++ .../com/dtstack/flink/sql/option/Options.java | 2 +- .../com/dtstack/flink/sql/TestGetPlan.java | 5 ----- 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/GetPlan.java b/core/src/main/java/com/dtstack/flink/sql/GetPlan.java index 07c8b3c2f..ba6518b3d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/GetPlan.java +++ b/core/src/main/java/com/dtstack/flink/sql/GetPlan.java @@ -34,10 +34,12 @@ public class GetPlan { public static String getExecutionPlan(String[] args) { try { + long start = System.currentTimeMillis(); ParamsInfo paramsInfo = ExecuteProcessHelper.parseParams(args); StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExecution(paramsInfo); String executionPlan = env.getExecutionPlan(); - return ApiResult.createSuccessResultJsonStr(executionPlan); + long end = System.currentTimeMillis(); + return ApiResult.createSuccessResultJsonStr(executionPlan, end - start); } catch (Exception e) { return ApiResult.createErrorResultJsonStr(ExceptionUtils.getFullStackTrace(e)); } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java b/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java index f38bc37d1..2da763988 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java @@ -15,23 +15,25 @@ public class ApiResult { private static final Logger LOG = LoggerFactory.getLogger(ApiResult.class); - private static final ObjectMapper objectMapper = new ObjectMapper(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); public static final Integer FAIL = 0; public static final Integer SUCCESS = 1; private int code; + private long space; private Object data; private String errorMsg; public ApiResult() { } - public static String createSuccessResultJsonStr(String message) { + public static String createSuccessResultJsonStr(String message,long space) { ApiResult apiResult = createSuccessResult(SUCCESS, message); + apiResult.setSpace(space); String result; try { - result = objectMapper.writeValueAsString(apiResult); + result = OBJECT_MAPPER.writeValueAsString(apiResult); } catch (Exception e) { LOG.error("", e); result = "code:" + SUCCESS + ",message:" + message; @@ -50,7 +52,7 @@ public static String createErrorResultJsonStr(String message) { ApiResult apiResult = createErrorResult(message, FAIL); String result; try { - result = objectMapper.writeValueAsString(apiResult); + result = OBJECT_MAPPER.writeValueAsString(apiResult); } catch (Exception e) { LOG.error("", e); result = "code:" + FAIL + ",message:" + message; @@ -88,4 +90,12 @@ public String getErrorMsg() { public void setErrorMsg(String errorMsg) { this.errorMsg = errorMsg; } + + public long getSpace() { + return space; + } + + public void setSpace(long space) { + this.space = space; + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index c1b8e39ef..5320821b5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -22,6 +22,7 @@ import com.dtstack.flink.sql.config.CalciteConfig; import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.enums.ECacheType; +import com.dtstack.flink.sql.enums.EPluginLoadMode; import com.dtstack.flink.sql.environment.MyLocalStreamEnvironment; import com.dtstack.flink.sql.environment.StreamEnvConfigManager; import com.dtstack.flink.sql.function.FunctionManager; @@ -43,6 +44,7 @@ import com.dtstack.flink.sql.util.PluginUtil; import com.dtstack.flink.sql.watermarker.WaterMarkerAssigner; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -50,6 +52,7 @@ import org.apache.calcite.sql.SqlInsert; import org.apache.calcite.sql.SqlNode; import org.apache.commons.io.Charsets; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -106,6 +109,8 @@ public static ParamsInfo parseParams(String[] args) throws Exception { String pluginLoadMode = options.getPluginLoadMode(); String deployMode = options.getMode(); + Preconditions.checkArgument(checkRemoteSqlPluginPath(remoteSqlPluginPath, deployMode, pluginLoadMode), + "Non-local mode or shipfile deployment mode, remoteSqlPluginPath is required"); String confProp = URLDecoder.decode(options.getConfProp(), Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); @@ -124,6 +129,22 @@ public static ParamsInfo parseParams(String[] args) throws Exception { } + /** + * 非local模式或者shipfile部署模式，remoteSqlPluginPath必填 + * @param remoteSqlPluginPath + * @param deployMode + * @param pluginLoadMode + * @return + */ + public static boolean checkRemoteSqlPluginPath(String remoteSqlPluginPath, String deployMode, String pluginLoadMode) { + if (StringUtils.isEmpty(remoteSqlPluginPath)) { + return StringUtils.equalsIgnoreCase(pluginLoadMode, EPluginLoadMode.SHIPFILE.name()) + || StringUtils.equalsIgnoreCase(deployMode, ClusterMode.local.name()); + } + return true; + } + + public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInfo) throws Exception { StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExeEnv(paramsInfo.getConfProp(), paramsInfo.getDeployMode()); StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); diff --git a/core/src/main/java/com/dtstack/flink/sql/option/Options.java b/core/src/main/java/com/dtstack/flink/sql/option/Options.java index 6adee2a01..3d54b5987 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/Options.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/Options.java @@ -48,7 +48,7 @@ public class Options { @OptionRequired(required = true,description = "Sql local plugin root") private String localSqlPluginPath; - @OptionRequired(required = true,description = "Sql remote plugin root") + @OptionRequired(required = false,description = "Sql remote plugin root") private String remoteSqlPluginPath ; @OptionRequired(description = "sql ext jar,eg udf jar") diff --git a/core/src/test/java/com/dtstack/flink/sql/TestGetPlan.java b/core/src/test/java/com/dtstack/flink/sql/TestGetPlan.java index 2ae5ee812..3e524a06f 100644 --- a/core/src/test/java/com/dtstack/flink/sql/TestGetPlan.java +++ b/core/src/test/java/com/dtstack/flink/sql/TestGetPlan.java @@ -46,13 +46,10 @@ public class TestGetPlan { @Test public void testGetExecutionPlan() throws Exception { List urls = new ArrayList(); - urls.addAll(getJarUrl("/Users/maqi/code/dtstack/dt-center-flinkStreamSQL/plugins/")); urls.addAll(getJarUrl("/Users/maqi/tmp/flink/flink-1.8.1/lib")); ClassLoader parentClassLoader = Thread.currentThread().getContextClassLoader(); DtClassLoader childClassLoader = new DtClassLoader(urls.toArray(new URL[urls.size()]), parentClassLoader); - - Thread.currentThread().setContextClassLoader(childClassLoader); Class aClass = childClassLoader.loadClass("com.dtstack.flink.sql.GetPlan"); String[] params = { @@ -62,8 +59,6 @@ public void testGetExecutionPlan() throws Exception { "%0d%0aCREATE+TABLE+MyTable(%0d%0a++++id+INT%2c%0d%0a++++channel+VARCHAR%2c%0d%0a++++pv+varchar%2c%0d%0a++++xctime+varchar%2c%0d%0a++++name+varchar%0d%0a+)WITH(%0d%0a++++type+%3d%27kafka11%27%2c%0d%0a++++bootstrapServers+%3d%27172.16.8.107%3a9092%27%2c%0d%0a++++zookeeperQuorum+%3d%27172.16.8.107%3a2181%2fkafka%27%2c%0d%0a++++offsetReset+%3d%27latest%27%2c%0d%0a++++topic+%3d%27mqTest02%27%2c%0d%0a++++timezone%3d%27Asia%2fShanghai%27%2c%0d%0a++++topicIsPattern+%3d%27false%27%2c%0d%0a++++parallelism+%3d%271%27%0d%0a+)%3b%0d%0a%0d%0a%0d%0aCREATE+TABLE+MyTable2(%0d%0a++++id2+INT%2c%0d%0a++++channel2+VARCHAR%2c%0d%0a++++pv2+varchar%2c%0d%0a++++xctime2+varchar%2c%0d%0a++++name2+varchar%0d%0a+)WITH(%0d%0a++++type+%3d%27kafka11%27%2c%0d%0a++++bootstrapServers+%3d%27172.16.8.107%3a9092%27%2c%0d%0a++++zookeeperQuorum+%3d%27172.16.8.107%3a2181%2fkafka%27%2c%0d%0a++++offsetReset+%3d%27latest%27%2c%0d%0a++++topic+%3d%27mqTest03%27%2c%0d%0a++++timezone%3d%27Asia%2fShanghai%27%2c%0d%0a++++topicIsPattern+%3d%27false%27%2c%0d%0a++++parallelism+%3d%271%27%0d%0a+)%3b%0d%0a%0d%0a%0d%0aCREATE+TABLE+sideTableA(%0d%0a++++id1+INT%2c%0d%0a++++channel1+varchar%2c%0d%0a++++time_info+varchar%2c%0d%0a++++name1+varchar%2c%0d%0a++++PRIMARY+KEY(channel1)+%2c%0d%0a++++PERIOD+FOR+SYSTEM_TIME%0d%0a+)WITH(%0d%0a++++type+%3d%27mysql%27%2c%0d%0a++++url+%3d%27jdbc%3amysql%3a%2f%2f172.16.8.109%3a3306%2ftest%27%2c%0d%0a++++userName+%3d%27dtstack%27%2c%0d%0a++++password+%3d%27abc123%27%2c%0d%0a++++tableName+%3d%27dimA%27%2c%0d%0a++++parallelism+%3d%271%27%2c%0d%0a++++cache+%3d+%27LRU%27%0d%0a%0d%0a+)%3b%0d%0a%0d%0a+CREATE+TABLE+sideTableB(%0d%0a++++id+INT%2c%0d%0a++++channel1+varchar%2c%0d%0a++++address+varchar%2c%0d%0a++++PRIMARY+KEY(channel1)%2c%0d%0a++++PERIOD+FOR+SYSTEM_TIME%0d%0a+)WITH(%0d%0a++++type+%3d%27mysql%27%2c%0d%0a++++url+%3d%27jdbc%3amysql%3a%2f%2f172.16.8.109%3a3306%2ftest%27%2c%0d%0a++++userName+%3d%27dtstack%27%2c%0d%0a++++password+%3d%27abc123%27%2c%0d%0a++++tableName+%3d%27dimB%27%2c%0d%0a++++parallelism+%3d%271%27%2c%0d%0a++++cache+%3d+%27LRU%27%2c%0d%0a++++asyncTimeout+%3d+%271000000%27%0d%0a+)%3b%0d%0a%0d%0aCREATE+TABLE+MyResult(%0d%0a++++xctime+VARCHAR%2c%0d%0a++++name+VARCHAR%2c%0d%0a++++time_info+VARCHAR%2c%0d%0a++++address+VARCHAR%0d%0a+)WITH(%0d%0a+++++type+%3d%27mysql%27%2c%0d%0a++++url+%3d%27jdbc%3amysql%3a%2f%2f172.16.8.109%3a3306%2ftest%27%2c%0d%0a++++userName+%3d%27dtstack%27%2c%0d%0a++++password+%3d%27abc123%27%2c%0d%0a++++tableName+%3d%27dimC%27%2c%0d%0a++++parallelism+%3d%271%27%2c%0d%0a+)%3b%0d%0a%0d%0ainsert+%0d%0ainto%0d%0a++MyResult%0d%0a++++select%0d%0a++++++++t1.xctime+as+xctime%2c%0d%0a++++++++t1.name+as+name%2c%0d%0a++++++++t3.channel1+as+time_info%2c%0d%0a++++++++t3.address+as+address+++++%0d%0a++++from%0d%0a++++++++MyTable+as+t1%0d%0a++++left+join%0d%0a++++++++--MyTable2+m2+%0d%0a++++++++sideTableA+m2+++++++++++++++++++++++++++%0d%0a++++++++on++t1.channel+%3d+m2.channel1%0d%0a++++left+join+sideTableB+t3%0d%0a++++++++on+t1.channel+%3d+t3.channel1%0d%0a%0d%0a%0d%0a", "-localSqlPluginPath", "/Users/maqi/code/dtstack/dt-center-flinkStreamSQL/plugins", - "-remoteSqlPluginPath", - "/Users/maqi/code/dtstack/dt-center-flinkStreamSQL/plugins", "-name","test"}; Method getExecutionPlan = aClass.getMethod("getExecutionPlan", String[].class); From e922674b8fa6e5a6a474105b91c1f312bb09997f Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 24 Feb 2020 21:53:37 +0800 Subject: [PATCH 040/523] add log level --- .../main/java/com/dtstack/flink/sql/Main.java | 1 + .../flink/sql/exec/ExecuteProcessHelper.java | 14 ++++++++++++++ .../com/dtstack/flink/sql/exec/ParamsInfo.java | 17 +++++++++++++++-- .../com/dtstack/flink/sql/option/Options.java | 2 +- 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index a9daf8125..4ff296f1f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql; + import com.dtstack.flink.sql.exec.ExecuteProcessHelper; import com.dtstack.flink.sql.exec.ParamsInfo; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 5320821b5..2c6d74df0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -18,6 +18,8 @@ package com.dtstack.flink.sql.exec; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.LoggerContext; import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.config.CalciteConfig; import com.dtstack.flink.sql.enums.ClusterMode; @@ -108,6 +110,7 @@ public static ParamsInfo parseParams(String[] args) throws Exception { String remoteSqlPluginPath = options.getRemoteSqlPluginPath(); String pluginLoadMode = options.getPluginLoadMode(); String deployMode = options.getMode(); + String logLevel = options.getLogLevel(); Preconditions.checkArgument(checkRemoteSqlPluginPath(remoteSqlPluginPath, deployMode, pluginLoadMode), "Non-local mode or shipfile deployment mode, remoteSqlPluginPath is required"); @@ -125,6 +128,7 @@ public static ParamsInfo parseParams(String[] args) throws Exception { .setDeployMode(deployMode) .setConfProp(confProperties) .setJarUrlList(jarURList) + .setLogLevel(logLevel) .build(); } @@ -150,6 +154,8 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); StreamQueryConfig streamQueryConfig = StreamEnvConfigManager.getStreamQueryConfig(tableEnv, paramsInfo.getConfProp()); + setLogLevel(paramsInfo.getLogLevel()); + SqlParser.setLocalSqlPluginRoot(paramsInfo.getLocalSqlPluginPath()); SqlTree sqlTree = SqlParser.parseSql(paramsInfo.getSql()); @@ -341,4 +347,12 @@ public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperti StreamEnvConfigManager.streamExecutionEnvironmentConfig(env, confProperties); return env; } + + private static void setLogLevel(String level){ + LoggerContext loggerContext= (LoggerContext) LoggerFactory.getILoggerFactory(); + //设置全局日志级别 + ch.qos.logback.classic.Logger logger = loggerContext.getLogger("root"); + logger.setLevel(Level.toLevel(level, Level.INFO)); + } + } \ No newline at end of file diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java index 0c98a4460..59249827c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java @@ -38,10 +38,11 @@ public class ParamsInfo { private String remoteSqlPluginPath; private String pluginLoadMode; private String deployMode; + private String logLevel; private Properties confProp; public ParamsInfo(String sql, String name, List jarUrlList, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, String deployMode, Properties confProp) { + String remoteSqlPluginPath, String pluginLoadMode, String deployMode, String logLevel, Properties confProp) { this.sql = sql; this.name = name; this.jarUrlList = jarUrlList; @@ -49,6 +50,7 @@ public ParamsInfo(String sql, String name, List jarUrlList, String localSql this.remoteSqlPluginPath = remoteSqlPluginPath; this.pluginLoadMode = pluginLoadMode; this.deployMode = deployMode; + this.logLevel = logLevel; this.confProp = confProp; } @@ -84,6 +86,10 @@ public Properties getConfProp() { return confProp; } + public String getLogLevel() { + return logLevel; + } + @Override public String toString() { return "ParamsInfo{" + @@ -94,6 +100,7 @@ public String toString() { ", remoteSqlPluginPath='" + remoteSqlPluginPath + '\'' + ", pluginLoadMode='" + pluginLoadMode + '\'' + ", deployMode='" + deployMode + '\'' + + ", logLevel='" + logLevel + '\'' + ", confProp=" + confProp + '}'; } @@ -114,6 +121,7 @@ public static class Builder { private String remoteSqlPluginPath; private String pluginLoadMode; private String deployMode; + private String logLevel; private Properties confProp; @@ -152,6 +160,11 @@ public ParamsInfo.Builder setDeployMode(String deployMode) { return this; } + public ParamsInfo.Builder setLogLevel(String logLevel) { + this.logLevel = logLevel; + return this; + } + public ParamsInfo.Builder setConfProp(Properties confProp) { this.confProp = confProp; return this; @@ -159,7 +172,7 @@ public ParamsInfo.Builder setConfProp(Properties confProp) { public ParamsInfo build() { return new ParamsInfo(sql, name, jarUrlList, localSqlPluginPath, - remoteSqlPluginPath, pluginLoadMode, deployMode, confProp); + remoteSqlPluginPath, pluginLoadMode, deployMode, logLevel, confProp); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/option/Options.java b/core/src/main/java/com/dtstack/flink/sql/option/Options.java index 74c26ed88..91ecf19fc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/Options.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/Options.java @@ -69,7 +69,7 @@ public class Options { @OptionRequired(description = "plugin load mode, by classpath or shipfile") private String pluginLoadMode = EPluginLoadMode.CLASSPATH.name(); - private String logLevel = "logLevel"; + private String logLevel; public String getMode() { return mode; From f7922de20ddce7fba80d49899e5ab8f70a38f0fa Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Mon, 24 Feb 2020 22:34:11 +0800 Subject: [PATCH 041/523] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E6=A0=BC=E5=BC=8F?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../side/cassandra/CassandraAllReqRow.java | 9 +- .../side/cassandra/CassandraAsyncReqRow.java | 15 +- .../sink/cassandra/CassandraOutputFormat.java | 9 +- .../dtstack/flink/sql/side/SideSqlExec.java | 256 ++++++++++-------- .../flink/sql/table/TableInfoParser.java | 6 +- .../sink/elasticsearch/ElasticsearchSink.java | 6 +- .../sql/sink/hbase/HbaseOutputFormat.java | 6 +- .../flink/sql/side/kudu/KuduAllReqRow.java | 21 +- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 15 +- .../sql/launcher/perjob/PerJobSubmitter.java | 1 - .../flink/sql/side/mongo/MongoAllReqRow.java | 6 +- .../sql/side/mongo/MongoAsyncReqRow.java | 12 +- .../flink/sql/side/rdb/all/RdbAllReqRow.java | 6 +- .../sql/side/rdb/all/RdbAllSideInfo.java | 6 +- .../sql/side/rdb/async/RdbAsyncSideInfo.java | 7 +- .../flink/sql/side/redis/RedisAllReqRow.java | 17 +- .../sql/side/redis/RedisAsyncReqRow.java | 6 +- .../sql/sink/redis/RedisOutputFormat.java | 12 +- 18 files changed, 268 insertions(+), 148 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index 8d9510e40..bc0a0f8cd 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -24,7 +24,14 @@ import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import com.datastax.driver.core.*; +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.ConsistencyLevel; +import com.datastax.driver.core.HostDistance; +import com.datastax.driver.core.PoolingOptions; +import com.datastax.driver.core.QueryOptions; +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.SocketOptions; import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; import com.dtstack.flink.sql.side.AllReqRow; diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index 922821f51..0173f2d6f 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -26,11 +26,22 @@ import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; -import com.datastax.driver.core.*; +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.ConsistencyLevel; +import com.datastax.driver.core.HostDistance; +import com.datastax.driver.core.PoolingOptions; +import com.datastax.driver.core.QueryOptions; +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.SocketOptions; import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.google.common.base.Function; diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java index 88ad81fcf..08a2358cb 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java @@ -44,7 +44,14 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; -import com.datastax.driver.core.*; +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.ConsistencyLevel; +import com.datastax.driver.core.HostDistance; +import com.datastax.driver.core.PoolingOptions; +import com.datastax.driver.core.QueryOptions; +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.SocketOptions; import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 7db118c45..479a330a2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -17,6 +17,7 @@ */ + package com.dtstack.flink.sql.side; import org.apache.flink.api.common.typeinfo.TypeInformation; @@ -44,7 +45,20 @@ import com.google.common.collect.HashBasedTable; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.calcite.sql.*; +import org.apache.calcite.sql.SqlAsOperator; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlInsert; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlOrderBy; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlWithItem; import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParserPos; @@ -54,7 +68,12 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.*; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; import static org.apache.calcite.sql.SqlKind.*; @@ -62,7 +81,6 @@ * Reason: * Date: 2018/7/24 * Company: www.dtstack.com - * * @author xuchao */ @@ -78,11 +96,11 @@ public class SideSqlExec { private SidePredicatesParser sidePredicatesParser = new SidePredicatesParser(); private Map localTableCache = Maps.newHashMap(); - private StreamTableEnvironment tableEnv; + private StreamTableEnvironment tableEnv ; public void exec(String sql, Map sideTableMap, StreamTableEnvironment tableEnv, Map tableCache, StreamQueryConfig queryConfig) throws Exception { - if (localSqlPluginPath == null) { + if(localSqlPluginPath == null){ throw new RuntimeException("need to set localSqlPluginPath"); } @@ -101,36 +119,36 @@ public void exec(String sql, Map sideTableMap, StreamTabl boolean preIsSideJoin = false; List replaceInfoList = Lists.newArrayList(); - while ((pollObj = exeQueue.poll()) != null) { + while((pollObj = exeQueue.poll()) != null){ - if (pollObj instanceof SqlNode) { + if(pollObj instanceof SqlNode){ SqlNode pollSqlNode = (SqlNode) pollObj; - if (preIsSideJoin) { + if(preIsSideJoin){ preIsSideJoin = false; List fieldNames = null; - for (FieldReplaceInfo replaceInfo : replaceInfoList) { + for(FieldReplaceInfo replaceInfo : replaceInfoList){ fieldNames = Lists.newArrayList(); replaceFieldName(pollSqlNode, replaceInfo); addAliasForFieldNode(pollSqlNode, fieldNames, replaceInfo.getMappingTable()); } } - if (pollSqlNode.getKind() == INSERT) { - System.out.println("----------real exec sql-----------"); + if(pollSqlNode.getKind() == INSERT){ + System.out.println("----------real exec sql-----------" ); System.out.println(pollSqlNode.toString()); FlinkSQLExec.sqlUpdate(tableEnv, pollSqlNode.toString(), queryConfig); - if (LOG.isInfoEnabled()) { + if(LOG.isInfoEnabled()){ LOG.info("exec sql: " + pollSqlNode.toString()); } - } else if (pollSqlNode.getKind() == AS) { + }else if(pollSqlNode.getKind() == AS){ AliasInfo aliasInfo = parseASNode(pollSqlNode); Table table = tableEnv.sqlQuery(aliasInfo.getName()); tableEnv.registerTable(aliasInfo.getAlias(), table); localTableCache.put(aliasInfo.getAlias(), table); FieldReplaceInfo fieldReplaceInfo = parseAsQuery((SqlBasicCall) pollSqlNode, tableCache); - if (fieldReplaceInfo != null) { + if(fieldReplaceInfo != null){ replaceInfoList.add(fieldReplaceInfo); } } else if (pollSqlNode.getKind() == WITH_ITEM) { @@ -140,7 +158,7 @@ public void exec(String sql, Map sideTableMap, StreamTabl tableEnv.registerTable(TableAlias, table); } - } else if (pollObj instanceof JoinInfo) { + }else if (pollObj instanceof JoinInfo){ preIsSideJoin = true; joinFun(pollObj, localTableCache, sideTableMap, tableEnv, replaceInfoList); } @@ -150,17 +168,16 @@ public void exec(String sql, Map sideTableMap, StreamTabl /** * 解析出as查询的表和字段的关系 - * * @param asSqlNode * @param tableCache * @return */ - private FieldReplaceInfo parseAsQuery(SqlBasicCall asSqlNode, Map tableCache) { + private FieldReplaceInfo parseAsQuery(SqlBasicCall asSqlNode, Map tableCache){ SqlNode info = asSqlNode.getOperands()[0]; SqlNode alias = asSqlNode.getOperands()[1]; SqlKind infoKind = info.getKind(); - if (infoKind != SELECT) { + if(infoKind != SELECT){ return null; } @@ -184,7 +201,6 @@ private FieldReplaceInfo parseAsQuery(SqlBasicCall asSqlNode, Map /** * 添加字段别名 - * * @param pollSqlNode * @param fieldList * @param mappingTable @@ -224,7 +240,7 @@ private void addAliasForFieldNode(SqlNode pollSqlNode, List fieldList, H } String name = sqlIdentifier.names.get(1); // avoid real field pv0 convert pv - if (name.endsWith("0") && !fieldList.contains(name) && !fieldList.contains(name.substring(0, name.length() - 1))) { + if (name.endsWith("0") && !fieldList.contains(name) && !fieldList.contains(name.substring(0, name.length() - 1))) { SqlOperator operator = new SqlAsOperator(); SqlParserPos sqlParserPos = new SqlParserPos(0, 0); @@ -245,11 +261,11 @@ private void addAliasForFieldNode(SqlNode pollSqlNode, List fieldList, H public AliasInfo parseASNode(SqlNode sqlNode) throws SqlParseException { SqlKind sqlKind = sqlNode.getKind(); - if (sqlKind != AS) { + if(sqlKind != AS){ throw new RuntimeException(sqlNode + " is not 'as' operator"); } - SqlNode info = ((SqlBasicCall) sqlNode).getOperands()[0]; + SqlNode info = ((SqlBasicCall)sqlNode).getOperands()[0]; SqlNode alias = ((SqlBasicCall) sqlNode).getOperands()[1]; AliasInfo aliasInfo = new AliasInfo(); @@ -276,9 +292,9 @@ public RowTypeInfo buildOutRowTypeInfo(List sideJoinFieldInfo, HashBa } + /** - * 对时间类型进行类型转换 - * + * 对时间类型进行类型转换 * @param leftTypeInfo * @return */ @@ -313,14 +329,14 @@ private void replaceFieldName(SqlNode sqlNode, FieldReplaceInfo replaceInfo) { break; case SELECT: SqlSelect sqlSelect = (SqlSelect) filterNodeWithTargetName(sqlNode, replaceInfo.getTargetTableName()); - if (sqlSelect == null) { + if(sqlSelect == null){ return; } SqlNode sqlSource1 = sqlSelect.getFrom(); - if (sqlSource1.getKind() == AS) { - String tableName = ((SqlBasicCall) sqlSource1).getOperands()[0].toString(); - if (tableName.equalsIgnoreCase(replaceInfo.getTargetTableName())) { + if(sqlSource1.getKind() == AS){ + String tableName = ((SqlBasicCall)sqlSource1).getOperands()[0].toString(); + if(tableName.equalsIgnoreCase(replaceInfo.getTargetTableName())){ SqlNodeList sqlSelectList = sqlSelect.getSelectList(); SqlNode whereNode = sqlSelect.getWhere(); SqlNodeList sqlGroup = sqlSelect.getGroup(); @@ -329,17 +345,17 @@ private void replaceFieldName(SqlNode sqlNode, FieldReplaceInfo replaceInfo) { SqlNode sqlHaving = sqlSelect.getHaving(); List newSelectNodeList = Lists.newArrayList(); - for (int i = 0; i < sqlSelectList.getList().size(); i++) { + for( int i=0; i replaceNodeList = replaceSelectStarFieldName(selectNode, replaceInfo); newSelectNodeList.addAll(replaceNodeList); continue; } SqlNode replaceNode = replaceSelectFieldName(selectNode, replaceInfo); - if (replaceNode == null) { + if(replaceNode == null){ continue; } @@ -351,23 +367,23 @@ private void replaceFieldName(SqlNode sqlNode, FieldReplaceInfo replaceInfo) { sqlSelect.setSelectList(newSelectList); //where - if (whereNode != null) { - SqlNode[] sqlNodeList = ((SqlBasicCall) whereNode).getOperands(); - for (int i = 0; i < sqlNodeList.length; i++) { + if(whereNode != null){ + SqlNode[] sqlNodeList = ((SqlBasicCall)whereNode).getOperands(); + for(int i =0; i localTableCache, String tableAlias, String tableName) { + private Table getTableFromCache(Map localTableCache, String tableAlias, String tableName){ Table table = localTableCache.get(tableAlias); - if (table == null) { + if(table == null){ table = localTableCache.get(tableName); } - if (table == null) { + if(table == null){ throw new RuntimeException("not register table " + tableName); } return table; } - private List replaceSelectStarFieldName(SqlNode selectNode, FieldReplaceInfo replaceInfo) { + private List replaceSelectStarFieldName(SqlNode selectNode, FieldReplaceInfo replaceInfo){ SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; List sqlNodes = Lists.newArrayList(); - if (sqlIdentifier.isStar()) {//处理 [* or table.*] + if(sqlIdentifier.isStar()){//处理 [* or table.*] int identifierSize = sqlIdentifier.names.size(); Collection columns = null; - if (identifierSize == 1) { + if(identifierSize == 1){ columns = replaceInfo.getMappingTable().values(); - } else { + }else{ columns = replaceInfo.getMappingTable().row(sqlIdentifier.names.get(0)).values(); } - for (String colAlias : columns) { + for(String colAlias : columns){ SqlParserPos sqlParserPos = new SqlParserPos(0, 0); List columnInfo = Lists.newArrayList(); columnInfo.add(replaceInfo.getTargetTableAlias()); @@ -518,7 +534,7 @@ private List replaceSelectStarFieldName(SqlNode selectNode, FieldReplac } return sqlNodes; - } else { + }else{ throw new RuntimeException("is not a star select field." + selectNode); } } @@ -532,25 +548,25 @@ private SqlNode replaceSelectFieldName(SqlNode selectNode, FieldReplaceInfo repl } return selectNode; - } else if (selectNode.getKind() == IDENTIFIER) { + }else if(selectNode.getKind() == IDENTIFIER){ SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; - if (sqlIdentifier.names.size() == 1) { + if(sqlIdentifier.names.size() == 1){ return selectNode; } //Same level mappingTable String mappingFieldName = replaceInfo.getTargetFieldName(sqlIdentifier.getComponent(0).getSimple(), sqlIdentifier.getComponent(1).getSimple()); if (mappingFieldName == null) { - throw new RuntimeException("can't find mapping fieldName:" + selectNode.toString()); + throw new RuntimeException("can't find mapping fieldName:" + selectNode.toString() ); } sqlIdentifier = sqlIdentifier.setName(0, replaceInfo.getTargetTableAlias()); sqlIdentifier = sqlIdentifier.setName(1, mappingFieldName); return sqlIdentifier; - } else if (selectNode.getKind() == LITERAL || selectNode.getKind() == LITERAL_CHAIN) {//字面含义 + }else if(selectNode.getKind() == LITERAL || selectNode.getKind() == LITERAL_CHAIN){//字面含义 return selectNode; - } else if (AGGREGATE.contains(selectNode.getKind()) + }else if( AGGREGATE.contains(selectNode.getKind()) || AVG_AGG_FUNCTIONS.contains(selectNode.getKind()) || COMPARISON.contains(selectNode.getKind()) || selectNode.getKind() == OTHER_FUNCTION @@ -580,20 +596,20 @@ private SqlNode replaceSelectFieldName(SqlNode selectNode, FieldReplaceInfo repl || selectNode.getKind() == TIMESTAMP_DIFF || selectNode.getKind() == LIKE - ) { + ){ SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; - for (int i = 0; i < sqlBasicCall.getOperands().length; i++) { + for(int i=0; i conditionFields = getConditionFields(conditionNode, sideTableAlias, sideTableInfo); - if (CollectionUtils.isEqualCollection(conditionFields, convertPrimaryAlias(sideTableInfo))) { + if(CollectionUtils.isEqualCollection(conditionFields, convertPrimaryAlias(sideTableInfo))){ return true; } return false; @@ -655,27 +671,27 @@ private List convertPrimaryAlias(SideTableInfo sideTableInfo) { return res; } - public List getConditionFields(SqlNode conditionNode, String specifyTableName, SideTableInfo sideTableInfo) { + public List getConditionFields(SqlNode conditionNode, String specifyTableName, SideTableInfo sideTableInfo){ List sqlNodeList = Lists.newArrayList(); ParseUtils.parseAnd(conditionNode, sqlNodeList); List conditionFields = Lists.newArrayList(); - for (SqlNode sqlNode : sqlNodeList) { + for(SqlNode sqlNode : sqlNodeList){ if (!SqlKind.COMPARISON.contains(sqlNode.getKind())) { throw new RuntimeException("not compare operator."); } - SqlIdentifier left = (SqlIdentifier) ((SqlBasicCall) sqlNode).getOperands()[0]; - SqlIdentifier right = (SqlIdentifier) ((SqlBasicCall) sqlNode).getOperands()[1]; + SqlIdentifier left = (SqlIdentifier)((SqlBasicCall)sqlNode).getOperands()[0]; + SqlIdentifier right = (SqlIdentifier)((SqlBasicCall)sqlNode).getOperands()[1]; String leftTableName = left.getComponent(0).getSimple(); String rightTableName = right.getComponent(0).getSimple(); String tableCol = ""; - if (leftTableName.equalsIgnoreCase(specifyTableName)) { + if(leftTableName.equalsIgnoreCase(specifyTableName)){ tableCol = left.getComponent(1).getSimple(); - } else if (rightTableName.equalsIgnoreCase(specifyTableName)) { + }else if(rightTableName.equalsIgnoreCase(specifyTableName)){ tableCol = right.getComponent(1).getSimple(); - } else { + }else{ throw new RuntimeException(String.format("side table:%s join condition is wrong", specifyTableName)); } tableCol = sideTableInfo.getPhysicalFields().getOrDefault(tableCol, tableCol); @@ -690,7 +706,7 @@ public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, Map tableCache) throws Exception { - if (localSqlPluginPath == null) { + if(localSqlPluginPath == null){ throw new RuntimeException("need to set localSqlPluginPath"); } @@ -702,12 +718,12 @@ public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, boolean preIsSideJoin = false; List replaceInfoList = Lists.newArrayList(); - while ((pollObj = exeQueue.poll()) != null) { + while((pollObj = exeQueue.poll()) != null){ - if (pollObj instanceof SqlNode) { + if(pollObj instanceof SqlNode){ SqlNode pollSqlNode = (SqlNode) pollObj; - if (preIsSideJoin) { + if(preIsSideJoin){ preIsSideJoin = false; List fieldNames = null; for (FieldReplaceInfo replaceInfo : replaceInfoList) { @@ -717,16 +733,16 @@ public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, } } - if (pollSqlNode.getKind() == INSERT) { + if(pollSqlNode.getKind() == INSERT){ tableEnv.sqlUpdate(pollSqlNode.toString()); - } else if (pollSqlNode.getKind() == AS) { + }else if(pollSqlNode.getKind() == AS){ dealAsSourceTable(tableEnv, pollSqlNode); - } else if (pollSqlNode.getKind() == SELECT) { + } else if (pollSqlNode.getKind() == SELECT){ Table table = tableEnv.sqlQuery(pollObj.toString()); - if (result.getFieldsInfoStr() == null) { + if (result.getFieldsInfoStr() == null){ tableEnv.registerTable(result.getTableName(), table); } else { - if (checkFieldsInfo(result, table)) { + if (checkFieldsInfo(result, table)){ table = table.as(tmpFields); tableEnv.registerTable(result.getTableName(), table); } else { @@ -737,7 +753,7 @@ public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, } - } else if (pollObj instanceof JoinInfo) { + }else if (pollObj instanceof JoinInfo){ preIsSideJoin = true; joinFun(pollObj, localTableCache, sideTableMap, tableEnv, replaceInfoList); } @@ -761,7 +777,7 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, SqlNode pollSq private void joinFun(Object pollObj, Map localTableCache, Map sideTableMap, StreamTableEnvironment tableEnv, - List replaceInfoList) throws Exception { + List replaceInfoList) throws Exception{ JoinInfo joinInfo = (JoinInfo) pollObj; JoinScope joinScope = new JoinScope(); @@ -770,7 +786,7 @@ private void joinFun(Object pollObj, Map localTableCache, leftScopeChild.setTableName(joinInfo.getLeftTableName()); SqlKind sqlKind = joinInfo.getLeftNode().getKind(); - if (sqlKind == AS) { + if(sqlKind == AS){ dealAsSourceTable(tableEnv, joinInfo.getLeftNode()); } @@ -782,15 +798,15 @@ private void joinFun(Object pollObj, Map localTableCache, rightScopeChild.setAlias(joinInfo.getRightTableAlias()); rightScopeChild.setTableName(joinInfo.getRightTableName()); SideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); - if (sideTableInfo == null) { + if(sideTableInfo == null){ sideTableInfo = sideTableMap.get(joinInfo.getRightTableAlias()); } - if (sideTableInfo == null) { + if(sideTableInfo == null){ throw new RuntimeException("can't not find side table:" + joinInfo.getRightTableName()); } - if (!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)) { + if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ throw new RuntimeException("ON condition must contain all equal fields!!!"); } @@ -804,7 +820,7 @@ private void joinFun(Object pollObj, Map localTableCache, String leftTableAlias = joinInfo.getLeftTableAlias(); Table targetTable = localTableCache.get(leftTableAlias); - if (targetTable == null) { + if(targetTable == null){ targetTable = localTableCache.get(joinInfo.getLeftTableName()); } @@ -825,9 +841,9 @@ private void joinFun(Object pollObj, Map localTableCache, } DataStream dsOut = null; - if (ECacheType.ALL.name().equalsIgnoreCase(sideTableInfo.getCacheType())) { + if(ECacheType.ALL.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ dsOut = SideWithAllCacheOperator.getSideJoinDataStream(adaptStream, sideTableInfo.getType(), localSqlPluginPath, typeInfo, joinInfo, sideJoinFieldInfo, sideTableInfo); - } else { + }else{ dsOut = SideAsyncOperator.getSideJoinDataStream(adaptStream, sideTableInfo.getType(), localSqlPluginPath, typeInfo, joinInfo, sideJoinFieldInfo, sideTableInfo); } @@ -847,9 +863,9 @@ private void joinFun(Object pollObj, Map localTableCache, replaceInfo.setTargetTableAlias(targetTableAlias); //判断之前是不是被替换过,被替换过则设置之前的替换信息作为上一个节点 - for (FieldReplaceInfo tmp : replaceInfoList) { - if (tmp.getTargetTableName().equalsIgnoreCase(joinInfo.getLeftTableName()) - || tmp.getTargetTableName().equalsIgnoreCase(joinInfo.getLeftTableAlias())) { + for(FieldReplaceInfo tmp : replaceInfoList){ + if(tmp.getTargetTableName().equalsIgnoreCase(joinInfo.getLeftTableName()) + ||tmp.getTargetTableName().equalsIgnoreCase(joinInfo.getLeftTableAlias())){ replaceInfo.setPreNode(tmp); break; } @@ -857,7 +873,7 @@ private void joinFun(Object pollObj, Map localTableCache, replaceInfoList.add(replaceInfo); - if (!tableEnv.isRegistered(joinInfo.getNewTableName())) { + if (!tableEnv.isRegistered(joinInfo.getNewTableName())){ Table joinTable = tableEnv.fromDataStream(dsOut); tableEnv.registerTable(joinInfo.getNewTableName(), joinTable); localTableCache.put(joinInfo.getNewTableName(), joinTable); @@ -880,17 +896,17 @@ private boolean checkFieldsInfo(CreateTmpTableParser.SqlParserResult result, Tab String[] fields = StringUtils.split(fieldsInfo, ","); for (int i = 0; i < fields.length; i++) { String[] filed = StringUtils.split(fields[i], "\\s"); - if (filed.length < 2 || fields.length != table.getSchema().getColumnNames().length) { + if (filed.length < 2 || fields.length != table.getSchema().getColumnNames().length){ return false; } else { String[] filedNameArr = new String[filed.length - 1]; System.arraycopy(filed, 0, filedNameArr, 0, filed.length - 1); String fieldName = String.join(" ", filedNameArr); fieldNames.add(fieldName); - String fieldType = filed[filed.length - 1].trim(); + String fieldType = filed[filed.length - 1 ].trim(); Class fieldClass = ClassUtil.stringConvertClass(fieldType); Class tableField = table.getSchema().getFieldType(i).get().getTypeClass(); - if (fieldClass == tableField) { + if (fieldClass == tableField){ continue; } else { return false; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java index 46ad3691a..f4c638381 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java @@ -110,11 +110,11 @@ public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserRe * @param tableField * @return */ - private static boolean checkIsSideTable(String tableField) { + private static boolean checkIsSideTable(String tableField){ String[] fieldInfos = StringUtils.split(tableField, ","); - for (String field : fieldInfos) { + for(String field : fieldInfos){ Matcher matcher = SIDE_PATTERN.matcher(field.trim()); - if (matcher.find()) { + if(matcher.find()){ return true; } } diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 5933ab7ec..491d1af27 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -40,7 +40,11 @@ import java.net.InetAddress; import java.net.InetSocketAddress; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * table output elastic5plugin diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 990686afc..d41c65723 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -31,7 +31,11 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 072d7f630..63ff31c50 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -7,7 +7,11 @@ import org.apache.flink.util.Collector; import org.apache.flink.util.Preconditions; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.PredicateInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; import com.google.common.collect.Lists; @@ -17,13 +21,24 @@ import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; -import org.apache.kudu.client.*; +import org.apache.kudu.client.KuduClient; +import org.apache.kudu.client.KuduException; +import org.apache.kudu.client.KuduPredicate; +import org.apache.kudu.client.KuduScanner; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.PartialRow; +import org.apache.kudu.client.RowResult; +import org.apache.kudu.client.RowResultIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.SQLException; import java.sql.Timestamp; -import java.util.*; +import java.util.Arrays; +import java.util.Calendar; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; public class KuduAllReqRow extends AllReqRow { diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index 4b47f9e92..e1705d796 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -9,7 +9,12 @@ import org.apache.flink.util.Preconditions; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.PredicateInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; @@ -21,7 +26,13 @@ import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; -import org.apache.kudu.client.*; +import org.apache.kudu.client.AsyncKuduClient; +import org.apache.kudu.client.AsyncKuduScanner; +import org.apache.kudu.client.KuduException; +import org.apache.kudu.client.KuduPredicate; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.RowResult; +import org.apache.kudu.client.RowResultIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index 17ec3591e..29cc4890a 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -32,7 +32,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.net.URLDecoder; import java.util.Arrays; import java.util.List; diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index bcde9431e..f90cfd973 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -32,7 +32,11 @@ import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.mongodb.*; +import com.mongodb.BasicDBObject; +import com.mongodb.MongoClient; +import com.mongodb.MongoClientOptions; +import com.mongodb.MongoCredential; +import com.mongodb.ServerAddress; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index ed238cc9b..42749e935 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -40,7 +40,11 @@ import com.mongodb.MongoCredential; import com.mongodb.ServerAddress; import com.mongodb.async.SingleResultCallback; -import com.mongodb.async.client.*; +import com.mongodb.async.client.MongoClient; +import com.mongodb.async.client.MongoClientSettings; +import com.mongodb.async.client.MongoClients; +import com.mongodb.async.client.MongoCollection; +import com.mongodb.async.client.MongoDatabase; import com.mongodb.connection.ClusterSettings; import com.mongodb.connection.ConnectionPoolSettings; import org.apache.commons.lang3.StringUtils; @@ -49,7 +53,11 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; /** diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java index 72338205f..7fdeeeca9 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java @@ -35,7 +35,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.*; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; import java.util.Calendar; import java.util.List; import java.util.Map; diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index a6e8501da..3b7444b77 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -20,7 +20,11 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.PredicateInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java index 707f9df31..37a77ee84 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java @@ -20,11 +20,14 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.PredicateInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; - import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index e1b5db711..9da5b2425 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -36,12 +36,23 @@ import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.*; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.JedisCommands; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisSentinelPool; import java.io.Closeable; import java.io.IOException; import java.sql.SQLException; -import java.util.*; +import java.util.Calendar; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -178,7 +189,7 @@ private void loadData(Map> tmpCache) throws SQLExcep kv.put(primaryKv[1], primaryKv[2]); String pattern = key + "*"; Set realKeys = ((Jedis) jedis).keys(pattern); - for (String realKey : realKeys) { + for (String realKey : realKeys){ kv.put(StringUtils.split(realKey, ":")[3], jedis.get(realKey)); } tmpCache.put(key, kv); diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 903092790..41f504bf0 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -25,7 +25,11 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index 4e2d54bde..0bd053473 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -28,11 +28,19 @@ import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.*; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.JedisCommands; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisSentinelPool; import java.io.Closeable; import java.io.IOException; -import java.util.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; public class RedisOutputFormat extends DtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(RedisOutputFormat.class); From cfa0226514c841e43728bbac10a5192a2d1f31af Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Tue, 25 Feb 2020 00:08:38 +0800 Subject: [PATCH 042/523] =?UTF-8?q?=E5=BC=82=E5=B8=B8=E6=97=B6=E5=B0=86?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=86=99=E5=85=A5=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/elasticsearch6Sink.md | 2 +- .../sql/sink/elasticsearch6/CustomerSinkFunc.java | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/elasticsearch6Sink.md b/docs/elasticsearch6Sink.md index 981693735..47514b20c 100644 --- a/docs/elasticsearch6Sink.md +++ b/docs/elasticsearch6Sink.md @@ -48,7 +48,7 @@ CREATE TABLE MyResult( aa INT, bb INT )WITH( - type ='elasticsearch', + type ='elasticsearch6', address ='172.16.10.47:9500', cluster='es_47_menghan', esType ='type1', diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java index 091e18ede..1973cf01f 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java @@ -68,19 +68,19 @@ public CustomerSinkFunc(String index, String type, List fieldNames, List @Override public void process(Tuple2 tuple2, RuntimeContext ctx, RequestIndexer indexer) { - try{ + try { Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); Row element = tupleTrans.getField(1); - if(!retract){ + if (!retract) { return; } - indexer.add(createIndexRequest(element)); outRecords.inc(); - }catch (Throwable e){ - logger.error("", e); + } catch (Throwable e) { + logger.error("Failed to store source data {}. ", tuple2.getField(1)); + logger.error("Failed to create index request exception. ", e); } } From 94e8a0b6030eeb5e57dfb35f6cf1fbe8c5509ee6 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Tue, 25 Feb 2020 09:43:42 +0800 Subject: [PATCH 043/523] =?UTF-8?q?=E5=BC=82=E5=B8=B8=E6=97=B6=E5=B0=86?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=86=99=E5=85=A5=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/elasticsearch6/Elasticsearch6AllReqRow.java | 6 ++++++ .../sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java | 5 ++--- .../dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java | 6 ++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index d9d9e9487..82a43de66 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -159,6 +159,7 @@ protected void initCache() throws SQLException { Map>> newCache = Maps.newConcurrentMap(); cacheRef.set(newCache); try { + // create search request and build where cause searchRequest = Es6Util.setSearchRequest(sideInfo); boolQueryBuilder = Es6Util.setPredicateclause(sideInfo); loadData(newCache); @@ -217,6 +218,7 @@ private void loadData(Map>> tmpCache) throws IO } } + // initialize searchSourceBuilder private SearchSourceBuilder initConfiguration(BoolQueryBuilder boolQueryBuilder){ SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); if (boolQueryBuilder != null) { @@ -225,6 +227,8 @@ private SearchSourceBuilder initConfiguration(BoolQueryBuilder boolQueryBuilder) searchSourceBuilder.size(getFetchSize()); searchSourceBuilder.sort("_id", SortOrder.DESC); + + // fields included in the source data String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); searchSourceBuilder.fetchSource(sideFieldNames, null); return searchSourceBuilder; @@ -240,6 +244,7 @@ private void searchData(SearchSourceBuilder searchSourceBuilder, Map>> tmpCache) { String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index e2351860c..f67177fe4 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -136,10 +136,12 @@ public void onResponse(SearchResponse searchResponse) { try { while (true) { loadDataToCache(searchHits, rowList, cacheContent, copyCrow); + // determine if all results haven been ferched if (searchHits.length < getFetchSize()) { break; } if (tableInfo == null && tmpRhlClient == null) { + // create new connection to fetch data tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); tmpRhlClient = Es6Util.getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword()); } @@ -187,7 +189,6 @@ private void loadDataToCache(SearchHit[] searchHits, List rowList, List getRows(CRow inputRow, List cacheContent, List results) { List rowList = Lists.newArrayList(); for (Object line : results) { @@ -257,7 +258,6 @@ private SearchSourceBuilder initConfiguration() { return searchSourceBuilder; } - private BoolQueryBuilder setInputParams(List inputParams, BoolQueryBuilder boolQueryBuilder) { if (boolQueryBuilder == null) { boolQueryBuilder = new BoolQueryBuilder(); @@ -273,7 +273,6 @@ private BoolQueryBuilder setInputParams(List inputParams, BoolQueryBuild return boolQueryBuilder; } - public int getFetchSize() { return 1000; } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java index 0956cebf6..cc591789e 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java @@ -51,6 +51,7 @@ public class Es6Util { private static final String KEY_WORD_TYPE = ".keyword"; private static final String APOSTROPHE = "\'"; + // connect to the elasticsearch public static RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, String userName, String password) { List httpHostList = new ArrayList<>(); String[] address = StringUtils.split(esAddress, ","); @@ -96,6 +97,7 @@ public static RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh return rhlClient; } + // add index and type to search request public static SearchRequest setSearchRequest(SideInfo sideInfo) { SearchRequest searchRequest = new SearchRequest(); Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); @@ -126,6 +128,7 @@ public static SearchRequest setSearchRequest(SideInfo sideInfo) { return searchRequest; } + // build where cause public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { BoolQueryBuilder boolQueryBuilder = null; @@ -140,6 +143,7 @@ public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { return boolQueryBuilder; } + // build filter condition public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, SideInfo sideInfo) { switch (info.getOperatorKind()) { case "IN": @@ -184,6 +188,7 @@ public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBu } + // remove extra spaces and apostrophes public static String[] removeSpaceAndApostrophe(String str) { String[] split = StringUtils.split(str, ","); for (int i = 0; i < split.length; i++) { @@ -196,6 +201,7 @@ public static String[] removeSpaceAndApostrophe(String str) { return split; } + // prevent word segmentation public static String textConvertToKeyword(String fieldName, SideInfo sideInfo) { String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); From 97ce51bd6a5930ee83b2475c3eed1ae050af81c6 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Tue, 25 Feb 2020 09:46:12 +0800 Subject: [PATCH 044/523] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=80=E4=BA=9B?= =?UTF-8?q?=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/elasticsearch6/Elasticsearch6AllReqRow.java | 6 ------ .../sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java | 5 +++-- .../dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java | 6 ------ 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index 82a43de66..d9d9e9487 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -159,7 +159,6 @@ protected void initCache() throws SQLException { Map>> newCache = Maps.newConcurrentMap(); cacheRef.set(newCache); try { - // create search request and build where cause searchRequest = Es6Util.setSearchRequest(sideInfo); boolQueryBuilder = Es6Util.setPredicateclause(sideInfo); loadData(newCache); @@ -218,7 +217,6 @@ private void loadData(Map>> tmpCache) throws IO } } - // initialize searchSourceBuilder private SearchSourceBuilder initConfiguration(BoolQueryBuilder boolQueryBuilder){ SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); if (boolQueryBuilder != null) { @@ -227,8 +225,6 @@ private SearchSourceBuilder initConfiguration(BoolQueryBuilder boolQueryBuilder) searchSourceBuilder.size(getFetchSize()); searchSourceBuilder.sort("_id", SortOrder.DESC); - - // fields included in the source data String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); searchSourceBuilder.fetchSource(sideFieldNames, null); return searchSourceBuilder; @@ -244,7 +240,6 @@ private void searchData(SearchSourceBuilder searchSourceBuilder, Map>> tmpCache) { String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index f67177fe4..e2351860c 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -136,12 +136,10 @@ public void onResponse(SearchResponse searchResponse) { try { while (true) { loadDataToCache(searchHits, rowList, cacheContent, copyCrow); - // determine if all results haven been ferched if (searchHits.length < getFetchSize()) { break; } if (tableInfo == null && tmpRhlClient == null) { - // create new connection to fetch data tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); tmpRhlClient = Es6Util.getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword()); } @@ -189,6 +187,7 @@ private void loadDataToCache(SearchHit[] searchHits, List rowList, List getRows(CRow inputRow, List cacheContent, List results) { List rowList = Lists.newArrayList(); for (Object line : results) { @@ -258,6 +257,7 @@ private SearchSourceBuilder initConfiguration() { return searchSourceBuilder; } + private BoolQueryBuilder setInputParams(List inputParams, BoolQueryBuilder boolQueryBuilder) { if (boolQueryBuilder == null) { boolQueryBuilder = new BoolQueryBuilder(); @@ -273,6 +273,7 @@ private BoolQueryBuilder setInputParams(List inputParams, BoolQueryBuild return boolQueryBuilder; } + public int getFetchSize() { return 1000; } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java index cc591789e..0956cebf6 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java @@ -51,7 +51,6 @@ public class Es6Util { private static final String KEY_WORD_TYPE = ".keyword"; private static final String APOSTROPHE = "\'"; - // connect to the elasticsearch public static RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, String userName, String password) { List httpHostList = new ArrayList<>(); String[] address = StringUtils.split(esAddress, ","); @@ -97,7 +96,6 @@ public static RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh return rhlClient; } - // add index and type to search request public static SearchRequest setSearchRequest(SideInfo sideInfo) { SearchRequest searchRequest = new SearchRequest(); Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); @@ -128,7 +126,6 @@ public static SearchRequest setSearchRequest(SideInfo sideInfo) { return searchRequest; } - // build where cause public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { BoolQueryBuilder boolQueryBuilder = null; @@ -143,7 +140,6 @@ public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { return boolQueryBuilder; } - // build filter condition public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, SideInfo sideInfo) { switch (info.getOperatorKind()) { case "IN": @@ -188,7 +184,6 @@ public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBu } - // remove extra spaces and apostrophes public static String[] removeSpaceAndApostrophe(String str) { String[] split = StringUtils.split(str, ","); for (int i = 0; i < split.length; i++) { @@ -201,7 +196,6 @@ public static String[] removeSpaceAndApostrophe(String str) { return split; } - // prevent word segmentation public static String textConvertToKeyword(String fieldName, SideInfo sideInfo) { String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); From 9742e745c6e7bd15d6087ec6e37714679e685803 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Tue, 25 Feb 2020 09:51:14 +0800 Subject: [PATCH 045/523] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=80=E4=BA=9B?= =?UTF-8?q?=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/elasticsearch6/Elasticsearch6AllReqRow.java | 6 ++++++ .../sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java | 5 ++--- .../dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java | 6 ++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index d9d9e9487..82a43de66 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -159,6 +159,7 @@ protected void initCache() throws SQLException { Map>> newCache = Maps.newConcurrentMap(); cacheRef.set(newCache); try { + // create search request and build where cause searchRequest = Es6Util.setSearchRequest(sideInfo); boolQueryBuilder = Es6Util.setPredicateclause(sideInfo); loadData(newCache); @@ -217,6 +218,7 @@ private void loadData(Map>> tmpCache) throws IO } } + // initialize searchSourceBuilder private SearchSourceBuilder initConfiguration(BoolQueryBuilder boolQueryBuilder){ SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); if (boolQueryBuilder != null) { @@ -225,6 +227,8 @@ private SearchSourceBuilder initConfiguration(BoolQueryBuilder boolQueryBuilder) searchSourceBuilder.size(getFetchSize()); searchSourceBuilder.sort("_id", SortOrder.DESC); + + // fields included in the source data String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); searchSourceBuilder.fetchSource(sideFieldNames, null); return searchSourceBuilder; @@ -240,6 +244,7 @@ private void searchData(SearchSourceBuilder searchSourceBuilder, Map>> tmpCache) { String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index e2351860c..f67177fe4 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -136,10 +136,12 @@ public void onResponse(SearchResponse searchResponse) { try { while (true) { loadDataToCache(searchHits, rowList, cacheContent, copyCrow); + // determine if all results haven been ferched if (searchHits.length < getFetchSize()) { break; } if (tableInfo == null && tmpRhlClient == null) { + // create new connection to fetch data tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); tmpRhlClient = Es6Util.getClient(tableInfo.getAddress(), tableInfo.isAuthMesh(), tableInfo.getUserName(), tableInfo.getPassword()); } @@ -187,7 +189,6 @@ private void loadDataToCache(SearchHit[] searchHits, List rowList, List getRows(CRow inputRow, List cacheContent, List results) { List rowList = Lists.newArrayList(); for (Object line : results) { @@ -257,7 +258,6 @@ private SearchSourceBuilder initConfiguration() { return searchSourceBuilder; } - private BoolQueryBuilder setInputParams(List inputParams, BoolQueryBuilder boolQueryBuilder) { if (boolQueryBuilder == null) { boolQueryBuilder = new BoolQueryBuilder(); @@ -273,7 +273,6 @@ private BoolQueryBuilder setInputParams(List inputParams, BoolQueryBuild return boolQueryBuilder; } - public int getFetchSize() { return 1000; } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java index 0956cebf6..cc591789e 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java @@ -51,6 +51,7 @@ public class Es6Util { private static final String KEY_WORD_TYPE = ".keyword"; private static final String APOSTROPHE = "\'"; + // connect to the elasticsearch public static RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh, String userName, String password) { List httpHostList = new ArrayList<>(); String[] address = StringUtils.split(esAddress, ","); @@ -96,6 +97,7 @@ public static RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh return rhlClient; } + // add index and type to search request public static SearchRequest setSearchRequest(SideInfo sideInfo) { SearchRequest searchRequest = new SearchRequest(); Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); @@ -126,6 +128,7 @@ public static SearchRequest setSearchRequest(SideInfo sideInfo) { return searchRequest; } + // build where cause public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { BoolQueryBuilder boolQueryBuilder = null; @@ -140,6 +143,7 @@ public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { return boolQueryBuilder; } + // build filter condition public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, SideInfo sideInfo) { switch (info.getOperatorKind()) { case "IN": @@ -184,6 +188,7 @@ public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBu } + // remove extra spaces and apostrophes public static String[] removeSpaceAndApostrophe(String str) { String[] split = StringUtils.split(str, ","); for (int i = 0; i < split.length; i++) { @@ -196,6 +201,7 @@ public static String[] removeSpaceAndApostrophe(String str) { return split; } + // prevent word segmentation public static String textConvertToKeyword(String fieldName, SideInfo sideInfo) { String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); From 7e2e65b287a57936e21f3c2806ad6e311486dce9 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 25 Feb 2020 10:22:07 +0800 Subject: [PATCH 046/523] remove unused method --- core/src/main/java/com/dtstack/flink/sql/Main.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index 4ff296f1f..b912128f9 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -27,8 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.LoggerContext; /** * Date: 2018/6/26 @@ -45,10 +43,4 @@ public static void main(String[] args) throws Exception { env.execute(paramsInfo.getName()); LOG.info("program {} execution success", paramsInfo.getName()); } - private static void setLogLevel(String level){ - LoggerContext loggerContext= (LoggerContext) LoggerFactory.getILoggerFactory(); - //设置全局日志级别 - ch.qos.logback.classic.Logger logger = loggerContext.getLogger("root"); - logger.setLevel(Level.toLevel(level, Level.INFO)); - } } From dbd490abf1a3b27b214176eb86d45fdfa6279f7f Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Tue, 25 Feb 2020 11:20:37 +0800 Subject: [PATCH 047/523] =?UTF-8?q?es=E6=B7=BB=E5=8A=A0=E8=84=8F=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E6=8C=87=E6=A0=87=E7=9A=84=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/elasticsearch6/CustomerSinkFunc.java | 9 ++++++++- .../sink/elasticsearch6/MetricElasticsearch6Sink.java | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java index 1973cf01f..357e742c3 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java @@ -53,7 +53,9 @@ public class CustomerSinkFunc implements ElasticsearchSinkFunction { private List fieldTypes; - public transient Counter outRecords; + private transient Counter outRecords; + + private transient Counter outDirtyRecords; /** 默认分隔符为'_' */ private char sp = '_'; @@ -79,6 +81,7 @@ public void process(Tuple2 tuple2, RuntimeContext ctx, RequestIndexer indexer) { indexer.add(createIndexRequest(element)); outRecords.inc(); } catch (Throwable e) { + outDirtyRecords.inc(); logger.error("Failed to store source data {}. ", tuple2.getField(1)); logger.error("Failed to create index request exception. ", e); } @@ -88,6 +91,10 @@ public void setOutRecords(Counter outRecords) { this.outRecords = outRecords; } + public void setOutDirtyRecords(Counter outDirtyRecords) { + this.outDirtyRecords = outDirtyRecords; + } + private IndexRequest createIndexRequest(Row element) { List idFieldList = new ArrayList<>(); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java index 4415e7675..aa03e943a 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java @@ -63,7 +63,10 @@ public void open(Configuration parameters) throws Exception { public void initMetric() { Counter counter = getRuntimeContext().getMetricGroup().counter(MetricConstant.DT_NUM_RECORDS_OUT); + Counter outDirtyRecords = getRuntimeContext().getMetricGroup().counter(MetricConstant.DT_NUM_DIRTY_RECORDS_OUT); + customerSinkFunc.setOutRecords(counter); + customerSinkFunc.setOutDirtyRecords(outDirtyRecords); outRecordsRate = getRuntimeContext().getMetricGroup().meter(MetricConstant.DT_NUM_RECORDS_OUT_RATE, new MeterView(counter, 20)); } } From 86d05ee69c838e1d70e09719281006639c32e55e Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 25 Feb 2020 15:01:30 +0800 Subject: [PATCH 048/523] mongo scan --- docs/mongoSide.md | 5 +- docs/mongoSink.md | 7 +-- .../flink/sql/side/mongo/MongoAllReqRow.java | 48 ++++---------- .../sql/side/mongo/MongoAsyncReqRow.java | 62 ++++++------------- .../flink/sql/side/mongo/utils/MongoUtil.java | 1 + .../sql/sink/mongo/MongoOutputFormat.java | 34 +--------- 6 files changed, 38 insertions(+), 119 deletions(-) diff --git a/docs/mongoSide.md b/docs/mongoSide.md index db557f441..73ce9644f 100644 --- a/docs/mongoSide.md +++ b/docs/mongoSide.md @@ -40,8 +40,6 @@ |----|---|---|----| | type |表明 输出表类型 mongo|是|| | address | 连接mongo数据库 jdbcUrl |是|| - | userName | mongo连接用户名|否|| - | password | mongo连接密码|否|| | tableName | mongo表名称|是|| | database | mongo表名称|是|| | cache | 维表缓存策略(NONE/LRU)|否|NONE| @@ -64,7 +62,8 @@ create table sideTable( PERIOD FOR SYSTEM_TIME )WITH( type ='mongo', - address ='172.21.32.1:27017,172.21.32.1:27017', + //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] + address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', database ='test', tableName ='sidetest', cache ='LRU', diff --git a/docs/mongoSink.md b/docs/mongoSink.md index b0f916aab..b5bb6df0d 100644 --- a/docs/mongoSink.md +++ b/docs/mongoSink.md @@ -33,8 +33,6 @@ CREATE TABLE tableName( |----|----|----|----| |type |表明 输出表类型 mongo|是|| |address | 连接mongo数据库 jdbcUrl |是|| -|userName | mongo连接用户名|否|| -|password | mongo连接密码|否|| |tableName | mongo表名称|是|| |database | mongo表名称|是|| |parallelism | 并行度设置|否|1| @@ -46,9 +44,8 @@ CREATE TABLE MyResult( pv VARCHAR )WITH( type ='mongo', - address ='172.21.32.1:27017,172.21.32.1:27017', - userName ='dtstack', - password ='abc123', + //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] + address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', database ='test', tableName ='pv', parallelism ='1' diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index b416e4820..3229101c6 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -26,16 +26,13 @@ import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.mongodb.BasicDBObject; import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; -import com.mongodb.MongoCredential; -import com.mongodb.ServerAddress; +import com.mongodb.MongoClientURI; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -49,7 +46,6 @@ import java.sql.SQLException; import java.sql.Timestamp; -import java.util.ArrayList; import java.util.Calendar; import java.util.List; import java.util.Map; @@ -180,34 +176,13 @@ private String buildKey(Map val, List equalFieldList) { return sb.toString(); } - private MongoCollection getConn(String address, String userName, String password, String database, String tableName) { + private MongoCollection getConn(String address, String database, String tableName) { MongoCollection dbCollection; - try { - MongoCredential credential; - String[] servers = address.split(","); - String host; - Integer port; - String[] hostAndPort; - List lists = new ArrayList<>(); - for (String server : servers) { - hostAndPort = server.split(":"); - host = hostAndPort[0]; - port = Integer.parseInt(hostAndPort[1]); - lists.add(new ServerAddress(host, port)); - } - if (!StringUtils.isEmpty(userName) || !StringUtils.isEmpty(password)) { - credential = MongoCredential.createCredential(userName, database, password.toCharArray()); - // To connect to mongodb server - mongoClient = new MongoClient(lists, credential, new MongoClientOptions.Builder().build()); - } else { - mongoClient = new MongoClient(lists); - } - db = mongoClient.getDatabase(database); - dbCollection = db.getCollection(tableName, Document.class); - return dbCollection; - } catch (Exception e) { - throw new RuntimeException("[connMongoDB]:" + e.getMessage()); - } + mongoClient = new MongoClient(new MongoClientURI(address)); + db = mongoClient.getDatabase(database); + dbCollection = db.getCollection(tableName, Document.class); + return dbCollection; + } private void loadData(Map>> tmpCache) throws SQLException { @@ -217,8 +192,7 @@ private void loadData(Map>> tmpCache) throws SQ try { for (int i = 0; i < CONN_RETRY_NUM; i++) { try { - dbCollection = getConn(tableInfo.getAddress(), tableInfo.getUserName(), tableInfo.getPassword(), - tableInfo.getDatabase(), tableInfo.getTableName()); + dbCollection = getConn(tableInfo.getAddress(), tableInfo.getDatabase(), tableInfo.getTableName()); break; } catch (Exception e) { if (i == CONN_RETRY_NUM - 1) { @@ -237,9 +211,9 @@ private void loadData(Map>> tmpCache) throws SQ //load data from table String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); - BasicDBObject basicDBObject = new BasicDBObject(); + BasicDBObject basicDbObject = new BasicDBObject(); for (String selectField : sideFieldNames) { - basicDBObject.append(selectField, 1); + basicDbObject.append(selectField, 1); } BasicDBObject filterObject = new BasicDBObject(); try { @@ -256,7 +230,7 @@ private void loadData(Map>> tmpCache) throws SQ } - FindIterable findIterable = dbCollection.find(filterObject).projection(basicDBObject).limit(FETCH_SIZE); + FindIterable findIterable = dbCollection.find(filterObject).projection(basicDbObject).limit(FETCH_SIZE); MongoCursor mongoCursor = findIterable.iterator(); while (mongoCursor.hasNext()) { Document doc = mongoCursor.next(); diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 8ac7d83ee..5ef06c80e 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -29,11 +29,12 @@ import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.mongodb.BasicDBObject; import com.mongodb.Block; +import com.mongodb.ConnectionString; import com.mongodb.MongoCredential; import com.mongodb.ServerAddress; import com.mongodb.async.SingleResultCallback; import com.mongodb.async.client.MongoClient; -import com.mongodb.async.client.MongoClientSettings; +import com.mongodb.MongoClientSettings; import com.mongodb.async.client.MongoClients; import com.mongodb.async.client.MongoCollection; import com.mongodb.async.client.MongoDatabase; @@ -70,13 +71,11 @@ public class MongoAsyncReqRow extends AsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(MongoAsyncReqRow.class); - private final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = 20; - private transient MongoClient mongoClient; private MongoDatabase db; - private MongoSideTableInfo MongoSideTableInfo; + private MongoSideTableInfo mongoSideTableInfo; public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new MongoAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); @@ -85,48 +84,25 @@ public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List lists = new ArrayList<>(); - for (String server : servers) { - hostAndPort = server.split(":"); - host = hostAndPort[0]; - port = Integer.parseInt(hostAndPort[1]); - lists.add(new ServerAddress(host, port)); - } - ClusterSettings clusterSettings = ClusterSettings.builder().hosts(lists).build(); - ConnectionPoolSettings connectionPoolSettings = ConnectionPoolSettings.builder() - .maxSize(DEFAULT_MAX_DB_CONN_POOL_SIZE) + public void connMongoDb() throws Exception { + String address = mongoSideTableInfo.getAddress(); + ConnectionString connectionString = new ConnectionString(address); + + MongoClientSettings settings = MongoClientSettings.builder() + .applyConnectionString(connectionString) .build(); - if (!StringUtils.isEmpty(MongoSideTableInfo.getUserName()) || !StringUtils.isEmpty(MongoSideTableInfo.getPassword())) { - mongoCredential = MongoCredential.createCredential(MongoSideTableInfo.getUserName(), MongoSideTableInfo.getDatabase(), - MongoSideTableInfo.getPassword().toCharArray()); - MongoClientSettings settings = MongoClientSettings.builder().credential(mongoCredential) - .clusterSettings(clusterSettings) - .connectionPoolSettings(connectionPoolSettings) - .build(); - mongoClient = MongoClients.create(settings); - } else { - MongoClientSettings settings = MongoClientSettings.builder().clusterSettings(clusterSettings) - .connectionPoolSettings(connectionPoolSettings) - .build(); - mongoClient = MongoClients.create(settings); - } - db = mongoClient.getDatabase(MongoSideTableInfo.getDatabase()); + mongoClient = MongoClients.create(settings); + db = mongoClient.getDatabase(mongoSideTableInfo.getDatabase()); } @Override public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { CRow inputCopy = new CRow(input.row(), input.change()); - BasicDBObject basicDBObject = new BasicDBObject(); + BasicDBObject basicDbObject = new BasicDBObject(); for (int i = 0; i < sideInfo.getEqualFieldList().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = inputCopy.row().getField(conValIndex); @@ -134,14 +110,14 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce dealMissKey(inputCopy, resultFuture); return; } - basicDBObject.put(sideInfo.getEqualFieldList().get(i), equalObj); + basicDbObject.put(sideInfo.getEqualFieldList().get(i), equalObj); } try { // 填充谓词 sideInfo.getSideTableInfo().getPredicateInfoes().stream().map(info -> { BasicDBObject filterCondition = MongoUtil.buildFilterObject(info); if (null != filterCondition) { - basicDBObject.append(info.getFieldName(), filterCondition); + basicDbObject.append(info.getFieldName(), filterCondition); } return info; }).count(); @@ -149,7 +125,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce LOG.info("add predicate infoes error ", e); } - String key = buildCacheKey(basicDBObject.values()); + String key = buildCacheKey(basicDbObject.values()); if (openCache()) { CacheObj val = getFromCache(key); if (val != null) { @@ -171,7 +147,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce } } AtomicInteger atomicInteger = new AtomicInteger(0); - MongoCollection dbCollection = db.getCollection(MongoSideTableInfo.getTableName(), Document.class); + MongoCollection dbCollection = db.getCollection(mongoSideTableInfo.getTableName(), Document.class); List cacheContent = Lists.newArrayList(); Block printDocumentBlock = new Block() { @Override @@ -197,7 +173,7 @@ public void onResult(final Void result, final Throwable t) { } } }; - dbCollection.find(basicDBObject).forEach(printDocumentBlock, callbackWhenFinished); + dbCollection.find(basicDbObject).forEach(printDocumentBlock, callbackWhenFinished); } @Override diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java index 6e0177759..4a9b77b9f 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java @@ -55,6 +55,7 @@ public static BasicDBObject buildFilterObject(PredicateInfo info) { return new BasicDBObject("$exists", true); case "IS NULL": return new BasicDBObject("$exists", false); + default: } return null; } diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index 82bc9c15f..daaded15e 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -21,13 +21,10 @@ import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; -import com.mongodb.MongoCredential; -import com.mongodb.ServerAddress; +import com.mongodb.MongoClientURI; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.result.UpdateResult; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; @@ -37,9 +34,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; /** * Reason: @@ -124,30 +118,8 @@ public void close() { } private void establishConnection() { - try { - MongoCredential credential; - String[] servers = address.split(","); - String host; - Integer port; - String[] hostAndPort; - List lists = new ArrayList<>(); - for (String server : servers) { - hostAndPort = server.split(":"); - host = hostAndPort[0]; - port = Integer.parseInt(hostAndPort[1]); - lists.add(new ServerAddress(host, port)); - } - if (!StringUtils.isEmpty(userName) || !StringUtils.isEmpty(password)) { - credential = MongoCredential.createCredential(userName, database, password.toCharArray()); - // To connect to mongodb server - mongoClient = new MongoClient(lists, credential, new MongoClientOptions.Builder().build()); - } else { - mongoClient = new MongoClient(lists); - } - db = mongoClient.getDatabase(database); - } catch (Exception e) { - throw new IllegalArgumentException("[connMongoDB]:" + e.getMessage()); - } + mongoClient = new MongoClient(new MongoClientURI(address)); + db = mongoClient.getDatabase(database); } private MongoOutputFormat() { From 23317adc5fa5bdcc95d79005351762b353c6c5a5 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 25 Feb 2020 15:15:38 +0800 Subject: [PATCH 049/523] =?UTF-8?q?=E9=87=8D=E5=90=AF=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0failure.interval=E5=92=8Cdelay.interval?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 +++ .../flink/sql/constrant/ConfigConstrant.java | 4 ++-- .../sql/environment/StreamEnvConfigManager.java | 13 +++++++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 583958908..a5b0457ab 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,9 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack * taskmanager.slots：per_job模式下指定每个taskmanager对应的slot数量(默认1) * savePointPath：任务恢复点的路径（默认无） * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） + * restore.enable：是否失败重启（默认是true） + * failure.interval：衡量失败率的时间段，单位分钟（默认6m） + * delay.interval：连续两次重启尝试间的间隔，单位是秒（默认10s） * [prometheus 相关参数](docs/prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java index 0dc060d79..a80962661 100644 --- a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java @@ -69,8 +69,8 @@ public class ConfigConstrant { // restart plocy public static final int failureRate = 3; - public static final int failureInterval = 6; //min + public static final String FAILUREINTERVAL = "failure.interval"; //min - public static final int delayInterval = 10; //sec + public static final String DELAYINTERVAL= "delay.interval"; //sec } diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index 9da34bcc2..28eac2af3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -104,8 +104,8 @@ public static void streamExecutionEnvironmentConfig(StreamExecutionEnvironment s if(isRestore(confProperties).get()){ streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( ConfigConstrant.failureRate, - Time.of(ConfigConstrant.failureInterval, TimeUnit.MINUTES), - Time.of(ConfigConstrant.delayInterval, TimeUnit.SECONDS) + Time.of(getFailureInterval(confProperties).get(), TimeUnit.MINUTES), + Time.of(getDelayInterval(confProperties).get(), TimeUnit.SECONDS) )); } else { streamEnv.setRestartStrategy(RestartStrategies.noRestart()); @@ -172,6 +172,15 @@ public static Optional isRestore(Properties properties){ return Optional.of(Boolean.valueOf(restoreEnable)); } + public static Optional getDelayInterval(Properties properties){ + String delayInterval = properties.getProperty(ConfigConstrant.DELAYINTERVAL, "10"); + return Optional.of(Integer.valueOf(delayInterval)); + } + public static Optional getFailureInterval(Properties properties){ + String failureInterval = properties.getProperty(ConfigConstrant.FAILUREINTERVAL, "6"); + return Optional.of(Integer.valueOf(failureInterval)); + } + /** * #ProcessingTime(默认), IngestionTime, EventTime * @param properties From 40460cb57da1ac1bc548112f65fade396976a615 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 25 Feb 2020 15:56:10 +0800 Subject: [PATCH 050/523] rdb scan --- .../side/clickhouse/ClickhouseAllReqRow.java | 7 +- .../clickhouse/ClickhouseAsyncReqRow.java | 2 +- .../sql/sink/clickhouse/ClickhouseSink.java | 9 +- .../flink/sql/side/db2/Db2AllReqRow.java | 7 +- .../flink/sql/side/db2/Db2AsyncReqRow.java | 2 +- .../com/dtstack/flink/sql/sink/db/DbSink.java | 9 +- .../sql/side/impala/ImpalaAllReqRow.java | 7 +- .../sql/side/impala/ImpalaAsyncReqRow.java | 2 +- .../flink/sql/sink/impala/ImpalaSink.java | 12 +- .../flink/sql/side/mysql/MysqlAllReqRow.java | 8 +- .../sql/side/mysql/MysqlAsyncReqRow.java | 2 +- .../flink/sql/sink/mysql/MysqlSink.java | 6 +- .../sql/side/oracle/OracleAllReqRow.java | 8 +- .../sql/side/oracle/OracleAsyncReqRow.java | 2 +- .../flink/sql/sink/oracle/OracleSink.java | 6 +- .../sql/side/polardb/PolardbAllReqRow.java | 30 +- .../sql/side/polardb/PolardbAllSideInfo.java | 22 ++ .../sql/side/polardb/PolardbAsyncReqRow.java | 24 +- .../side/polardb/PolardbAsyncSideInfo.java | 22 ++ .../side/polardb/table/PolardbSideParser.java | 23 ++ .../flink/sql/sink/polardb/PolardbSink.java | 30 +- .../sink/polardb/table/PolardbSinkParser.java | 22 ++ .../side/postgresql/PostgresqlAllReqRow.java | 8 +- .../postgresql/PostgresqlAsyncReqRow.java | 2 +- .../sql/sink/postgresql/PostgresqlSink.java | 6 +- .../flink/sql/side/rdb/all/RdbAllReqRow.java | 228 ------------ .../sql/side/rdb/async/RdbAsyncReqRow.java | 24 +- .../provider/DTC3P0DataSourceProvider.java | 10 +- .../flink/sql/side/rdb/util/SwitchUtil.java | 1 + .../{RdbSink.java => AbstractRdbSink.java} | 18 +- .../flink/sql/sink/rdb/JDBCOptions.java | 22 +- .../rdb/format/AbstractJDBCOutputFormat.java | 10 +- .../rdb/format/JDBCUpsertOutputFormat.java | 15 +- .../sql/sink/rdb/writer/AppendOnlyWriter.java | 9 +- .../sql/sink/rdb/writer/UpsertWriter.java | 338 ------------------ .../side/sqlserver/SqlserverAllReqRow.java | 16 +- .../side/sqlserver/SqlserverAllSideInfo.java | 6 +- .../side/sqlserver/SqlserverAsyncReqRow.java | 8 +- .../sqlserver/SqlserverAsyncSideInfo.java | 7 +- .../sqlserver/table/SqlserverSideParser.java | 6 +- .../sql/sink/sqlserver/SqlserverSink.java | 6 +- 41 files changed, 296 insertions(+), 706 deletions(-) delete mode 100644 rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java rename rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/{RdbSink.java => AbstractRdbSink.java} (90%) delete mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java index 68c0c7984..a2bf8add9 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java @@ -21,20 +21,17 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; -import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Connection; import java.sql.DriverManager; import java.util.List; -import java.util.Map; -public class ClickhouseAllReqRow extends RdbAllReqRow { +public class ClickhouseAllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(ClickhouseAllReqRow.class); diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index 305d65118..3733d7350 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -58,7 +58,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); } } diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java index bbb6a6fe4..5f7e41f7b 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java @@ -22,14 +22,11 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import java.util.List; -import java.util.Map; - -public class ClickhouseSink extends RdbSink implements IStreamSinkGener { +public class ClickhouseSink extends AbstractRdbSink implements IStreamSinkGener { public ClickhouseSink() { super(new ClickhouseDialect()); } @@ -37,7 +34,7 @@ public ClickhouseSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java index a77201b80..e64e55987 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java @@ -21,9 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.google.common.collect.Maps; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +29,6 @@ import java.sql.Connection; import java.sql.DriverManager; import java.util.List; -import java.util.Map; /** * Date: 2019/11/20 @@ -40,7 +37,7 @@ * @author xiuzhu */ -public class Db2AllReqRow extends RdbAllReqRow { +public class Db2AllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(Db2AllReqRow.class); diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index 3d85c6fbd..59571aa56 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -74,7 +74,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, db2lientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, db2lientConfig)); } } diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java index 9942d4438..da1434ace 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java @@ -1,13 +1,10 @@ package com.dtstack.flink.sql.sink.db; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import java.util.List; -import java.util.Map; - -public class DbSink extends RdbSink { +public class DbSink extends AbstractRdbSink { public DbSink() { super(new DbDialect()); @@ -15,7 +12,7 @@ public DbSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java index 9cd8c9194..4118421ee 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java @@ -22,10 +22,8 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; -import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.JDBCUtils; -import com.google.common.collect.Maps; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -36,7 +34,6 @@ import java.sql.Connection; import java.sql.DriverManager; import java.util.List; -import java.util.Map; /** * side operator with cache for all(period reload) @@ -46,7 +43,7 @@ * @author xiuzhu */ -public class ImpalaAllReqRow extends RdbAllReqRow { +public class ImpalaAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 63d53b80d..1b983ca2c 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -66,7 +66,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); } public JsonObject getClientConfig() { diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index 41fa00926..91b6532bf 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.impala.table.ImpalaTableInfo; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.hadoop.conf.Configuration; @@ -36,7 +36,7 @@ * @author xiuzhu */ -public class ImpalaSink extends RdbSink implements IStreamSinkGener { +public class ImpalaSink extends AbstractRdbSink implements IStreamSinkGener { private ImpalaTableInfo impalaTableInfo; @@ -47,7 +47,7 @@ public ImpalaSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(getImpalaJdbcUrl()) + .setDbUrl(getImpalaJdbcUrl()) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) @@ -70,8 +70,8 @@ public JDBCUpsertOutputFormat getOutputFormat() { public String getImpalaJdbcUrl() { Integer authMech = impalaTableInfo.getAuthMech(); - String newUrl = dbURL; - StringBuffer urlBuffer = new StringBuffer(dbURL); + String newUrl = dbUrl; + StringBuffer urlBuffer = new StringBuffer(dbUrl); if (authMech == EAuthMech.NoAuthentication.getType()) { return newUrl; } else if (authMech == EAuthMech.Kerberos.getType()) { @@ -121,7 +121,7 @@ public String getImpalaJdbcUrl() { } @Override - public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { + public AbstractRdbSink genStreamSink(TargetTableInfo targetTableInfo) { super.genStreamSink(targetTableInfo); this.impalaTableInfo = (ImpalaTableInfo) targetTableInfo; return this; diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java index b6b7f45cb..8cf6ab597 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; @@ -41,7 +41,7 @@ * @author xuchao */ -public class MysqlAllReqRow extends RdbAllReqRow { +public class MysqlAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -54,13 +54,13 @@ public MysqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index 6120767ef..2d2f5a72a 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -76,7 +76,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); } } diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java index 8c1146ede..0a1749f04 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; @@ -33,7 +33,7 @@ * @author xuchao */ -public class MysqlSink extends RdbSink implements IStreamSinkGener { +public class MysqlSink extends AbstractRdbSink implements IStreamSinkGener { public MysqlSink() { super(new MySQLDialect()); @@ -42,7 +42,7 @@ public MysqlSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java index 18d9ba045..5477514a5 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; @@ -36,7 +36,7 @@ /** * side operator with cache for all(period reload) */ -public class OracleAllReqRow extends RdbAllReqRow { +public class OracleAllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(OracleAllReqRow.class); @@ -47,11 +47,11 @@ public OracleAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List addParams = Maps.newHashMap(); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index f973ffee2..f6827f41b 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -68,6 +68,6 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); } } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index ee0239a1f..2257d2815 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** @@ -29,7 +29,7 @@ * * @author maqi */ -public class OracleSink extends RdbSink implements IStreamSinkGener { +public class OracleSink extends AbstractRdbSink implements IStreamSinkGener { public OracleSink() { super(new OracleDialect()); @@ -38,7 +38,7 @@ public OracleSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java index f9e0950dd..a5eec511b 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java @@ -1,9 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.collect.Maps; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -15,7 +32,12 @@ import java.util.List; import java.util.Map; -public class PolardbAllReqRow extends RdbAllReqRow { +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ +public class PolardbAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -28,13 +50,13 @@ public PolardbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java index 2802bd402..fa2ad63e7 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; @@ -8,6 +25,11 @@ import java.util.List; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbAllSideInfo extends RdbAllSideInfo { public PolardbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 3cbb0b368..0f8f82738 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; @@ -16,6 +33,11 @@ import java.util.List; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbAsyncReqRow extends RdbAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(PolardbAsyncReqRow.class); @@ -48,6 +70,6 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); } } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java index 0b47cf3f7..f6afb8da8 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; @@ -8,6 +25,11 @@ import java.util.List; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbAsyncSideInfo extends RdbAsyncSideInfo { public PolardbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { diff --git a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java index 348b3aed9..1afb661f0 100644 --- a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java +++ b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java @@ -1,3 +1,21 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; @@ -5,6 +23,11 @@ import java.util.Map; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbSideParser extends RdbSideParser { private static final String CURR_TYPE = "polardb"; diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java index 44d558140..c69f31e8f 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java @@ -1,11 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.dtstack.flink.sql.sink.polardb; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; - -public class PolardbSink extends RdbSink { +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ +public class PolardbSink extends AbstractRdbSink { public PolardbSink() { super(new PolardbDialect()); } @@ -13,7 +35,7 @@ public PolardbSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java index 68eead938..a57089941 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.sink.polardb.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; @@ -5,6 +22,11 @@ import java.util.Map; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "polardb"; diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java index 6d68cfdca..1c9aecfe1 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; @@ -41,7 +41,7 @@ * @author tcm */ -public class PostgresqlAllReqRow extends RdbAllReqRow { +public class PostgresqlAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -54,13 +54,13 @@ public PostgresqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index 02b333819..a26c8b229 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -68,7 +68,7 @@ public void open(Configuration parameters) throws Exception { vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, pgClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, pgClientConfig)); } } diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java index cf45a6bd9..bb731152c 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java @@ -22,13 +22,13 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** * @author maqi */ -public class PostgresqlSink extends RdbSink implements IStreamSinkGener { +public class PostgresqlSink extends AbstractRdbSink implements IStreamSinkGener { public PostgresqlSink() { super(new PostgresqlDialect()); } @@ -36,7 +36,7 @@ public PostgresqlSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java deleted file mode 100644 index 48d2407aa..000000000 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.side.rdb.all; - -import com.dtstack.flink.sql.side.AllReqRow; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; -import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; -import org.apache.calcite.sql.JoinType; -import org.apache.commons.collections.CollectionUtils; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Timestamp; -import java.util.Calendar; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; - -/** - * side operator with cache for all(period reload) - * Date: 2018/11/26 - * Company: www.dtstack.com - * - * @author maqi - */ - -public abstract class RdbAllReqRow extends AllReqRow { - - private static final long serialVersionUID = 2098635140857937718L; - - private static final Logger LOG = LoggerFactory.getLogger(RdbAllReqRow.class); - - private static final int CONN_RETRY_NUM = 3; - - private AtomicReference>>> cacheRef = new AtomicReference<>(); - - public RdbAllReqRow(SideInfo sideInfo) { - super(sideInfo); - } - - @Override - public Row fillData(Row input, Object sideInput) { - Map cacheInfo = (Map) sideInput; - Row row = new Row(sideInfo.getOutFieldInfoList().size()); - for (Map.Entry entry : sideInfo.getInFieldIndex().entrySet()) { - Object obj = input.getField(entry.getValue()); - boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); - - //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. - if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); - } - - row.setField(entry.getKey(), obj); - } - - for (Map.Entry entry : sideInfo.getSideFieldNameIndex().entrySet()) { - if (cacheInfo == null) { - row.setField(entry.getKey(), null); - } else { - row.setField(entry.getKey(), cacheInfo.get(entry.getValue())); - } - } - - return row; - } - - @Override - protected void initCache() throws SQLException { - Map>> newCache = Maps.newConcurrentMap(); - cacheRef.set(newCache); - loadData(newCache); - } - - @Override - protected void reloadCache() { - //reload cacheRef and replace to old cacheRef - Map>> newCache = Maps.newConcurrentMap(); - try { - loadData(newCache); - } catch (SQLException e) { - LOG.error("", e); - } - - cacheRef.set(newCache); - LOG.info("----- rdb all cacheRef reload end:{}", Calendar.getInstance()); - } - - - @Override - public void flatMap(CRow value, Collector out) throws Exception { - List inputParams = Lists.newArrayList(); - for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = value.row().getField(conValIndex); - if (equalObj == null) { - if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value.row(), null); - out.collect(new CRow(row, value.change())); - } - return; - } - inputParams.add(equalObj); - } - - String key = buildKey(inputParams); - List> cacheList = cacheRef.get().get(key); - if (CollectionUtils.isEmpty(cacheList)) { - if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value.row(), null); - out.collect(new CRow(row, value.change())); - } else { - return; - } - - return; - } - - for (Map one : cacheList) { - out.collect(new CRow(fillData(value.row(), one), value.change())); - } - } - - private String buildKey(List equalValList) { - StringBuilder sb = new StringBuilder(""); - for (Object equalVal : equalValList) { - sb.append(equalVal).append("_"); - } - - return sb.toString(); - } - - private String buildKey(Map val, List equalFieldList) { - StringBuilder sb = new StringBuilder(""); - for (String equalField : equalFieldList) { - sb.append(val.get(equalField)).append("_"); - } - - return sb.toString(); - } - - public abstract Connection getConn(String dbURL, String userName, String password); - - - private void loadData(Map>> tmpCache) throws SQLException { - RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - Connection connection = null; - - try { - for (int i = 0; i < CONN_RETRY_NUM; i++) { - try { - connection = getConn(tableInfo.getUrl(), tableInfo.getUserName(), tableInfo.getPassword()); - break; - } catch (Exception e) { - if (i == CONN_RETRY_NUM - 1) { - throw new RuntimeException("", e); - } - try { - String connInfo = "url:" + tableInfo.getUrl() + ";userName:" + tableInfo.getUserName() + ",pwd:" + tableInfo.getPassword(); - LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); - Thread.sleep(5 * 1000); - } catch (InterruptedException e1) { - LOG.error("", e1); - } - } - - } - - //load data from table - String sql = sideInfo.getSqlCondition(); - Statement statement = connection.createStatement(); - statement.setFetchSize(getFetchSize()); - ResultSet resultSet = statement.executeQuery(sql); - String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); - String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); - while (resultSet.next()) { - Map oneRow = Maps.newHashMap(); - for (String fieldName : sideFieldNames) { - Object object = resultSet.getObject(fieldName.trim()); - int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); - object = SwitchUtil.getTarget(object, fields[fieldIndex]); - oneRow.put(fieldName.trim(), object); - } - - String cacheKey = buildKey(oneRow, sideInfo.getEqualFieldList()); - List> list = tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()); - list.add(oneRow); - } - } catch (Exception e) { - LOG.error("", e); - } finally { - if (connection != null) { - connection.close(); - } - } - } - - public int getFetchSize() { - return 1000; - } - -} diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 7123e257b..bf7f8f774 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -20,16 +20,12 @@ package com.dtstack.flink.sql.side.rdb.async; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; -import io.vertx.core.AsyncResult; -import io.vertx.core.Handler; -import io.vertx.core.Vertx; -import io.vertx.core.VertxOptions; import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.jdbc.JDBCClient; import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import com.google.common.collect.Lists; @@ -41,8 +37,6 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -73,7 +67,7 @@ public class RdbAsyncReqRow extends AsyncReqRow { public final static String PREFERRED_TEST_QUERY_SQL = "select 1 from dual"; - private transient SQLClient rdbSQLClient; + private transient SQLClient rdbSqlClient; public RdbAsyncReqRow(SideInfo sideInfo) { super(sideInfo); @@ -113,7 +107,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce } } - rdbSQLClient.getConnection(conn -> { + rdbSqlClient.getConnection(conn -> { if (conn.failed()) { //Treatment failures resultFuture.completeExceptionally(conn.cause()); @@ -195,8 +189,8 @@ public Row fillData(Row input, Object line) { @Override public void close() throws Exception { super.close(); - if (rdbSQLClient != null) { - rdbSQLClient.close(); + if (rdbSqlClient != null) { + rdbSqlClient.close(); } } @@ -211,8 +205,8 @@ public String buildCacheKey(JsonArray jsonArray) { return sb.toString(); } - public void setRdbSQLClient(SQLClient rdbSQLClient) { - this.rdbSQLClient = rdbSQLClient; + public void setRdbSqlClient(SQLClient rdbSqlClient) { + this.rdbSqlClient = rdbSqlClient; } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java index fcca4f0ef..542258257 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java @@ -25,13 +25,19 @@ import javax.sql.DataSource; import java.beans.PropertyVetoException; import java.sql.SQLException; - +/** + * Date: 2019/9/17 + * Company: www.dtstack.com + * @author maqi + */ public class DTC3P0DataSourceProvider extends C3P0DataSourceProvider { @Override public DataSource getDataSource(JsonObject config) throws SQLException { String url = config.getString("url"); - if (url == null) throw new NullPointerException("url cannot be null"); + if (url == null) { + throw new NullPointerException("url cannot be null"); + } String driverClass = config.getString("driver_class"); String user = config.getString("user"); String password = config.getString("password"); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java index 1205c5521..49d163d4d 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java @@ -78,6 +78,7 @@ public static Object getTarget(Object obj, String targetType) { case "timestamp": case "datetime": return MathUtil.getTimestamp(obj); + default: } return obj; } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java similarity index 90% rename from rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java rename to rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java index c65696903..b7990f9f4 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java @@ -21,7 +21,6 @@ import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -34,10 +33,6 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import java.io.Serializable; -import java.math.BigDecimal; -import java.sql.Date; -import java.sql.Timestamp; -import java.sql.Types; import java.util.Arrays; import java.util.List; @@ -48,11 +43,8 @@ * * @author maqi */ -public abstract class RdbSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { - - protected String driverName; - - protected String dbURL; +public abstract class AbstractRdbSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { + protected String dbUrl; protected String userName; @@ -88,18 +80,18 @@ public abstract class RdbSink implements RetractStreamTableSink, Serializab protected String updateMode; - public RdbSink(JDBCDialect jdbcDialect) { + public AbstractRdbSink(JDBCDialect jdbcDialect) { this.jdbcDialect = jdbcDialect; } @Override - public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { + public AbstractRdbSink genStreamSink(TargetTableInfo targetTableInfo) { RdbTableInfo rdbTableInfo = (RdbTableInfo) targetTableInfo; this.batchNum = rdbTableInfo.getBatchSize() == null ? batchNum : rdbTableInfo.getBatchSize(); this.batchWaitInterval = rdbTableInfo.getBatchWaitInterval() == null ? batchWaitInterval : rdbTableInfo.getBatchWaitInterval(); this.parallelism = rdbTableInfo.getParallelism() == null ? parallelism : rdbTableInfo.getParallelism(); - this.dbURL = rdbTableInfo.getUrl(); + this.dbUrl = rdbTableInfo.getUrl(); this.userName = rdbTableInfo.getUserName(); this.password = rdbTableInfo.getPassword(); this.tableName = rdbTableInfo.getTableName(); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java index e9a4f391f..f476bfadf 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java @@ -29,7 +29,7 @@ public class JDBCOptions { - private String dbURL; + private String dbUrl; private String tableName; private String driverName; private String username; @@ -37,9 +37,9 @@ public class JDBCOptions { private String schema; private JDBCDialect dialect; - private JDBCOptions(String dbURL, String tableName, String driverName, String username, + private JDBCOptions(String dbUrl, String tableName, String driverName, String username, String password, String schema, JDBCDialect dialect) { - this.dbURL = dbURL; + this.dbUrl = dbUrl; this.tableName = tableName; this.driverName = driverName; this.username = username; @@ -48,8 +48,8 @@ private JDBCOptions(String dbURL, String tableName, String driverName, String us this.dialect = dialect; } - public String getDbURL() { - return dbURL; + public String getDbUrl() { + return dbUrl; } public String getTableName() { @@ -84,7 +84,7 @@ public static Builder builder() { public boolean equals(Object o) { if (o instanceof JDBCOptions) { JDBCOptions options = (JDBCOptions) o; - return Objects.equals(dbURL, options.dbURL) && + return Objects.equals(dbUrl, options.dbUrl) && Objects.equals(tableName, options.tableName) && Objects.equals(driverName, options.driverName) && Objects.equals(username, options.username) && @@ -100,7 +100,7 @@ public boolean equals(Object o) { * Builder of {@link JDBCOptions}. */ public static class Builder { - private String dbURL; + private String dbUrl; private String tableName; private String driverName; private String username; @@ -152,8 +152,8 @@ public Builder setSchema(String schema) { /** * required, JDBC DB url. */ - public Builder setDBUrl(String dbURL) { - this.dbURL = dbURL; + public Builder setDbUrl(String dbUrl) { + this.dbUrl = dbUrl; return this; } @@ -163,7 +163,7 @@ public Builder setDialect(JDBCDialect dialect) { } public JDBCOptions build() { - checkNotNull(dbURL, "No dbURL supplied."); + checkNotNull(dbUrl, "No dbURL supplied."); checkNotNull(tableName, "No tableName supplied."); if (this.driverName == null) { @@ -173,7 +173,7 @@ public JDBCOptions build() { }); } - return new JDBCOptions(dbURL, tableName, driverName, username, password, schema, dialect); + return new JDBCOptions(dbUrl, tableName, driverName, username, password, schema, dialect); } } } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java index 73f17d8fd..3378ff3f2 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java @@ -47,16 +47,16 @@ public abstract class AbstractJDBCOutputFormat extends DtRichOutputFormat protected final String username; protected final String password; - private final String drivername; + private final String driverName; protected final String dbURL; protected transient Connection connection; - public AbstractJDBCOutputFormat(String username, String password, String drivername, String dbURL) { + public AbstractJDBCOutputFormat(String username, String password, String driverName, String dbUrl) { this.username = username; this.password = password; - this.drivername = drivername; - this.dbURL = dbURL; + this.driverName = driverName; + this.dbURL = dbUrl; } @Override @@ -64,7 +64,7 @@ public void configure(Configuration parameters) { } protected void establishConnection() throws SQLException, ClassNotFoundException, IOException { - JDBCUtils.forName(drivername, getClass().getClassLoader()); + JDBCUtils.forName(driverName, getClass().getClassLoader()); if (username == null) { connection = DriverManager.getConnection(dbURL); } else { diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index ff7b24979..b13b39537 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -25,7 +25,7 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.dtstack.flink.sql.sink.rdb.writer.AppendOnlyWriter; import com.dtstack.flink.sql.sink.rdb.writer.JDBCWriter; -import com.dtstack.flink.sql.sink.rdb.writer.UpsertWriter; +import com.dtstack.flink.sql.sink.rdb.writer.AbstractUpsertWriter; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; @@ -44,6 +44,7 @@ /** * An upsert OutputFormat for JDBC. + * @author maqi */ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat> { @@ -83,7 +84,7 @@ public JDBCUpsertOutputFormat( long flushIntervalMills, boolean allReplace, String updateMode) { - super(options.getUsername(), options.getPassword(), options.getDriverName(), options.getDbURL()); + super(options.getUsername(), options.getPassword(), options.getDriverName(), options.getDbUrl()); this.schema = options.getSchema(); this.tableName = options.getTableName(); this.dialect = options.getDialect(); @@ -111,12 +112,12 @@ public void open(int taskNumber, int numTasks) throws IOException { initMetric(); if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.APPEND.name()) || keyFields == null || keyFields.length == 0) { - String insertSQL = dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields); - LOG.info("execute insert sql： {}", insertSQL); - System.out.println("execute insert sql :" + insertSQL); - jdbcWriter = new AppendOnlyWriter(insertSQL, fieldTypes, this); + String insertSql = dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields); + LOG.info("execute insert sql： {}", insertSql); + System.out.println("execute insert sql :" + insertSql); + jdbcWriter = new AppendOnlyWriter(insertSql, fieldTypes, this); } else { - jdbcWriter = UpsertWriter.create( + jdbcWriter = AbstractUpsertWriter.create( dialect, schema, tableName, fieldNames, fieldTypes, keyFields, partitionFields, getRuntimeContext().getExecutionConfig().isObjectReuseEnabled(), allReplace, this); } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 59eee0858..2aa7b01f9 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -34,6 +34,7 @@ /** * Just append record to jdbc, can not receive retract/delete message. + * @author maqi */ public class AppendOnlyWriter implements JDBCWriter { @@ -41,7 +42,7 @@ public class AppendOnlyWriter implements JDBCWriter { private static final Logger LOG = LoggerFactory.getLogger(AppendOnlyWriter.class); - private final String insertSQL; + private final String insertSql; private final int[] fieldTypes; private transient PreparedStatement statement; @@ -49,8 +50,8 @@ public class AppendOnlyWriter implements JDBCWriter { // only use metric private transient DtRichOutputFormat metricOutputFormat; - public AppendOnlyWriter(String insertSQL, int[] fieldTypes, DtRichOutputFormat metricOutputFormat) { - this.insertSQL = insertSQL; + public AppendOnlyWriter(String insertSql, int[] fieldTypes, DtRichOutputFormat metricOutputFormat) { + this.insertSql = insertSql; this.fieldTypes = fieldTypes; this.metricOutputFormat = metricOutputFormat; } @@ -58,7 +59,7 @@ public AppendOnlyWriter(String insertSQL, int[] fieldTypes, DtRichOutputFormat m @Override public void open(Connection connection) throws SQLException { this.rows = new ArrayList(); - this.statement = connection.prepareStatement(insertSQL); + this.statement = connection.prepareStatement(insertSql); } /** diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java deleted file mode 100644 index c7074de59..000000000 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java +++ /dev/null @@ -1,338 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.rdb.writer; - -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; -import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.types.Row; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; - -import static com.dtstack.flink.sql.sink.rdb.JDBCTypeConvertUtils.setRecordToStatement; -import static org.apache.flink.util.Preconditions.checkNotNull; - -/** - * Upsert writer to deal with upsert, delete message.dd - */ -public abstract class UpsertWriter implements JDBCWriter { - - private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(UpsertWriter.class); - - - public static UpsertWriter create( - JDBCDialect dialect, - String schema, - String tableName, - String[] fieldNames, - int[] fieldTypes, - String[] keyFields, - String[] partitionFields, - boolean objectReuse, - boolean allReplace, - DtRichOutputFormat metricOutputFormat) { - - checkNotNull(keyFields); - - List nameList = Arrays.asList(fieldNames); - int[] pkFields = Arrays.stream(keyFields).mapToInt(nameList::indexOf).toArray(); - int[] pkTypes = fieldTypes == null ? null : - Arrays.stream(pkFields).map(f -> fieldTypes[f]).toArray(); - - String deleteSQL = dialect.getDeleteStatement(schema, tableName, keyFields); - LOG.info("deleteSQL is :{}", deleteSQL); - System.out.println("deleteSQL is :" + deleteSQL); - - Optional upsertSQL = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); - LOG.info("execute UpsertStatement: {}", upsertSQL.orElse("use UsingInsertUpdateStatement")); - System.out.println("execute UpsertStatement: " + upsertSQL.orElse("use UsingInsertUpdateStatement")); - - return upsertSQL.map((Function) sql -> - new UpsertWriterUsingUpsertStatement( - fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, sql, metricOutputFormat)) - .orElseGet(() -> - new UpsertWriterUsingInsertUpdateStatement( - fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, - dialect.getRowExistsStatement(tableName, keyFields), - dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields), - dialect.getUpdateStatement(tableName, fieldNames, keyFields), - metricOutputFormat)); - } - - final int[] fieldTypes; - final int[] pkTypes; - private final int[] pkFields; - private final String deleteSQL; - private final boolean objectReuse; - - private transient Map> keyToRows; - private transient PreparedStatement deleteStatement; - // only use metric - private transient DtRichOutputFormat metricOutputFormat; - - private UpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSQL, boolean objectReuse, DtRichOutputFormat metricOutputFormat) { - this.fieldTypes = fieldTypes; - this.pkFields = pkFields; - this.pkTypes = pkTypes; - this.deleteSQL = deleteSQL; - this.objectReuse = objectReuse; - this.metricOutputFormat = metricOutputFormat; - } - - @Override - public void open(Connection connection) throws SQLException { - this.keyToRows = new HashMap<>(); - this.deleteStatement = connection.prepareStatement(deleteSQL); - } - - @Override - public void addRecord(Tuple2 record) throws SQLException { - // we don't need perform a deep copy, because jdbc field are immutable object. - Tuple2 tuple2 = objectReuse ? new Tuple2<>(record.f0, Row.copy(record.f1)) : record; - // add records to buffer - keyToRows.put(getPrimaryKey(tuple2.f1), tuple2); - } - - @Override - public void executeBatch(Connection connection) throws SQLException { - try { - if (keyToRows.size() > 0) { - for (Map.Entry> entry : keyToRows.entrySet()) { - Row pk = entry.getKey(); - Tuple2 tuple = entry.getValue(); - if (tuple.f0) { - processOneRowInBatch(pk, tuple.f1); - } else { - setRecordToStatement(deleteStatement, pkTypes, pk); - deleteStatement.addBatch(); - } - } - internalExecuteBatch(); - deleteStatement.executeBatch(); - connection.commit(); - keyToRows.clear(); - } - } catch (Exception e) { - // 清理批处理中的正确字段，防止重复写入 - connection.rollback(); - connection.commit(); - cleanBatchWhenError(); - executeUpdate(connection); - } - } - - @Override - public void executeUpdate(Connection connection) throws SQLException { - if (keyToRows.size() > 0) { - for (Map.Entry> entry : keyToRows.entrySet()) { - try { - Row pk = entry.getKey(); - Tuple2 tuple = entry.getValue(); - if (tuple.f0) { - processOneRowInBatch(pk, tuple.f1); - internalExecuteBatch(); - } else { - setRecordToStatement(deleteStatement, pkTypes, pk); - deleteStatement.executeUpdate(); - } - connection.commit(); - } catch (Exception e) { - System.out.println(e.getCause()); - // deal pg error: current transaction is aborted, commands ignored until end of transaction block - connection.rollback(); - connection.commit(); - if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { - LOG.error("record insert failed ,this row is {}", entry.getValue()); - LOG.error("", e); - } - metricOutputFormat.outDirtyRecords.inc(); - } - } - keyToRows.clear(); - } - } - - abstract void processOneRowInBatch(Row pk, Row row) throws SQLException; - - abstract void internalExecuteBatch() throws SQLException; - - @Override - public void close() throws SQLException { - if (deleteStatement != null) { - deleteStatement.close(); - deleteStatement = null; - } - } - - private Row getPrimaryKey(Row row) { - Row pks = new Row(pkFields.length); - for (int i = 0; i < pkFields.length; i++) { - pks.setField(i, row.getField(pkFields[i])); - } - return pks; - } - - // ---------------------------------------------------------------------------------------- - - private static final class UpsertWriterUsingUpsertStatement extends UpsertWriter { - - private static final long serialVersionUID = 1L; - private final String upsertSQL; - - private transient PreparedStatement upsertStatement; - - private UpsertWriterUsingUpsertStatement( - int[] fieldTypes, - int[] pkFields, - int[] pkTypes, - boolean objectReuse, - String deleteSQL, - String upsertSQL, - DtRichOutputFormat metricOutputFormat) { - super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); - this.upsertSQL = upsertSQL; - } - - @Override - public void open(Connection connection) throws SQLException { - super.open(connection); - upsertStatement = connection.prepareStatement(upsertSQL); - } - - @Override - void processOneRowInBatch(Row pk, Row row) throws SQLException { - setRecordToStatement(upsertStatement, fieldTypes, row); - upsertStatement.addBatch(); - } - - @Override - public void cleanBatchWhenError() throws SQLException { - upsertStatement.clearBatch(); - upsertStatement.clearParameters(); - } - - @Override - void internalExecuteBatch() throws SQLException { - upsertStatement.executeBatch(); - } - - @Override - public void close() throws SQLException { - super.close(); - if (upsertStatement != null) { - upsertStatement.close(); - upsertStatement = null; - } - } - } - - private static final class UpsertWriterUsingInsertUpdateStatement extends UpsertWriter { - - private static final long serialVersionUID = 1L; - private final String existSQL; - private final String insertSQL; - private final String updateSQL; - - private transient PreparedStatement existStatement; - private transient PreparedStatement insertStatement; - private transient PreparedStatement updateStatement; - - private UpsertWriterUsingInsertUpdateStatement( - int[] fieldTypes, - int[] pkFields, - int[] pkTypes, - boolean objectReuse, - String deleteSQL, - String existSQL, - String insertSQL, - String updateSQL, - DtRichOutputFormat metricOutputFormat) { - super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); - this.existSQL = existSQL; - this.insertSQL = insertSQL; - this.updateSQL = updateSQL; - } - - @Override - public void open(Connection connection) throws SQLException { - super.open(connection); - existStatement = connection.prepareStatement(existSQL); - insertStatement = connection.prepareStatement(insertSQL); - updateStatement = connection.prepareStatement(updateSQL); - } - - @Override - void processOneRowInBatch(Row pk, Row row) throws SQLException { - setRecordToStatement(existStatement, pkTypes, pk); - ResultSet resultSet = existStatement.executeQuery(); - boolean exist = resultSet.next(); - resultSet.close(); - if (exist) { - // do update - setRecordToStatement(updateStatement, fieldTypes, row); - updateStatement.addBatch(); - } else { - // do insert - setRecordToStatement(insertStatement, fieldTypes, row); - insertStatement.addBatch(); - } - } - - @Override - public void cleanBatchWhenError() throws SQLException { - updateStatement.clearBatch(); - insertStatement.clearBatch(); - } - - @Override - void internalExecuteBatch() throws SQLException { - updateStatement.executeBatch(); - insertStatement.executeBatch(); - } - - @Override - public void close() throws SQLException { - super.close(); - if (existStatement != null) { - existStatement.close(); - existStatement = null; - } - if (insertStatement != null) { - insertStatement.close(); - insertStatement = null; - } - if (updateStatement != null) { - updateStatement.close(); - updateStatement = null; - } - } - } -} diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java index 961539fde..af1d94171 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java @@ -21,21 +21,26 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.sql.Connection; import java.sql.DriverManager; import java.util.List; import java.util.Map; + /** - * side operator with cache for all(period reload) + * side operator with cache for all(period reload) + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi */ -public class SqlserverAllReqRow extends RdbAllReqRow { +public class SqlserverAllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(SqlserverAllReqRow.class); @@ -46,13 +51,12 @@ public SqlserverAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List addParams = Maps.newHashMap(); - //addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java index 2b9913e7e..8f0fc63a4 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java @@ -24,7 +24,11 @@ import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; - +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverAllSideInfo extends RdbAllSideInfo { public SqlserverAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index cb4240ca6..070bfe334 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -33,7 +33,11 @@ import java.util.List; - +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverAsyncReqRow extends RdbAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(SqlserverAsyncReqRow.class); @@ -67,6 +71,6 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); } } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java index 347ca9bf4..d3c833c96 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java @@ -23,10 +23,15 @@ import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import java.util.List; +import java.util.List; +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverAsyncSideInfo extends RdbAsyncSideInfo { public SqlserverAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { diff --git a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java index 3631dcd8a..be8ebb152 100644 --- a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java +++ b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java @@ -21,7 +21,11 @@ import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; - +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverSideParser extends RdbSideParser { private static final String CURR_TYPE = "sqlserver"; diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java index dbbeb820e..3c331cdd0 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** @@ -29,7 +29,7 @@ * * @author maqi */ -public class SqlserverSink extends RdbSink implements IStreamSinkGener { +public class SqlserverSink extends AbstractRdbSink implements IStreamSinkGener { public SqlserverSink() { super(new SqlserverDialect()); @@ -38,7 +38,7 @@ public SqlserverSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) From 36549fb01ed85eefa816e2de7c9a1adf46245e2f Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 25 Feb 2020 15:58:56 +0800 Subject: [PATCH 051/523] rename --- .../side/rdb/all/AbstractRdbAllReqRow.java | 228 ++++++++++++ .../sink/rdb/writer/AbstractUpsertWriter.java | 339 ++++++++++++++++++ 2 files changed, 567 insertions(+) create mode 100644 rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java create mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java new file mode 100644 index 000000000..aeb88f7a3 --- /dev/null +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.rdb.all; + +import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; +import org.apache.calcite.sql.JoinType; +import org.apache.commons.collections.CollectionUtils; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.util.Calendar; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +/** + * side operator with cache for all(period reload) + * Date: 2018/11/26 + * Company: www.dtstack.com + * + * @author maqi + */ + +public abstract class AbstractRdbAllReqRow extends AllReqRow { + + private static final long serialVersionUID = 2098635140857937718L; + + private static final Logger LOG = LoggerFactory.getLogger(AbstractRdbAllReqRow.class); + + private static final int CONN_RETRY_NUM = 3; + + private AtomicReference>>> cacheRef = new AtomicReference<>(); + + public AbstractRdbAllReqRow(SideInfo sideInfo) { + super(sideInfo); + } + + @Override + public Row fillData(Row input, Object sideInput) { + Map cacheInfo = (Map) sideInput; + Row row = new Row(sideInfo.getOutFieldInfoList().size()); + for (Map.Entry entry : sideInfo.getInFieldIndex().entrySet()) { + Object obj = input.getField(entry.getValue()); + boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); + + //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. + if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { + obj = ((Timestamp) obj).getTime(); + } + + row.setField(entry.getKey(), obj); + } + + for (Map.Entry entry : sideInfo.getSideFieldNameIndex().entrySet()) { + if (cacheInfo == null) { + row.setField(entry.getKey(), null); + } else { + row.setField(entry.getKey(), cacheInfo.get(entry.getValue())); + } + } + + return row; + } + + @Override + protected void initCache() throws SQLException { + Map>> newCache = Maps.newConcurrentMap(); + cacheRef.set(newCache); + loadData(newCache); + } + + @Override + protected void reloadCache() { + //reload cacheRef and replace to old cacheRef + Map>> newCache = Maps.newConcurrentMap(); + try { + loadData(newCache); + } catch (SQLException e) { + LOG.error("", e); + } + + cacheRef.set(newCache); + LOG.info("----- rdb all cacheRef reload end:{}", Calendar.getInstance()); + } + + + @Override + public void flatMap(CRow value, Collector out) throws Exception { + List inputParams = Lists.newArrayList(); + for (Integer conValIndex : sideInfo.getEqualValIndex()) { + Object equalObj = value.row().getField(conValIndex); + if (equalObj == null) { + if (sideInfo.getJoinType() == JoinType.LEFT) { + Row row = fillData(value.row(), null); + out.collect(new CRow(row, value.change())); + } + return; + } + inputParams.add(equalObj); + } + + String key = buildKey(inputParams); + List> cacheList = cacheRef.get().get(key); + if (CollectionUtils.isEmpty(cacheList)) { + if (sideInfo.getJoinType() == JoinType.LEFT) { + Row row = fillData(value.row(), null); + out.collect(new CRow(row, value.change())); + } else { + return; + } + + return; + } + + for (Map one : cacheList) { + out.collect(new CRow(fillData(value.row(), one), value.change())); + } + } + + private String buildKey(List equalValList) { + StringBuilder sb = new StringBuilder(""); + for (Object equalVal : equalValList) { + sb.append(equalVal).append("_"); + } + + return sb.toString(); + } + + private String buildKey(Map val, List equalFieldList) { + StringBuilder sb = new StringBuilder(""); + for (String equalField : equalFieldList) { + sb.append(val.get(equalField)).append("_"); + } + + return sb.toString(); + } + + public abstract Connection getConn(String dbUrl, String userName, String password); + + + private void loadData(Map>> tmpCache) throws SQLException { + RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + Connection connection = null; + + try { + for (int i = 0; i < CONN_RETRY_NUM; i++) { + try { + connection = getConn(tableInfo.getUrl(), tableInfo.getUserName(), tableInfo.getPassword()); + break; + } catch (Exception e) { + if (i == CONN_RETRY_NUM - 1) { + throw new RuntimeException("", e); + } + try { + String connInfo = "url:" + tableInfo.getUrl() + ";userName:" + tableInfo.getUserName() + ",pwd:" + tableInfo.getPassword(); + LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); + Thread.sleep(5 * 1000); + } catch (InterruptedException e1) { + LOG.error("", e1); + } + } + + } + + //load data from table + String sql = sideInfo.getSqlCondition(); + Statement statement = connection.createStatement(); + statement.setFetchSize(getFetchSize()); + ResultSet resultSet = statement.executeQuery(sql); + String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); + String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); + while (resultSet.next()) { + Map oneRow = Maps.newHashMap(); + for (String fieldName : sideFieldNames) { + Object object = resultSet.getObject(fieldName.trim()); + int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); + object = SwitchUtil.getTarget(object, fields[fieldIndex]); + oneRow.put(fieldName.trim(), object); + } + + String cacheKey = buildKey(oneRow, sideInfo.getEqualFieldList()); + List> list = tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()); + list.add(oneRow); + } + } catch (Exception e) { + LOG.error("", e); + } finally { + if (connection != null) { + connection.close(); + } + } + } + + public int getFetchSize() { + return 1000; + } + +} diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java new file mode 100644 index 000000000..373a59f6a --- /dev/null +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.rdb.writer; + +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.types.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +import static com.dtstack.flink.sql.sink.rdb.JDBCTypeConvertUtils.setRecordToStatement; +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * Upsert writer to deal with upsert, delete message.dd + * @author maqi + */ +public abstract class AbstractUpsertWriter implements JDBCWriter { + + private static final long serialVersionUID = 1L; + private static final Logger LOG = LoggerFactory.getLogger(AbstractUpsertWriter.class); + + + public static AbstractUpsertWriter create( + JDBCDialect dialect, + String schema, + String tableName, + String[] fieldNames, + int[] fieldTypes, + String[] keyFields, + String[] partitionFields, + boolean objectReuse, + boolean allReplace, + DtRichOutputFormat metricOutputFormat) { + + checkNotNull(keyFields); + + List nameList = Arrays.asList(fieldNames); + int[] pkFields = Arrays.stream(keyFields).mapToInt(nameList::indexOf).toArray(); + int[] pkTypes = fieldTypes == null ? null : + Arrays.stream(pkFields).map(f -> fieldTypes[f]).toArray(); + + String deleteSql = dialect.getDeleteStatement(schema, tableName, keyFields); + LOG.info("deleteSQL is :{}", deleteSql); + System.out.println("deleteSQL is :" + deleteSql); + + Optional upsertSql = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); + LOG.info("execute UpsertStatement: {}", upsertSql.orElse("use UsingInsertUpdateStatement")); + System.out.println("execute UpsertStatement: " + upsertSql.orElse("use UsingInsertUpdateStatement")); + + return upsertSql.map((Function) sql -> + new UpsertWriterUsingUpsertStatement( + fieldTypes, pkFields, pkTypes, objectReuse, deleteSql, sql, metricOutputFormat)) + .orElseGet(() -> + new UpsertWriterUsingInsertUpdateStatement( + fieldTypes, pkFields, pkTypes, objectReuse, deleteSql, + dialect.getRowExistsStatement(tableName, keyFields), + dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields), + dialect.getUpdateStatement(tableName, fieldNames, keyFields), + metricOutputFormat)); + } + + final int[] fieldTypes; + final int[] pkTypes; + private final int[] pkFields; + private final String deleteSql; + private final boolean objectReuse; + + private transient Map> keyToRows; + private transient PreparedStatement deleteStatement; + // only use metric + private transient DtRichOutputFormat metricOutputFormat; + + private AbstractUpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSql, boolean objectReuse, DtRichOutputFormat metricOutputFormat) { + this.fieldTypes = fieldTypes; + this.pkFields = pkFields; + this.pkTypes = pkTypes; + this.deleteSql = deleteSql; + this.objectReuse = objectReuse; + this.metricOutputFormat = metricOutputFormat; + } + + @Override + public void open(Connection connection) throws SQLException { + this.keyToRows = new HashMap<>(); + this.deleteStatement = connection.prepareStatement(deleteSql); + } + + @Override + public void addRecord(Tuple2 record) throws SQLException { + // we don't need perform a deep copy, because jdbc field are immutable object. + Tuple2 tuple2 = objectReuse ? new Tuple2<>(record.f0, Row.copy(record.f1)) : record; + // add records to buffer + keyToRows.put(getPrimaryKey(tuple2.f1), tuple2); + } + + @Override + public void executeBatch(Connection connection) throws SQLException { + try { + if (keyToRows.size() > 0) { + for (Map.Entry> entry : keyToRows.entrySet()) { + Row pk = entry.getKey(); + Tuple2 tuple = entry.getValue(); + if (tuple.f0) { + processOneRowInBatch(pk, tuple.f1); + } else { + setRecordToStatement(deleteStatement, pkTypes, pk); + deleteStatement.addBatch(); + } + } + internalExecuteBatch(); + deleteStatement.executeBatch(); + connection.commit(); + keyToRows.clear(); + } + } catch (Exception e) { + // 清理批处理中的正确字段，防止重复写入 + connection.rollback(); + connection.commit(); + cleanBatchWhenError(); + executeUpdate(connection); + } + } + + @Override + public void executeUpdate(Connection connection) throws SQLException { + if (keyToRows.size() > 0) { + for (Map.Entry> entry : keyToRows.entrySet()) { + try { + Row pk = entry.getKey(); + Tuple2 tuple = entry.getValue(); + if (tuple.f0) { + processOneRowInBatch(pk, tuple.f1); + internalExecuteBatch(); + } else { + setRecordToStatement(deleteStatement, pkTypes, pk); + deleteStatement.executeUpdate(); + } + connection.commit(); + } catch (Exception e) { + System.out.println(e.getCause()); + // deal pg error: current transaction is aborted, commands ignored until end of transaction block + connection.rollback(); + connection.commit(); + if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { + LOG.error("record insert failed ,this row is {}", entry.getValue()); + LOG.error("", e); + } + metricOutputFormat.outDirtyRecords.inc(); + } + } + keyToRows.clear(); + } + } + + abstract void processOneRowInBatch(Row pk, Row row) throws SQLException; + + abstract void internalExecuteBatch() throws SQLException; + + @Override + public void close() throws SQLException { + if (deleteStatement != null) { + deleteStatement.close(); + deleteStatement = null; + } + } + + private Row getPrimaryKey(Row row) { + Row pks = new Row(pkFields.length); + for (int i = 0; i < pkFields.length; i++) { + pks.setField(i, row.getField(pkFields[i])); + } + return pks; + } + + // ---------------------------------------------------------------------------------------- + + private static final class UpsertWriterUsingUpsertStatement extends AbstractUpsertWriter { + + private static final long serialVersionUID = 1L; + private final String upsertSql; + + private transient PreparedStatement upsertStatement; + + private UpsertWriterUsingUpsertStatement( + int[] fieldTypes, + int[] pkFields, + int[] pkTypes, + boolean objectReuse, + String deleteSql, + String upsertSql, + DtRichOutputFormat metricOutputFormat) { + super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); + this.upsertSql = upsertSql; + } + + @Override + public void open(Connection connection) throws SQLException { + super.open(connection); + upsertStatement = connection.prepareStatement(upsertSql); + } + + @Override + void processOneRowInBatch(Row pk, Row row) throws SQLException { + setRecordToStatement(upsertStatement, fieldTypes, row); + upsertStatement.addBatch(); + } + + @Override + public void cleanBatchWhenError() throws SQLException { + upsertStatement.clearBatch(); + upsertStatement.clearParameters(); + } + + @Override + void internalExecuteBatch() throws SQLException { + upsertStatement.executeBatch(); + } + + @Override + public void close() throws SQLException { + super.close(); + if (upsertStatement != null) { + upsertStatement.close(); + upsertStatement = null; + } + } + } + + private static final class UpsertWriterUsingInsertUpdateStatement extends AbstractUpsertWriter { + + private static final long serialVersionUID = 1L; + private final String existSql; + private final String insertSql; + private final String updateSql; + + private transient PreparedStatement existStatement; + private transient PreparedStatement insertStatement; + private transient PreparedStatement updateStatement; + + private UpsertWriterUsingInsertUpdateStatement( + int[] fieldTypes, + int[] pkFields, + int[] pkTypes, + boolean objectReuse, + String deleteSql, + String existSql, + String insertSql, + String updateSql, + DtRichOutputFormat metricOutputFormat) { + super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); + this.existSql = existSql; + this.insertSql = insertSql; + this.updateSql = updateSql; + } + + @Override + public void open(Connection connection) throws SQLException { + super.open(connection); + existStatement = connection.prepareStatement(existSql); + insertStatement = connection.prepareStatement(insertSql); + updateStatement = connection.prepareStatement(updateSql); + } + + @Override + void processOneRowInBatch(Row pk, Row row) throws SQLException { + setRecordToStatement(existStatement, pkTypes, pk); + ResultSet resultSet = existStatement.executeQuery(); + boolean exist = resultSet.next(); + resultSet.close(); + if (exist) { + // do update + setRecordToStatement(updateStatement, fieldTypes, row); + updateStatement.addBatch(); + } else { + // do insert + setRecordToStatement(insertStatement, fieldTypes, row); + insertStatement.addBatch(); + } + } + + @Override + public void cleanBatchWhenError() throws SQLException { + updateStatement.clearBatch(); + insertStatement.clearBatch(); + } + + @Override + void internalExecuteBatch() throws SQLException { + updateStatement.executeBatch(); + insertStatement.executeBatch(); + } + + @Override + public void close() throws SQLException { + super.close(); + if (existStatement != null) { + existStatement.close(); + existStatement = null; + } + if (insertStatement != null) { + insertStatement.close(); + insertStatement = null; + } + if (updateStatement != null) { + updateStatement.close(); + updateStatement = null; + } + } + } +} From aac3a273f52d39318e943c2a4616d4e5a97394e8 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 25 Feb 2020 16:13:25 +0800 Subject: [PATCH 052/523] redis can --- .../flink/sql/side/redis/RedisAllReqRow.java | 7 ++-- .../sql/side/redis/RedisAllSideInfo.java | 4 ++- .../sql/side/redis/RedisAsyncReqRow.java | 5 ++- .../sql/side/redis/RedisAsyncSideInfo.java | 4 ++- .../sql/side/redis/table/RedisSideParser.java | 3 ++ .../side/redis/table/RedisSideTableInfo.java | 4 ++- .../sql/sink/redis/RedisOutputFormat.java | 35 +++++++++++++------ .../flink/sql/sink/redis/RedisSink.java | 4 ++- .../sql/sink/redis/table/RedisSinkParser.java | 5 +-- .../sql/sink/redis/table/RedisTableInfo.java | 4 ++- .../CustomerSocketTextStreamFunction.java | 2 +- .../table/ServersocketSourceTableInfo.java | 2 +- 12 files changed, 56 insertions(+), 23 deletions(-) diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 59997f5cb..a215c9d75 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -39,7 +39,9 @@ import java.util.*; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; - +/** + * @author yanxi + */ public class RedisAllReqRow extends AllReqRow{ private static final long serialVersionUID = 7578879189085344807L; @@ -259,7 +261,8 @@ private JedisCommands getJedis(RedisSideTableInfo tableInfo) { break; //集群 case 3: - jedis = new JedisCluster(addresses, timeout, timeout,1, poolConfig); + jedis = new JedisCluster(addresses, timeout, timeout, 1, poolConfig); + default: } return jedis; diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java index 81e7b1f4f..fc24bc4ef 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java @@ -28,7 +28,9 @@ import com.google.common.collect.Lists; import java.util.List; - +/** + * @author yanxi + */ public class RedisAllSideInfo extends SideInfo { private static final long serialVersionUID = 1998703966487857613L; diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index d20875e01..ee951a928 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -43,7 +43,9 @@ import java.util.List; import java.util.Map; import java.util.function.Consumer; - +/** + * @author yanxi + */ public class RedisAsyncReqRow extends AsyncReqRow { private static final long serialVersionUID = -2079908694523987738L; @@ -108,6 +110,7 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ clusterClient = RedisClusterClient.create(clusterUri.toString()); clusterConnection = clusterClient.connect(); async = clusterConnection.async(); + default: } } diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java index 9bdefe8c5..2736fc8a7 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java @@ -28,7 +28,9 @@ import com.google.common.collect.Lists; import java.util.List; - +/** + * @author yanxi + */ public class RedisAsyncSideInfo extends SideInfo { private static final long serialVersionUID = -4851348392924455039L; diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java index f8a83b868..2d0fb77d4 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java @@ -24,6 +24,9 @@ import java.util.Map; +/** + * @author yanxi + */ public class RedisSideParser extends AbsSideTableParser { @Override diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java index 88cfcb8a2..ae6cdd017 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java @@ -20,7 +20,9 @@ import com.dtstack.flink.sql.side.SideTableInfo; import com.google.common.base.Preconditions; - +/** + * @author yanxi + */ public class RedisSideTableInfo extends SideTableInfo { private static final long serialVersionUID = -1L; diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index 62d14cabd..bbf49dff7 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -26,11 +26,23 @@ import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.*; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.JedisCommands; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisSentinelPool; + import java.io.Closeable; import java.io.IOException; -import java.util.*; - +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +/** + * @author yanxi + */ public class RedisOutputFormat extends DtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(RedisOutputFormat.class); @@ -129,7 +141,8 @@ private void establishConnection() { break; //集群 case 3: - jedis = new JedisCluster(addresses, timeout, timeout,10, password, poolConfig); + jedis = new JedisCluster(addresses, timeout, timeout, 10, password, poolConfig); + default: } } @@ -145,10 +158,10 @@ public void writeRecord(Tuple2 record) throws IOException { return; } - HashMap map = new HashMap<>(); - for (String primaryKey : primaryKeys){ - for (int i=0; i map = new HashMap<>(8); + for (String primaryKey : primaryKeys) { + for (int i = 0; i < fieldNames.length; i++) { + if (fieldNames[i].equals(primaryKey)) { map.put(primaryKey, i); } } @@ -156,10 +169,10 @@ public void writeRecord(Tuple2 record) throws IOException { List kvList = new LinkedList<>(); for (String primaryKey : primaryKeys){ - StringBuilder primaryKV = new StringBuilder(); + StringBuilder primaryKv = new StringBuilder(); int index = map.get(primaryKey).intValue(); - primaryKV.append(primaryKey).append(":").append(row.getField(index)); - kvList.add(primaryKV.toString()); + primaryKv.append(primaryKey).append(":").append(row.getField(index)); + kvList.add(primaryKv.toString()); } String perKey = String.join(":", kvList); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java index d2e28c01f..007c7fd3b 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java @@ -33,7 +33,9 @@ import org.apache.flink.types.Row; import java.util.List; - +/** + * @author yanxi + */ public class RedisSink implements RetractStreamTableSink, IStreamSinkGener { protected String[] fieldNames; diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 49861bb9a..201abab2c 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -24,11 +24,12 @@ import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; - +/** + * @author yanxi + */ public class RedisSinkParser extends AbsTableParser { @Override public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java index 60d6dd12f..5c9940252 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java @@ -20,7 +20,9 @@ import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; - +/** + * @author yanxi + */ public class RedisTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "redis"; diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java index e46e0a76f..2f508b3e5 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java @@ -94,7 +94,7 @@ public void run(SourceContext ctx) throws Exception { while (buffer.length() >= delimiter.length() && (delimPos = buffer.indexOf(delimiter)) != -1) { String record = buffer.substring(0, delimPos); // truncate trailing carriage return - if (delimiter.equals("\n") && record.endsWith("\r")) { + if ("\n".equals(delimiter) && "\r".endsWith(record)) { record = record.substring(0, record.length() - 1); } try { diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java index 796728eb3..02f6e9f30 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java @@ -28,7 +28,7 @@ * @author maqi */ public class ServersocketSourceTableInfo extends SourceTableInfo { - //version + private static final String CURR_TYPE = "serversocket"; public static final String HOSTNAME_KEY = "host"; From 42db63ec3d88d2234b2415846619b40e703dd07f Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 25 Feb 2020 16:20:24 +0800 Subject: [PATCH 053/523] merge 1.8_dev --- core/src/main/java/com/dtstack/flink/sql/Main.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index 4ff296f1f..b912128f9 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -27,8 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.LoggerContext; /** * Date: 2018/6/26 @@ -45,10 +43,4 @@ public static void main(String[] args) throws Exception { env.execute(paramsInfo.getName()); LOG.info("program {} execution success", paramsInfo.getName()); } - private static void setLogLevel(String level){ - LoggerContext loggerContext= (LoggerContext) LoggerFactory.getILoggerFactory(); - //设置全局日志级别 - ch.qos.logback.classic.Logger logger = loggerContext.getLogger("root"); - logger.setLevel(Level.toLevel(level, Level.INFO)); - } } From bfc1f0e18ecf166cad5cbb8f690e023e5f823279 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Wed, 26 Feb 2020 12:20:33 +0800 Subject: [PATCH 054/523] =?UTF-8?q?=E8=A7=A3=E5=86=B3elasticsearch-side?= =?UTF-8?q?=E5=88=86=E6=94=AF=E4=B8=8Etest=5F1.8=5F3.1.0x=E7=9A=84?= =?UTF-8?q?=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 -- .../side/cassandra/CassandraAllReqRow.java | 9 ++-- .../side/cassandra/CassandraAsyncReqRow.java | 7 ++-- .../sink/cassandra/CassandraOutputFormat.java | 7 ++-- .../flink/sql/constrant/ConfigConstrant.java | 6 +-- .../environment/StreamEnvConfigManager.java | 41 ++++++------------- .../flink/sql/table/TableInfoParser.java | 3 +- .../dtstack/flink/sql/util/DtStringUtil.java | 6 +-- .../sink/elasticsearch/ElasticsearchSink.java | 4 +- .../flink/sql/sink/elasticsearch/EsUtil.java | 3 +- .../flink/sql/side/hbase/RowKeyBuilder.java | 5 +-- .../sql/side/hbase/table/HbaseSideParser.java | 3 +- .../sql/sink/hbase/HbaseOutputFormat.java | 2 +- .../sql/sink/hbase/table/HbaseSinkParser.java | 7 ++-- .../side/impala/table/ImpalaSideParser.java | 5 +-- .../sink/impala/table/ImpalaTableInfo.java | 3 +- .../flink/sql/source/kafka/KafkaConsumer.java | 3 +- .../sql/source/kafka/KafkaConsumer09.java | 3 +- .../sql/source/kafka/KafkaConsumer010.java | 3 +- .../sql/source/kafka/KafkaConsumer011.java | 3 +- .../flink/sql/side/kudu/KuduAllReqRow.java | 7 ++-- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 5 +-- .../flink/sql/side/kudu/utils/KuduUtil.java | 3 +- .../sql/launcher/perjob/PerJobSubmitter.java | 2 +- .../flink/sql/side/mongo/MongoAllReqRow.java | 6 +-- .../sql/side/mongo/MongoAsyncReqRow.java | 4 +- .../flink/sql/side/mongo/utils/MongoUtil.java | 5 +-- .../sql/sink/mongo/MongoOutputFormat.java | 4 +- .../flink/sql/side/rdb/all/RdbAllReqRow.java | 3 +- .../sql/side/rdb/all/RdbAllSideInfo.java | 2 +- .../sql/side/rdb/async/RdbAsyncSideInfo.java | 3 +- .../flink/sql/side/redis/RedisAllReqRow.java | 19 ++++----- .../sql/side/redis/RedisAsyncReqRow.java | 3 +- .../sql/sink/redis/RedisOutputFormat.java | 7 ++-- .../sql/sink/redis/table/RedisSinkParser.java | 2 +- 35 files changed, 78 insertions(+), 123 deletions(-) diff --git a/README.md b/README.md index a5b0457ab..583958908 100644 --- a/README.md +++ b/README.md @@ -149,9 +149,6 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack * taskmanager.slots：per_job模式下指定每个taskmanager对应的slot数量(默认1) * savePointPath：任务恢复点的路径（默认无） * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） - * restore.enable：是否失败重启（默认是true） - * failure.interval：衡量失败率的时间段，单位分钟（默认6m） - * delay.interval：连续两次重启尝试间的间隔，单位是秒（默认10s） * [prometheus 相关参数](docs/prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index bc0a0f8cd..4439c2b7a 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -43,7 +43,6 @@ import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -224,9 +223,9 @@ private Session getConn(CassandraSideTableInfo tableInfo) { //重试策略 RetryPolicy retryPolicy = DowngradingConsistencyRetryPolicy.INSTANCE; - for (String server : StringUtils.split(address, ",")) { - cassandraPort = Integer.parseInt(StringUtils.split(server, ":")[1]); - serversList.add(InetAddress.getByName(StringUtils.split(server, ":")[0])); + for (String server : address.split(",")) { + cassandraPort = Integer.parseInt(server.split(":")[1]); + serversList.add(InetAddress.getByName(server.split(":")[0])); } if (userName == null || userName.isEmpty() || password == null || password.isEmpty()) { @@ -280,7 +279,7 @@ private void loadData(Map>> tmpCache) throws SQ //load data from table String sql = sideInfo.getSqlCondition() + " limit " + FETCH_SIZE; ResultSet resultSet = session.execute(sql); - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); + String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); for (com.datastax.driver.core.Row row : resultSet) { Map oneRow = Maps.newHashMap(); for (String fieldName : sideFieldNames) { diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index 0173f2d6f..7a209f30e 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -51,7 +51,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import io.vertx.core.json.JsonArray; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,9 +134,9 @@ private void connCassandraDB(CassandraSideTableInfo tableInfo) { //重试策略 RetryPolicy retryPolicy = DowngradingConsistencyRetryPolicy.INSTANCE; - for (String server : StringUtils.split(address, ",")) { - cassandraPort = Integer.parseInt(StringUtils.split(server, ":")[1]); - serversList.add(InetAddress.getByName(StringUtils.split(server, ":")[0])); + for (String server : address.split(",")) { + cassandraPort = Integer.parseInt(server.split(":")[1]); + serversList.add(InetAddress.getByName(server.split(":")[0])); } if (userName == null || userName.isEmpty() || password == null || password.isEmpty()) { diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java index 14c13fc8c..d8f4eb451 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java @@ -55,7 +55,6 @@ import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -148,9 +147,9 @@ public void open(int taskNumber, int numTasks) { //重试策略 RetryPolicy retryPolicy = DowngradingConsistencyRetryPolicy.INSTANCE; - for (String server : StringUtils.split(address, ",")) { - cassandraPort = Integer.parseInt(StringUtils.split(server, ":")[1]); - serversList.add(InetAddress.getByName(StringUtils.split(server, ":")[0])); + for (String server : address.split(",")) { + cassandraPort = Integer.parseInt(server.split(":")[1]); + serversList.add(InetAddress.getByName(server.split(":")[0])); } if (userName == null || userName.isEmpty() || password == null || password.isEmpty()) { diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java index a80962661..dcda4c093 100644 --- a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java @@ -63,14 +63,12 @@ public class ConfigConstrant { public static final String CHECKPOINTS_DIRECTORY_KEY = "state.checkpoints.dir"; public static final String STATE_BACKEND_INCREMENTAL_KEY = "state.backend.incremental"; - public static final String RESTOREENABLE = "restore.enable"; - // restart plocy public static final int failureRate = 3; - public static final String FAILUREINTERVAL = "failure.interval"; //min + public static final int failureInterval = 6; //min - public static final String DELAYINTERVAL= "delay.interval"; //sec + public static final int delayInterval = 10; //sec } diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index 28eac2af3..ce763ea99 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -18,12 +18,6 @@ package com.dtstack.flink.sql.environment; -import com.dtstack.flink.sql.constrant.ConfigConstrant; -import com.dtstack.flink.sql.enums.EStateBackend; -import com.dtstack.flink.sql.util.MathUtil; -import com.dtstack.flink.sql.util.PropertiesUtils; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.time.Time; @@ -40,6 +34,13 @@ import org.apache.flink.table.api.StreamQueryConfig; import org.apache.flink.table.api.java.StreamTableEnvironment; +import com.dtstack.flink.sql.constrant.ConfigConstrant; +import com.dtstack.flink.sql.enums.EStateBackend; +import com.dtstack.flink.sql.util.MathUtil; +import com.dtstack.flink.sql.util.PropertiesUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; + import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -101,15 +102,11 @@ public static void streamExecutionEnvironmentConfig(StreamExecutionEnvironment s } }); - if(isRestore(confProperties).get()){ - streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( - ConfigConstrant.failureRate, - Time.of(getFailureInterval(confProperties).get(), TimeUnit.MINUTES), - Time.of(getDelayInterval(confProperties).get(), TimeUnit.SECONDS) - )); - } else { - streamEnv.setRestartStrategy(RestartStrategies.noRestart()); - } + streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( + ConfigConstrant.failureRate, + Time.of(ConfigConstrant.failureInterval, TimeUnit.MINUTES), + Time.of(ConfigConstrant.delayInterval, TimeUnit.SECONDS) + )); // checkpoint config Optional checkpointingEnabled = isCheckpointingEnabled(confProperties); @@ -167,20 +164,6 @@ public static Optional getAutoWatermarkInterval(Properties properties) { return StringUtils.isNotBlank(autoWatermarkInterval) ? Optional.of(Long.valueOf(autoWatermarkInterval)) : Optional.empty(); } - public static Optional isRestore(Properties properties){ - String restoreEnable = properties.getProperty(ConfigConstrant.RESTOREENABLE, "true"); - return Optional.of(Boolean.valueOf(restoreEnable)); - } - - public static Optional getDelayInterval(Properties properties){ - String delayInterval = properties.getProperty(ConfigConstrant.DELAYINTERVAL, "10"); - return Optional.of(Integer.valueOf(delayInterval)); - } - public static Optional getFailureInterval(Properties properties){ - String failureInterval = properties.getProperty(ConfigConstrant.FAILUREINTERVAL, "6"); - return Optional.of(Integer.valueOf(failureInterval)); - } - /** * #ProcessingTime(默认), IngestionTime, EventTime * @param properties diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java index f4c638381..ae98d90ae 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java @@ -29,7 +29,6 @@ import com.dtstack.flink.sql.util.MathUtil; import com.google.common.base.Strings; import com.google.common.collect.Maps; -import org.apache.commons.lang3.StringUtils; import java.util.Map; import java.util.regex.Matcher; @@ -111,7 +110,7 @@ public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserRe * @return */ private static boolean checkIsSideTable(String tableField){ - String[] fieldInfos = StringUtils.split(tableField, ","); + String[] fieldInfos = tableField.split(","); for(String field : fieldInfos){ Matcher matcher = SIDE_PATTERN.matcher(field.trim()); if(matcher.find()){ diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index 84fe49fd3..862b0700e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -180,14 +180,14 @@ public static String addJdbcParam(String dbUrl, Map addParams, b return dbUrl; } - String[] splits = StringUtils.split(dbUrl, "\\?"); + String[] splits = dbUrl.split("\\?"); String preStr = splits[0]; Map params = Maps.newHashMap(); if(splits.length > 1){ String existsParamStr = splits[1]; - String[] existsParams = StringUtils.split(existsParamStr, "&"); + String[] existsParams = existsParamStr.split("&"); for(String oneParam : existsParams){ - String[] kv = StringUtils.split(oneParam, "="); + String[] kv = oneParam.split("="); if(kv.length != 2){ throw new RuntimeException("illegal dbUrl:" + dbUrl); } diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 491d1af27..0bfc338e3 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -118,7 +118,7 @@ private RichSinkFunction createEsSinkFunction(){ List transports = new ArrayList<>(); for(String address : esAddressList){ - String[] infoArray = StringUtils.split(address, ":"); + String[] infoArray = address.split(":"); int port = 9300; String host = infoArray[0]; if(infoArray.length > 1){ @@ -169,7 +169,7 @@ public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { esTableInfo = elasticsearchTableInfo; clusterName = elasticsearchTableInfo.getClusterName(); String address = elasticsearchTableInfo.getAddress(); - String[] addr = StringUtils.split(address, ","); + String[] addr = address.split(","); esAddressList = Arrays.asList(addr); index = elasticsearchTableInfo.getIndex(); type = elasticsearchTableInfo.getEsType(); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java index 24c5d3ab9..5a0cd205e 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java @@ -22,7 +22,6 @@ import org.apache.flink.util.Preconditions; import com.dtstack.flink.sql.util.DtStringUtil; -import org.apache.commons.lang3.StringUtils; import java.util.HashMap; import java.util.List; @@ -42,7 +41,7 @@ public static Map rowToJsonMap(Row row, List fields, Lis int i = 0; for(; i < fields.size(); ++i) { String field = fields.get(i); - String[] parts = StringUtils.split(field, "\\."); + String[] parts = field.split("\\."); Map currMap = jsonMap; for(int j = 0; j < parts.length - 1; ++j) { String key = parts[j]; diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java index 6ad5544ab..114b7fa6a 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java @@ -22,7 +22,6 @@ import com.dtstack.flink.sql.side.hbase.enums.EReplaceType; import com.google.common.collect.Lists; -import org.apache.commons.lang3.StringUtils; import java.io.Serializable; import java.util.List; @@ -59,7 +58,7 @@ public void init(String rowKeyTempl){ fieldCols = infoAlias; } - String[] fieldArr = StringUtils.split(fieldCols, "\\+"); + String[] fieldArr = fieldCols.split("\\+"); List fieldList = Lists.newArrayList(); for(String oneField : fieldArr){ ReplaceInfo replaceInfo = getReplaceInfo(oneField); @@ -109,7 +108,7 @@ private String buildStr(List fieldList, Map refData public static String[] splitIgnoreQuotaBrackets(String str, String delimiter){ String splitPatternStr = delimiter + "(?![^()]*+\\))(?![^{}]*+})(?![^\\[\\]]*+\\])"; - return StringUtils.split(str, splitPatternStr); + return str.split(splitPatternStr); } /** diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 1ba9a9907..7b627da4d 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -23,7 +23,6 @@ import com.dtstack.flink.sql.table.AbsSideTableParser; import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; -import org.apache.commons.lang3.StringUtils; import java.util.Map; import java.util.regex.Matcher; @@ -84,7 +83,7 @@ private void dealField(Matcher matcher, TableInfo tableInfo){ String filedDefineStr = matcher.group(1); String aliasStr = matcher.group(2); - String[] filedInfoArr = StringUtils.split(filedDefineStr, "\\s+"); + String[] filedInfoArr = filedDefineStr.split("\\s+"); if(filedInfoArr.length < 2){ throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), matcher.group(0))); } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index d41c65723..fd94e0db2 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -281,7 +281,7 @@ public HbaseOutputFormat finish() { String[] columns = keySet.toArray(new String[keySet.size()]); for (int i = 0; i < columns.length; ++i) { String col = columns[i]; - String[] part = StringUtils.split(col, ":"); + String[] part = col.split(":"); families[i] = part[0]; qualifiers[i] = part[1]; } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index e93e8e612..f1ae72b6b 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -26,7 +26,6 @@ import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.MathUtil; -import org.apache.commons.lang3.StringUtils; import java.util.LinkedHashMap; import java.util.List; @@ -69,7 +68,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); + super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java index 78eabde09..2fa0d16b6 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java @@ -31,7 +31,6 @@ import org.apache.flink.util.SerializedValue; import com.dtstack.flink.sql.format.DeserializationMetricWrapper; -import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.Map; @@ -51,7 +50,7 @@ public class KafkaConsumer09 extends FlinkKafkaConsumer09 { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer09(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); + super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java index b78f47cd5..74f45581c 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java @@ -30,7 +30,6 @@ import org.apache.flink.util.SerializedValue; import com.dtstack.flink.sql.format.DeserializationMetricWrapper; -import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.Map; @@ -50,7 +49,7 @@ public class KafkaConsumer010 extends FlinkKafkaConsumer010 { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer010(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); + super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java index 276ce211f..a88fb1f9c 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java @@ -30,7 +30,6 @@ import org.apache.flink.util.SerializedValue; import com.dtstack.flink.sql.format.DeserializationMetricWrapper; -import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.Map; @@ -50,7 +49,7 @@ public class KafkaConsumer011 extends FlinkKafkaConsumer011 { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer011(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); + super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 63ff31c50..374261d40 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -18,7 +18,6 @@ import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.KuduClient; @@ -161,7 +160,7 @@ private void loadData(Map>> tmpCache) { } //load data from table assert scanner != null; - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); + String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); while (scanner.hasMoreRows()) { @@ -264,7 +263,7 @@ private KuduScanner buildScanner(KuduScanner.KuduScannerBuilder builder, Schema Long limitNum = tableInfo.getLimitNum(); Boolean isFaultTolerant = tableInfo.getFaultTolerant(); //查询需要的字段 - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); + String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); //主键过滤条件 主键最小值 String lowerBoundPrimaryKey = tableInfo.getLowerBoundPrimaryKey(); //主键过滤条件 主键最大值 @@ -318,7 +317,7 @@ private KuduScanner buildScanner(KuduScanner.KuduScannerBuilder builder, Schema } private String[] splitString(String data) { - return StringUtils.split(data, ","); + return data.split(","); } @Override diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index e1705d796..588bd00ca 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -23,7 +23,6 @@ import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; import io.vertx.core.json.JsonArray; -import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.AsyncKuduClient; @@ -114,7 +113,7 @@ private void connKuDu() throws KuduException { Long limitNum = kuduSideTableInfo.getLimitNum(); Boolean isFaultTolerant = kuduSideTableInfo.getFaultTolerant(); //查询需要的字段 - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); + String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); if (null == limitNum || limitNum <= 0) { scannerBuilder.limit(FETCH_SIZE); @@ -281,7 +280,7 @@ public GetListRowCB() { public Deferred> call(RowResultIterator results) throws Exception { for (RowResult result : results) { Map oneRow = Maps.newHashMap(); - for (String sideFieldName1 : StringUtils.split(sideInfo.getSideSelectFields(), ",")) { + for (String sideFieldName1 : sideInfo.getSideSelectFields().split(",")) { String sideFieldName = sideFieldName1.trim(); ColumnSchema columnSchema = table.getSchema().getColumn(sideFieldName); if (null != columnSchema) { diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java index b750e2d4a..c5cd8cdb2 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java @@ -20,7 +20,6 @@ import com.dtstack.flink.sql.side.PredicateInfo; import org.apache.commons.lang.math.NumberUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.Type; @@ -164,7 +163,7 @@ public static KuduPredicate buildKuduPredicate(Schema schema, PredicateInfo info case "IN": case "NOT_IN": case "BETWEEN": - value = Arrays.asList(StringUtils.split(info.getCondition(), ",")).stream() + value = Arrays.asList(info.getCondition().split(",")).stream() .map(val -> KuduUtil.getValue(val.trim(), column.getType())).collect(Collectors.toList()); break; case "IS_NOT_NULL": diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index 29cc4890a..918451f7a 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -83,7 +83,7 @@ private static List getJarPaths(String addjarPath) { if (addjarPath.length() > 2) { addjarPath = addjarPath.substring(1,addjarPath.length()-1).replace("\"",""); } - List paths = Arrays.asList(StringUtils.split(addjarPath, ",")); + List paths = Arrays.asList(addjarPath.split(",")); return paths; } diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index f90cfd973..1a0565268 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -185,13 +185,13 @@ private MongoCollection getConn(String address, String userName, String password MongoCollection dbCollection; try { MongoCredential credential; - String[] servers = StringUtils.split(address, ","); + String[] servers = address.split(","); String host; Integer port; String[] hostAndPort; List lists = new ArrayList<>(); for (String server : servers) { - hostAndPort = StringUtils.split(server, ":"); + hostAndPort = server.split(":"); host = hostAndPort[0]; port = Integer.parseInt(hostAndPort[1]); lists.add(new ServerAddress(host, port)); @@ -237,7 +237,7 @@ private void loadData(Map>> tmpCache) throws SQ } //load data from table - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); + String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); BasicDBObject basicDBObject = new BasicDBObject(); for (String selectField : sideFieldNames) { basicDBObject.append(selectField, 1); diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 42749e935..ac6b4120c 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -92,13 +92,13 @@ public void open(Configuration parameters) throws Exception { public void connMongoDB() throws Exception { MongoCredential mongoCredential; - String[] servers = StringUtils.split(MongoSideTableInfo.getAddress(), ","); + String[] servers = MongoSideTableInfo.getAddress().split(","); String host; Integer port; String[] hostAndPort; List lists = new ArrayList<>(); for (String server : servers) { - hostAndPort = StringUtils.split(server, ":"); + hostAndPort = server.split(":"); host = hostAndPort[0]; port = Integer.parseInt(hostAndPort[1]); lists.add(new ServerAddress(host, port)); diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java index 2f1a2092d..0b345d947 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java @@ -20,7 +20,6 @@ import com.dtstack.flink.sql.side.PredicateInfo; import com.mongodb.BasicDBObject; -import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.stream.Collectors; @@ -46,11 +45,11 @@ public static BasicDBObject buildFilterObject(PredicateInfo info) { case "<>": return new BasicDBObject("$ne", info.getCondition()); case "IN": - Object[] values = Arrays.stream(StringUtils.split(info.getCondition(), ",")).map(String::trim) + Object[] values = Arrays.stream(info.getCondition().split(",")).map(String::trim) .collect(Collectors.toList()).toArray(); return new BasicDBObject("$in", values); case "NOT IN": - return new BasicDBObject("$nin", StringUtils.split(info.getCondition(), ",")); + return new BasicDBObject("$nin", info.getCondition().split(",")); case "IS NOT NULL": return new BasicDBObject("$exists", true); case "IS NULL": diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index 337fbad59..d5a9b4a82 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -127,13 +127,13 @@ public void close() { private void establishConnection() { try { MongoCredential credential; - String[] servers = StringUtils.split(address, ","); + String[] servers = address.split(","); String host; Integer port; String[] hostAndPort; List lists = new ArrayList<>(); for (String server : servers) { - hostAndPort = StringUtils.split(server, ":"); + hostAndPort = server.split(":"); host = hostAndPort[0]; port = Integer.parseInt(hostAndPort[1]); lists.add(new ServerAddress(host, port)); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java index 7fdeeeca9..c746d1185 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java @@ -31,7 +31,6 @@ import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -199,7 +198,7 @@ private void loadData(Map>> tmpCache) throws SQ Statement statement = connection.createStatement(); statement.setFetchSize(getFetchSize()); ResultSet resultSet = statement.executeQuery(sql); - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); + String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); while (resultSet.next()) { Map oneRow = Maps.newHashMap(); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 3b7444b77..313594050 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -56,7 +56,7 @@ public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List> tmpCache) throws SQLExcep Set keys = ((Jedis) jedis).keys(perKey); List newPerKeys = new LinkedList<>(); for (String key : keys){ - String[] splitKey = StringUtils.split(key, ":"); + String[] splitKey = key.split(":"); String newKey = splitKey[0] + ":" + splitKey[1] + ":" + splitKey[2]; newPerKeys.add(newKey); } List list = newPerKeys.stream().distinct().collect(Collectors.toList()); for(String key : list){ Map kv = Maps.newHashMap(); - String[] primaryKv = StringUtils.split(key, ":"); + String[] primaryKv = key.split(":"); kv.put(primaryKv[1], primaryKv[2]); String pattern = key + "*"; Set realKeys = ((Jedis) jedis).keys(pattern); for (String realKey : realKeys){ - kv.put(StringUtils.split(realKey, ":")[3], jedis.get(realKey)); + kv.put(realKey.split(":")[3], jedis.get(realKey)); } tmpCache.put(key, kv); } @@ -199,19 +198,19 @@ private void loadData(Map> tmpCache) throws SQLExcep Set keys = keys((JedisCluster) jedis, perKey); List newPerKeys = new LinkedList<>(); for (String key : keys){ - String[] splitKey = StringUtils.split(key, ":"); + String[] splitKey = key.split(":"); String newKey = splitKey[0] + ":" + splitKey[1] + ":" + splitKey[2]; newPerKeys.add(newKey); } List list = newPerKeys.stream().distinct().collect(Collectors.toList()); for(String key : list){ Map kv = Maps.newHashMap(); - String[] primaryKv = StringUtils.split(key, ":"); + String[] primaryKv = key.split(":"); kv.put(primaryKv[1], primaryKv[2]); String pattern = key + "*"; Set realKeys = keys((JedisCluster) jedis, pattern); for (String realKey : realKeys){ - kv.put(StringUtils.split(key, ":")[3], jedis.get(realKey)); + kv.put(realKey.split(":")[3], jedis.get(realKey)); } tmpCache.put(key, kv); } @@ -246,15 +245,15 @@ private JedisCommands getJedis(RedisSideTableInfo tableInfo) { timeout = 1000; } - String[] nodes = StringUtils.split(url, ","); - String[] firstIpPort = StringUtils.split(nodes[0], ":"); + String[] nodes = url.split(","); + String[] firstIpPort = nodes[0].split(":"); String firstIp = firstIpPort[0]; String firstPort = firstIpPort[1]; Set addresses = new HashSet<>(); Set ipPorts = new HashSet<>(); for (String ipPort : nodes) { ipPorts.add(ipPort); - String[] ipPortPair = StringUtils.split(ipPort, ":"); + String[] ipPortPair = ipPort.split(":"); addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.valueOf(ipPortPair[1].trim()))); } if (timeout == 0){ diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 41f504bf0..1688b2d2c 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -43,7 +43,6 @@ import io.lettuce.core.api.async.RedisStringAsyncCommands; import io.lettuce.core.cluster.RedisClusterClient; import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; -import org.apache.commons.lang3.StringUtils; import java.util.Collections; import java.util.List; @@ -173,7 +172,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce public void accept(List> keyValues) { if (keyValues.size() != 0) { for (int i = 0; i < keyValues.size(); i++) { - String[] splitKeys = StringUtils.split(keyValues.get(i).getKey(), ":"); + String[] splitKeys = keyValues.get(i).getKey().split(":"); keyValue.put(splitKeys[1], splitKeys[2]); keyValue.put(splitKeys[3], keyValues.get(i).getValue()); } diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index 0bd053473..9460c4a76 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -24,7 +24,6 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,15 +107,15 @@ private GenericObjectPoolConfig setPoolConfig(String maxTotal, String maxIdle, S private void establishConnection() { poolConfig = setPoolConfig(maxTotal, maxIdle, minIdle); - String[] nodes = StringUtils.split(url, ","); - String[] firstIpPort = StringUtils.split(nodes[0], ":"); + String[] nodes = url.split(","); + String[] firstIpPort = nodes[0].split(":"); String firstIp = firstIpPort[0]; String firstPort = firstIpPort[1]; Set addresses = new HashSet<>(); Set ipPorts = new HashSet<>(); for (String ipPort : nodes) { ipPorts.add(ipPort); - String[] ipPortPair = StringUtils.split(ipPort, ":"); + String[] ipPortPair = ipPort.split(":"); addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.valueOf(ipPortPair[1].trim()))); } if (timeout == 0){ diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 0c9b0a64a..f01709874 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -50,7 +50,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map primaryKeysList = Lists.newArrayList(); if (!StringUtils.isEmpty(primaryKeysStr)) { - primaryKeysList = Arrays.asList(StringUtils.split(primaryKeysStr, ",")); + primaryKeysList = Arrays.asList(primaryKeysStr.split(",")); } redisTableInfo.setPrimaryKeys(primaryKeysList); From ae504c5f56892201f056fbdcbbeda221f13cad3e Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Wed, 26 Feb 2020 12:28:47 +0800 Subject: [PATCH 055/523] =?UTF-8?q?=E8=A7=A3=E5=86=B3elasticsearch-side?= =?UTF-8?q?=E5=88=86=E6=94=AF=E4=B8=8Etest=5F1.8=5F3.1.0x=E7=9A=84?= =?UTF-8?q?=E5=86=B2=E7=AA=812?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/parser/CreateTableParser.java | 3 +-- .../main/java/com/dtstack/flink/sql/side/SideSqlExec.java | 5 ++--- .../java/com/dtstack/flink/sql/table/AbsTableParser.java | 2 +- .../dtstack/flink/sql/sink/redis/table/RedisSinkParser.java | 1 + 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java index 9a2d064d9..ae6e1f708 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java @@ -22,7 +22,6 @@ import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.collect.Maps; -import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Map; @@ -70,7 +69,7 @@ public void parseSql(String sql, SqlTree sqlTree) { } private Map parseProp(String propsStr){ - String[] strs = StringUtils.split(propsStr.trim(), "'\\s*,"); + String[] strs = propsStr.trim().split("'\\s*,"); Map propMap = Maps.newHashMap(); for(int i=0; i ss = DtStringUtil.splitIgnoreQuota(strs[i], '='); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 479a330a2..4e718460e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -63,7 +63,6 @@ import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -893,9 +892,9 @@ private TypeInformation projectedTypeInfo(int[] fields, TableSchema schema) private boolean checkFieldsInfo(CreateTmpTableParser.SqlParserResult result, Table table) { List fieldNames = new LinkedList<>(); String fieldsInfo = result.getFieldsInfoStr(); - String[] fields = StringUtils.split(fieldsInfo, ","); + String[] fields = fieldsInfo.split(","); for (int i = 0; i < fields.length; i++) { - String[] filed = StringUtils.split(fields[i], "\\s"); + String[] filed = fields[i].split("\\s"); if (filed.length < 2 || fields.length != table.getSchema().getColumnNames().length){ return false; } else { diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java index 2895f2b2e..31e70caa2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java @@ -92,7 +92,7 @@ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ throw new RuntimeException(String.format("table [%s],exists field empty.", tableInfo.getName())); } - String[] filedInfoArr = StringUtils.split(fieldRow, "\\s+"); + String[] filedInfoArr = fieldRow.split("\\s+"); if(filedInfoArr.length < 2 ){ throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow)); } diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index f01709874..49861bb9a 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -24,6 +24,7 @@ import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; From db77f9a9261d350c10ba494d1aa5c311320e6517 Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Wed, 26 Feb 2020 15:10:57 +0800 Subject: [PATCH 056/523] =?UTF-8?q?=E4=BC=98=E5=8C=96es6=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E7=9B=B8=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/AllReqRow.java | 12 +++++ docs/elasticsearch6Side | 2 +- .../Elasticsearch6AllReqRow.java | 44 +++++++++---------- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java index 7d16ee726..b56c5d7f7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java @@ -21,9 +21,12 @@ package com.dtstack.flink.sql.side; import com.dtstack.flink.sql.factory.DTThreadFactory; +import org.apache.calcite.sql.JoinType; import org.apache.flink.api.common.functions.RichFlatMapFunction; import org.apache.flink.configuration.Configuration; import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; import java.sql.SQLException; import java.util.concurrent.Executors; @@ -64,4 +67,13 @@ public void open(Configuration parameters) throws Exception { es.scheduleAtFixedRate(() -> reloadCache(), sideTableInfo.getCacheTimeout(), sideTableInfo.getCacheTimeout(), TimeUnit.MILLISECONDS); } + protected void sendOutputRow(CRow value, Object sideInput, Collector out){ + if(sideInput == null && sideInfo.getJoinType() != JoinType.LEFT){ + return; + } + + Row row = fillData(value.row(), sideInput); + out.collect(new CRow(row, value.change())); + } + } diff --git a/docs/elasticsearch6Side b/docs/elasticsearch6Side index 279db888d..36b229867 100644 --- a/docs/elasticsearch6Side +++ b/docs/elasticsearch6Side @@ -70,7 +70,7 @@ create table sideTable( PRIMARY KEY(channel), PERIOD FOR SYSTEM_TIME )WITH( - type ='elasticsearch', + type ='elasticsearch6', address ='172.16.10.47:9500', cluster='es_47_menghan', estype ='type1', diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index 82a43de66..648f09fba 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -80,11 +80,7 @@ public void flatMap(CRow value, Collector out) throws Exception { for (Integer conValIndex : sideInfo.getEqualValIndex()) { Object equalObj = value.row().getField(conValIndex); if (equalObj == null) { - if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value.row(), null); - out.collect(new CRow(row, value.change())); - } - + sendOutputRow(value, null, out); return; } @@ -94,18 +90,12 @@ public void flatMap(CRow value, Collector out) throws Exception { String key = buildKey(inputParams); List> cacheList = cacheRef.get().get(key); if (CollectionUtils.isEmpty(cacheList)) { - if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value.row(), null); - out.collect(new CRow(row, value.change())); - } else { - return; - } - + sendOutputRow(value, null, out); return; } for (Map one : cacheList) { - out.collect(new CRow(fillData(value.row(), one), value.change())); + sendOutputRow(value, one, out); } } @@ -155,7 +145,7 @@ private String buildKey(Map val, List equalFieldList) { } @Override - protected void initCache() throws SQLException { + protected void initCache() { Map>> newCache = Maps.newConcurrentMap(); cacheRef.set(newCache); try { @@ -165,6 +155,7 @@ protected void initCache() throws SQLException { loadData(newCache); } catch (Exception e) { LOG.error("", e); + throw new RuntimeException(e); } } @@ -176,6 +167,7 @@ protected void reloadCache() { loadData(newCache); } catch (Exception e) { LOG.error("", e); + throw new RuntimeException(e); } cacheRef.set(newCache); @@ -210,6 +202,7 @@ private void loadData(Map>> tmpCache) throws IO } catch (Exception e) { LOG.error("", e); + throw new RuntimeException(e); } finally { if (rhlClient != null) { @@ -220,18 +213,18 @@ private void loadData(Map>> tmpCache) throws IO // initialize searchSourceBuilder private SearchSourceBuilder initConfiguration(BoolQueryBuilder boolQueryBuilder){ - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - if (boolQueryBuilder != null) { - searchSourceBuilder.query(boolQueryBuilder); - } + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + if (boolQueryBuilder != null) { + searchSourceBuilder.query(boolQueryBuilder); + } - searchSourceBuilder.size(getFetchSize()); - searchSourceBuilder.sort("_id", SortOrder.DESC); + searchSourceBuilder.size(getFetchSize()); + searchSourceBuilder.sort("_id", SortOrder.DESC); - // fields included in the source data - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); - searchSourceBuilder.fetchSource(sideFieldNames, null); - return searchSourceBuilder; + // fields included in the source data + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); + searchSourceBuilder.fetchSource(sideFieldNames, null); + return searchSourceBuilder; } @@ -247,6 +240,7 @@ private void searchData(SearchSourceBuilder searchSourceBuilder, Map>> tmpCache) { String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); + for (SearchHit searchHit : searchHits) { Map oneRow = Maps.newHashMap(); for (String fieldName : sideFieldNames) { @@ -275,6 +270,7 @@ private void loadToCache(SearchHit[] searchHits, Map> list = tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()); list.add(oneRow); From 55b22b419248f2c1cbee5f6ed1cbe1c7d3605531 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Wed, 26 Feb 2020 16:47:28 +0800 Subject: [PATCH 057/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=80=E9=83=A8?= =?UTF-8?q?=E5=88=86string.split(=E6=AD=A3=E5=88=99=E8=A1=A8=E8=BE=BE?= =?UTF-8?q?=E5=BC=8F)=E9=94=99=E8=AF=AF=E6=9B=BF=E6=8D=A2=E6=88=90StringUt?= =?UTF-8?q?ils.split()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/parser/CreateTableParser.java | 3 +-- .../src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java | 2 +- .../main/java/com/dtstack/flink/sql/table/AbsTableParser.java | 2 +- .../main/java/com/dtstack/flink/sql/util/DtStringUtil.java | 2 +- .../java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java | 2 +- .../java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java | 4 ++-- .../dtstack/flink/sql/side/hbase/table/HbaseSideParser.java | 3 +-- .../dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java | 2 +- 8 files changed, 9 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java index 9a2d064d9..ae6e1f708 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java @@ -22,7 +22,6 @@ import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.collect.Maps; -import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Map; @@ -70,7 +69,7 @@ public void parseSql(String sql, SqlTree sqlTree) { } private Map parseProp(String propsStr){ - String[] strs = StringUtils.split(propsStr.trim(), "'\\s*,"); + String[] strs = propsStr.trim().split("'\\s*,"); Map propMap = Maps.newHashMap(); for(int i=0; i ss = DtStringUtil.splitIgnoreQuota(strs[i], '='); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 479a330a2..2d769f5a8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -895,7 +895,7 @@ private boolean checkFieldsInfo(CreateTmpTableParser.SqlParserResult result, Tab String fieldsInfo = result.getFieldsInfoStr(); String[] fields = StringUtils.split(fieldsInfo, ","); for (int i = 0; i < fields.length; i++) { - String[] filed = StringUtils.split(fields[i], "\\s"); + String[] filed = fields[i].split("\\s"); if (filed.length < 2 || fields.length != table.getSchema().getColumnNames().length){ return false; } else { diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java index 2895f2b2e..31e70caa2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java @@ -92,7 +92,7 @@ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ throw new RuntimeException(String.format("table [%s],exists field empty.", tableInfo.getName())); } - String[] filedInfoArr = StringUtils.split(fieldRow, "\\s+"); + String[] filedInfoArr = fieldRow.split("\\s+"); if(filedInfoArr.length < 2 ){ throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow)); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index 84fe49fd3..b2486b1ab 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -180,7 +180,7 @@ public static String addJdbcParam(String dbUrl, Map addParams, b return dbUrl; } - String[] splits = StringUtils.split(dbUrl, "\\?"); + String[] splits = dbUrl.split("\\?"); String preStr = splits[0]; Map params = Maps.newHashMap(); if(splits.length > 1){ diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java index 24c5d3ab9..4258d5aaa 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java @@ -42,7 +42,7 @@ public static Map rowToJsonMap(Row row, List fields, Lis int i = 0; for(; i < fields.size(); ++i) { String field = fields.get(i); - String[] parts = StringUtils.split(field, "\\."); + String[] parts = field.split("\\."); Map currMap = jsonMap; for(int j = 0; j < parts.length - 1; ++j) { String key = parts[j]; diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java index 6ad5544ab..f7c2aaba8 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java @@ -59,7 +59,7 @@ public void init(String rowKeyTempl){ fieldCols = infoAlias; } - String[] fieldArr = StringUtils.split(fieldCols, "\\+"); + String[] fieldArr = fieldCols.split("\\+"); List fieldList = Lists.newArrayList(); for(String oneField : fieldArr){ ReplaceInfo replaceInfo = getReplaceInfo(oneField); @@ -109,7 +109,7 @@ private String buildStr(List fieldList, Map refData public static String[] splitIgnoreQuotaBrackets(String str, String delimiter){ String splitPatternStr = delimiter + "(?![^()]*+\\))(?![^{}]*+})(?![^\\[\\]]*+\\])"; - return StringUtils.split(str, splitPatternStr); + return str.split(splitPatternStr); } /** diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 1ba9a9907..7b627da4d 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -23,7 +23,6 @@ import com.dtstack.flink.sql.table.AbsSideTableParser; import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; -import org.apache.commons.lang3.StringUtils; import java.util.Map; import java.util.regex.Matcher; @@ -84,7 +83,7 @@ private void dealField(Matcher matcher, TableInfo tableInfo){ String filedDefineStr = matcher.group(1); String aliasStr = matcher.group(2); - String[] filedInfoArr = StringUtils.split(filedDefineStr, "\\s+"); + String[] filedInfoArr = filedDefineStr.split("\\s+"); if(filedInfoArr.length < 2){ throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), matcher.group(0))); } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index e93e8e612..064b13e69 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -81,7 +81,7 @@ public void parseFieldsInfo(String fieldsInfo, HbaseTableInfo tableInfo){ for(String fieldRow : fieldRows){ fieldRow = fieldRow.trim(); - String[] filedInfoArr = StringUtils.split(fieldRow, "\\s+"); + String[] filedInfoArr = fieldRow.split("\\s+"); if(filedInfoArr.length < 2 ){ throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow)); } From dc7134ef992a54bb121accf637727f149f60e81d Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Wed, 26 Feb 2020 17:10:58 +0800 Subject: [PATCH 058/523] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E7=9A=84import?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java | 1 - .../java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java | 1 - 2 files changed, 2 deletions(-) diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java index 4258d5aaa..5a0cd205e 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/EsUtil.java @@ -22,7 +22,6 @@ import org.apache.flink.util.Preconditions; import com.dtstack.flink.sql.util.DtStringUtil; -import org.apache.commons.lang3.StringUtils; import java.util.HashMap; import java.util.List; diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java index f7c2aaba8..114b7fa6a 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java @@ -22,7 +22,6 @@ import com.dtstack.flink.sql.side.hbase.enums.EReplaceType; import com.google.common.collect.Lists; -import org.apache.commons.lang3.StringUtils; import java.io.Serializable; import java.util.List; From dff67abe298433669c4af5e79e595d4fd7a89108 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 26 Feb 2020 20:37:13 +0800 Subject: [PATCH 059/523] remove log level --- .../flink/sql/exec/ExecuteProcessHelper.java | 10 ---------- .../com/dtstack/flink/sql/exec/ParamsInfo.java | 14 ++------------ .../java/com/dtstack/flink/sql/option/Options.java | 9 --------- 3 files changed, 2 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 2c6d74df0..98b6a03ef 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -110,7 +110,6 @@ public static ParamsInfo parseParams(String[] args) throws Exception { String remoteSqlPluginPath = options.getRemoteSqlPluginPath(); String pluginLoadMode = options.getPluginLoadMode(); String deployMode = options.getMode(); - String logLevel = options.getLogLevel(); Preconditions.checkArgument(checkRemoteSqlPluginPath(remoteSqlPluginPath, deployMode, pluginLoadMode), "Non-local mode or shipfile deployment mode, remoteSqlPluginPath is required"); @@ -128,7 +127,6 @@ public static ParamsInfo parseParams(String[] args) throws Exception { .setDeployMode(deployMode) .setConfProp(confProperties) .setJarUrlList(jarURList) - .setLogLevel(logLevel) .build(); } @@ -154,8 +152,6 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); StreamQueryConfig streamQueryConfig = StreamEnvConfigManager.getStreamQueryConfig(tableEnv, paramsInfo.getConfProp()); - setLogLevel(paramsInfo.getLogLevel()); - SqlParser.setLocalSqlPluginRoot(paramsInfo.getLocalSqlPluginPath()); SqlTree sqlTree = SqlParser.parseSql(paramsInfo.getSql()); @@ -348,11 +344,5 @@ public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperti return env; } - private static void setLogLevel(String level){ - LoggerContext loggerContext= (LoggerContext) LoggerFactory.getILoggerFactory(); - //设置全局日志级别 - ch.qos.logback.classic.Logger logger = loggerContext.getLogger("root"); - logger.setLevel(Level.toLevel(level, Level.INFO)); - } } \ No newline at end of file diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java index 59249827c..b36dc31eb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java @@ -38,11 +38,10 @@ public class ParamsInfo { private String remoteSqlPluginPath; private String pluginLoadMode; private String deployMode; - private String logLevel; private Properties confProp; public ParamsInfo(String sql, String name, List jarUrlList, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, String deployMode, String logLevel, Properties confProp) { + String remoteSqlPluginPath, String pluginLoadMode, String deployMode, Properties confProp) { this.sql = sql; this.name = name; this.jarUrlList = jarUrlList; @@ -50,7 +49,6 @@ public ParamsInfo(String sql, String name, List jarUrlList, String localSql this.remoteSqlPluginPath = remoteSqlPluginPath; this.pluginLoadMode = pluginLoadMode; this.deployMode = deployMode; - this.logLevel = logLevel; this.confProp = confProp; } @@ -86,9 +84,6 @@ public Properties getConfProp() { return confProp; } - public String getLogLevel() { - return logLevel; - } @Override public String toString() { @@ -100,7 +95,6 @@ public String toString() { ", remoteSqlPluginPath='" + remoteSqlPluginPath + '\'' + ", pluginLoadMode='" + pluginLoadMode + '\'' + ", deployMode='" + deployMode + '\'' + - ", logLevel='" + logLevel + '\'' + ", confProp=" + confProp + '}'; } @@ -160,10 +154,6 @@ public ParamsInfo.Builder setDeployMode(String deployMode) { return this; } - public ParamsInfo.Builder setLogLevel(String logLevel) { - this.logLevel = logLevel; - return this; - } public ParamsInfo.Builder setConfProp(Properties confProp) { this.confProp = confProp; @@ -172,7 +162,7 @@ public ParamsInfo.Builder setConfProp(Properties confProp) { public ParamsInfo build() { return new ParamsInfo(sql, name, jarUrlList, localSqlPluginPath, - remoteSqlPluginPath, pluginLoadMode, deployMode, logLevel, confProp); + remoteSqlPluginPath, pluginLoadMode, deployMode, confProp); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/option/Options.java b/core/src/main/java/com/dtstack/flink/sql/option/Options.java index 91ecf19fc..75f9ed764 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/Options.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/Options.java @@ -69,8 +69,6 @@ public class Options { @OptionRequired(description = "plugin load mode, by classpath or shipfile") private String pluginLoadMode = EPluginLoadMode.CLASSPATH.name(); - private String logLevel; - public String getMode() { return mode; } @@ -175,11 +173,4 @@ public void setPluginLoadMode(String pluginLoadMode) { this.pluginLoadMode = pluginLoadMode; } - public String getLogLevel() { - return logLevel; - } - - public void setLogLevel(String logLevel) { - this.logLevel = logLevel; - } } From 7a57f791c19a3d206551b0b0aea867a00490232b Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Wed, 26 Feb 2020 21:34:07 +0800 Subject: [PATCH 060/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=8C=85=E5=90=8D?= =?UTF-8?q?=E3=80=81=E7=B1=BB=E5=90=8D=EF=BC=8C=E8=A7=A3=E5=86=B3=E6=89=BE?= =?UTF-8?q?=E4=B8=8D=E5=88=B0=E7=B1=BB=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../CustomerSinkFunc.java | 2 +- .../ElasticsearchSink.java} | 10 +++++----- .../{elasticsearch6 => elasticsearch}/Es6Util.java | 2 +- .../ExtendEs6ApiCallBridge.java | 4 ++-- .../MetricElasticsearch6Sink.java | 4 ++-- .../table/ElasticsearchSinkParser.java | 2 +- .../table/ElasticsearchTableInfo.java | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) rename elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/{elasticsearch6 => elasticsearch}/CustomerSinkFunc.java (98%) rename elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/{elasticsearch6/Elasticsearch6Sink.java => elasticsearch/ElasticsearchSink.java} (94%) rename elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/{elasticsearch6 => elasticsearch}/Es6Util.java (97%) rename elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/{elasticsearch6 => elasticsearch}/ExtendEs6ApiCallBridge.java (97%) rename elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/{elasticsearch6 => elasticsearch}/MetricElasticsearch6Sink.java (95%) rename elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/{elasticsearch6 => elasticsearch}/table/ElasticsearchSinkParser.java (98%) rename elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/{elasticsearch6 => elasticsearch}/table/ElasticsearchTableInfo.java (98%) diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java similarity index 98% rename from elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java rename to elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index 357e742c3..d6d207c9e 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/CustomerSinkFunc.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.dtstack.flink.sql.sink.elasticsearch6; +package com.dtstack.flink.sql.sink.elasticsearch; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.api.java.tuple.Tuple2; diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Elasticsearch6Sink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java similarity index 94% rename from elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Elasticsearch6Sink.java rename to elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 5ac50bc8d..115c3c5d3 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Elasticsearch6Sink.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.dtstack.flink.sql.sink.elasticsearch6; +package com.dtstack.flink.sql.sink.elasticsearch; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -30,7 +30,7 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.elasticsearch6.table.ElasticsearchTableInfo; +import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.collect.Maps; import org.apache.commons.lang.StringUtils; @@ -47,9 +47,9 @@ * @author yinxi * @date 2020/1/9 - 15:08 */ -public class Elasticsearch6Sink implements RetractStreamTableSink, IStreamSinkGener { +public class ElasticsearchSink implements RetractStreamTableSink, IStreamSinkGener { - private final Logger logger = LoggerFactory.getLogger(Elasticsearch6Sink.class); + private final Logger logger = LoggerFactory.getLogger(ElasticsearchSink.class); private String clusterName; @@ -150,7 +150,7 @@ public void setBulkFlushMaxActions(int bulkFlushMaxActions) { } @Override - public Elasticsearch6Sink genStreamSink(TargetTableInfo targetTableInfo) { + public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { ElasticsearchTableInfo elasticsearchTableInfo = (ElasticsearchTableInfo) targetTableInfo; esTableInfo = elasticsearchTableInfo; clusterName = elasticsearchTableInfo.getClusterName(); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Es6Util.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/Es6Util.java similarity index 97% rename from elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Es6Util.java rename to elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/Es6Util.java index df892381d..ccbb0d252 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/Es6Util.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/Es6Util.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.dtstack.flink.sql.sink.elasticsearch6; +package com.dtstack.flink.sql.sink.elasticsearch; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendEs6ApiCallBridge.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ExtendEs6ApiCallBridge.java similarity index 97% rename from elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendEs6ApiCallBridge.java rename to elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ExtendEs6ApiCallBridge.java index 2eafb8a2b..5f9cf2fff 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/ExtendEs6ApiCallBridge.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ExtendEs6ApiCallBridge.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.dtstack.flink.sql.sink.elasticsearch6; +package com.dtstack.flink.sql.sink.elasticsearch; import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchApiCallBridge; import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase; import org.apache.flink.util.Preconditions; -import com.dtstack.flink.sql.sink.elasticsearch6.table.ElasticsearchTableInfo; +import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/MetricElasticsearch6Sink.java similarity index 95% rename from elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java rename to elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/MetricElasticsearch6Sink.java index aa03e943a..b17b1243a 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/MetricElasticsearch6Sink.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/MetricElasticsearch6Sink.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.dtstack.flink.sql.sink.elasticsearch6; +package com.dtstack.flink.sql.sink.elasticsearch; import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.Counter; @@ -27,7 +27,7 @@ import org.apache.flink.streaming.connectors.elasticsearch.util.NoOpFailureHandler; import com.dtstack.flink.sql.metric.MetricConstant; -import com.dtstack.flink.sql.sink.elasticsearch6.table.ElasticsearchTableInfo; +import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; import org.elasticsearch.client.RestHighLevelClient; import java.util.List; diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchSinkParser.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java similarity index 98% rename from elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchSinkParser.java rename to elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 4b577f7ac..8902b953c 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchSinkParser.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.dtstack.flink.sql.sink.elasticsearch6.table; +package com.dtstack.flink.sql.sink.elasticsearch.table; import com.dtstack.flink.sql.table.AbsTableParser; import com.dtstack.flink.sql.table.TableInfo; diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchTableInfo.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java similarity index 98% rename from elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchTableInfo.java rename to elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index eb7dc1c4f..b0156b660 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch6/table/ElasticsearchTableInfo.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.dtstack.flink.sql.sink.elasticsearch6.table; +package com.dtstack.flink.sql.sink.elasticsearch.table; import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; From 73befd6e0420286f409fdb0c4411ae7a1e2f39a2 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Wed, 26 Feb 2020 22:32:01 +0800 Subject: [PATCH 061/523] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E4=B8=8E1.8=5Ftest?= =?UTF-8?q?=5F3.1.0x=E5=88=86=E6=94=AF=E7=9A=84=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 -- .../side/cassandra/CassandraAllReqRow.java | 9 ++-- .../side/cassandra/CassandraAsyncReqRow.java | 7 ++-- .../sink/cassandra/CassandraOutputFormat.java | 9 ++-- core/pom.xml | 4 +- .../main/java/com/dtstack/flink/sql/Main.java | 11 +---- .../flink/sql/constrant/ConfigConstrant.java | 6 +-- .../environment/StreamEnvConfigManager.java | 41 ++++++------------- .../com/dtstack/flink/sql/exec/ApiResult.java | 2 + .../com/dtstack/flink/sql/side/AllReqRow.java | 15 +------ .../dtstack/flink/sql/side/SideSqlExec.java | 3 +- .../flink/sql/table/TableInfoParser.java | 3 +- .../dtstack/flink/sql/util/DtStringUtil.java | 4 +- .../sink/elasticsearch/ElasticsearchSink.java | 4 +- .../sql/sink/hbase/HbaseOutputFormat.java | 2 +- .../sql/sink/hbase/table/HbaseSinkParser.java | 5 +-- .../side/impala/table/ImpalaSideParser.java | 5 +-- .../sink/impala/table/ImpalaTableInfo.java | 3 +- .../flink/sql/source/kafka/KafkaConsumer.java | 3 +- .../sql/source/kafka/KafkaConsumer09.java | 3 +- .../sql/source/kafka/KafkaConsumer010.java | 3 +- .../sql/source/kafka/KafkaConsumer011.java | 3 +- .../flink/sql/side/kudu/KuduAllReqRow.java | 7 ++-- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 5 +-- .../flink/sql/side/kudu/utils/KuduUtil.java | 3 +- .../flink/sql/sink/kudu/KuduOutputFormat.java | 7 ++-- .../sql/launcher/perjob/PerJobSubmitter.java | 2 +- .../flink/sql/side/mongo/MongoAllReqRow.java | 6 +-- .../sql/side/mongo/MongoAsyncReqRow.java | 4 +- .../flink/sql/side/mongo/utils/MongoUtil.java | 5 +-- .../sql/sink/mongo/MongoOutputFormat.java | 4 +- .../flink/sql/side/rdb/all/RdbAllReqRow.java | 3 +- .../sql/side/rdb/all/RdbAllSideInfo.java | 2 +- .../sql/side/rdb/async/RdbAsyncSideInfo.java | 3 +- .../flink/sql/side/redis/RedisAllReqRow.java | 19 ++++----- .../sql/side/redis/RedisAsyncReqRow.java | 3 +- .../sql/sink/redis/RedisOutputFormat.java | 7 ++-- .../sql/sink/redis/table/RedisSinkParser.java | 2 +- 38 files changed, 87 insertions(+), 143 deletions(-) diff --git a/README.md b/README.md index a5b0457ab..583958908 100644 --- a/README.md +++ b/README.md @@ -149,9 +149,6 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack * taskmanager.slots：per_job模式下指定每个taskmanager对应的slot数量(默认1) * savePointPath：任务恢复点的路径（默认无） * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） - * restore.enable：是否失败重启（默认是true） - * failure.interval：衡量失败率的时间段，单位分钟（默认6m） - * delay.interval：连续两次重启尝试间的间隔，单位是秒（默认10s） * [prometheus 相关参数](docs/prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index bc0a0f8cd..4439c2b7a 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -43,7 +43,6 @@ import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -224,9 +223,9 @@ private Session getConn(CassandraSideTableInfo tableInfo) { //重试策略 RetryPolicy retryPolicy = DowngradingConsistencyRetryPolicy.INSTANCE; - for (String server : StringUtils.split(address, ",")) { - cassandraPort = Integer.parseInt(StringUtils.split(server, ":")[1]); - serversList.add(InetAddress.getByName(StringUtils.split(server, ":")[0])); + for (String server : address.split(",")) { + cassandraPort = Integer.parseInt(server.split(":")[1]); + serversList.add(InetAddress.getByName(server.split(":")[0])); } if (userName == null || userName.isEmpty() || password == null || password.isEmpty()) { @@ -280,7 +279,7 @@ private void loadData(Map>> tmpCache) throws SQ //load data from table String sql = sideInfo.getSqlCondition() + " limit " + FETCH_SIZE; ResultSet resultSet = session.execute(sql); - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); + String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); for (com.datastax.driver.core.Row row : resultSet) { Map oneRow = Maps.newHashMap(); for (String fieldName : sideFieldNames) { diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index 0173f2d6f..7a209f30e 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -51,7 +51,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import io.vertx.core.json.JsonArray; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,9 +134,9 @@ private void connCassandraDB(CassandraSideTableInfo tableInfo) { //重试策略 RetryPolicy retryPolicy = DowngradingConsistencyRetryPolicy.INSTANCE; - for (String server : StringUtils.split(address, ",")) { - cassandraPort = Integer.parseInt(StringUtils.split(server, ":")[1]); - serversList.add(InetAddress.getByName(StringUtils.split(server, ":")[0])); + for (String server : address.split(",")) { + cassandraPort = Integer.parseInt(server.split(":")[1]); + serversList.add(InetAddress.getByName(server.split(":")[0])); } if (userName == null || userName.isEmpty() || password == null || password.isEmpty()) { diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java index 08a2358cb..d8f4eb451 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java @@ -55,7 +55,6 @@ import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -148,9 +147,9 @@ public void open(int taskNumber, int numTasks) { //重试策略 RetryPolicy retryPolicy = DowngradingConsistencyRetryPolicy.INSTANCE; - for (String server : StringUtils.split(address, ",")) { - cassandraPort = Integer.parseInt(StringUtils.split(server, ":")[1]); - serversList.add(InetAddress.getByName(StringUtils.split(server, ":")[0])); + for (String server : address.split(",")) { + cassandraPort = Integer.parseInt(server.split(":")[1]); + serversList.add(InetAddress.getByName(server.split(":")[0])); } if (userName == null || userName.isEmpty() || password == null || password.isEmpty()) { @@ -219,7 +218,7 @@ private void insertWrite(Row row) { } } catch (Exception e) { if(outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0){ - LOG.error("record insert failed ..", row.toString().substring(0, 100)); + LOG.error("record insert failed, total dirty num:{}, current record:{}", outDirtyRecords.getCount(), row.toString()); LOG.error("", e); } diff --git a/core/pom.xml b/core/pom.xml index 4b5a3e67b..b17209612 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -20,6 +20,7 @@ 1.16.0 2.7.9 19.0 + 1.1.7 @@ -121,10 +122,11 @@ junit 4.12 + ch.qos.logback logback-classic - 1.1.7 + ${logback.version} diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index 4ff296f1f..b54c3feb2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -21,14 +21,13 @@ +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + import com.dtstack.flink.sql.exec.ExecuteProcessHelper; import com.dtstack.flink.sql.exec.ParamsInfo; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.LoggerContext; /** * Date: 2018/6/26 @@ -45,10 +44,4 @@ public static void main(String[] args) throws Exception { env.execute(paramsInfo.getName()); LOG.info("program {} execution success", paramsInfo.getName()); } - private static void setLogLevel(String level){ - LoggerContext loggerContext= (LoggerContext) LoggerFactory.getILoggerFactory(); - //设置全局日志级别 - ch.qos.logback.classic.Logger logger = loggerContext.getLogger("root"); - logger.setLevel(Level.toLevel(level, Level.INFO)); - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java index a80962661..dcda4c093 100644 --- a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java @@ -63,14 +63,12 @@ public class ConfigConstrant { public static final String CHECKPOINTS_DIRECTORY_KEY = "state.checkpoints.dir"; public static final String STATE_BACKEND_INCREMENTAL_KEY = "state.backend.incremental"; - public static final String RESTOREENABLE = "restore.enable"; - // restart plocy public static final int failureRate = 3; - public static final String FAILUREINTERVAL = "failure.interval"; //min + public static final int failureInterval = 6; //min - public static final String DELAYINTERVAL= "delay.interval"; //sec + public static final int delayInterval = 10; //sec } diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index 28eac2af3..ce763ea99 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -18,12 +18,6 @@ package com.dtstack.flink.sql.environment; -import com.dtstack.flink.sql.constrant.ConfigConstrant; -import com.dtstack.flink.sql.enums.EStateBackend; -import com.dtstack.flink.sql.util.MathUtil; -import com.dtstack.flink.sql.util.PropertiesUtils; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.time.Time; @@ -40,6 +34,13 @@ import org.apache.flink.table.api.StreamQueryConfig; import org.apache.flink.table.api.java.StreamTableEnvironment; +import com.dtstack.flink.sql.constrant.ConfigConstrant; +import com.dtstack.flink.sql.enums.EStateBackend; +import com.dtstack.flink.sql.util.MathUtil; +import com.dtstack.flink.sql.util.PropertiesUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; + import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -101,15 +102,11 @@ public static void streamExecutionEnvironmentConfig(StreamExecutionEnvironment s } }); - if(isRestore(confProperties).get()){ - streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( - ConfigConstrant.failureRate, - Time.of(getFailureInterval(confProperties).get(), TimeUnit.MINUTES), - Time.of(getDelayInterval(confProperties).get(), TimeUnit.SECONDS) - )); - } else { - streamEnv.setRestartStrategy(RestartStrategies.noRestart()); - } + streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( + ConfigConstrant.failureRate, + Time.of(ConfigConstrant.failureInterval, TimeUnit.MINUTES), + Time.of(ConfigConstrant.delayInterval, TimeUnit.SECONDS) + )); // checkpoint config Optional checkpointingEnabled = isCheckpointingEnabled(confProperties); @@ -167,20 +164,6 @@ public static Optional getAutoWatermarkInterval(Properties properties) { return StringUtils.isNotBlank(autoWatermarkInterval) ? Optional.of(Long.valueOf(autoWatermarkInterval)) : Optional.empty(); } - public static Optional isRestore(Properties properties){ - String restoreEnable = properties.getProperty(ConfigConstrant.RESTOREENABLE, "true"); - return Optional.of(Boolean.valueOf(restoreEnable)); - } - - public static Optional getDelayInterval(Properties properties){ - String delayInterval = properties.getProperty(ConfigConstrant.DELAYINTERVAL, "10"); - return Optional.of(Integer.valueOf(delayInterval)); - } - public static Optional getFailureInterval(Properties properties){ - String failureInterval = properties.getProperty(ConfigConstrant.FAILUREINTERVAL, "6"); - return Optional.of(Integer.valueOf(failureInterval)); - } - /** * #ProcessingTime(默认), IngestionTime, EventTime * @param properties diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java b/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java index d6ca857c3..2da763988 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java @@ -4,6 +4,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.UUID; + /** * API调用结果返回 * Date: 2020/2/24 diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java index b56c5d7f7..a956137b0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java @@ -20,13 +20,11 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.factory.DTThreadFactory; -import org.apache.calcite.sql.JoinType; import org.apache.flink.api.common.functions.RichFlatMapFunction; import org.apache.flink.configuration.Configuration; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; + +import com.dtstack.flink.sql.factory.DTThreadFactory; import java.sql.SQLException; import java.util.concurrent.Executors; @@ -67,13 +65,4 @@ public void open(Configuration parameters) throws Exception { es.scheduleAtFixedRate(() -> reloadCache(), sideTableInfo.getCacheTimeout(), sideTableInfo.getCacheTimeout(), TimeUnit.MILLISECONDS); } - protected void sendOutputRow(CRow value, Object sideInput, Collector out){ - if(sideInput == null && sideInfo.getJoinType() != JoinType.LEFT){ - return; - } - - Row row = fillData(value.row(), sideInput); - out.collect(new CRow(row, value.change())); - } - } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 2d769f5a8..4e718460e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -63,7 +63,6 @@ import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -893,7 +892,7 @@ private TypeInformation projectedTypeInfo(int[] fields, TableSchema schema) private boolean checkFieldsInfo(CreateTmpTableParser.SqlParserResult result, Table table) { List fieldNames = new LinkedList<>(); String fieldsInfo = result.getFieldsInfoStr(); - String[] fields = StringUtils.split(fieldsInfo, ","); + String[] fields = fieldsInfo.split(","); for (int i = 0; i < fields.length; i++) { String[] filed = fields[i].split("\\s"); if (filed.length < 2 || fields.length != table.getSchema().getColumnNames().length){ diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java index f4c638381..ae98d90ae 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java @@ -29,7 +29,6 @@ import com.dtstack.flink.sql.util.MathUtil; import com.google.common.base.Strings; import com.google.common.collect.Maps; -import org.apache.commons.lang3.StringUtils; import java.util.Map; import java.util.regex.Matcher; @@ -111,7 +110,7 @@ public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserRe * @return */ private static boolean checkIsSideTable(String tableField){ - String[] fieldInfos = StringUtils.split(tableField, ","); + String[] fieldInfos = tableField.split(","); for(String field : fieldInfos){ Matcher matcher = SIDE_PATTERN.matcher(field.trim()); if(matcher.find()){ diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index b2486b1ab..862b0700e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -185,9 +185,9 @@ public static String addJdbcParam(String dbUrl, Map addParams, b Map params = Maps.newHashMap(); if(splits.length > 1){ String existsParamStr = splits[1]; - String[] existsParams = StringUtils.split(existsParamStr, "&"); + String[] existsParams = existsParamStr.split("&"); for(String oneParam : existsParams){ - String[] kv = StringUtils.split(oneParam, "="); + String[] kv = oneParam.split("="); if(kv.length != 2){ throw new RuntimeException("illegal dbUrl:" + dbUrl); } diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 491d1af27..0bfc338e3 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -118,7 +118,7 @@ private RichSinkFunction createEsSinkFunction(){ List transports = new ArrayList<>(); for(String address : esAddressList){ - String[] infoArray = StringUtils.split(address, ":"); + String[] infoArray = address.split(":"); int port = 9300; String host = infoArray[0]; if(infoArray.length > 1){ @@ -169,7 +169,7 @@ public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { esTableInfo = elasticsearchTableInfo; clusterName = elasticsearchTableInfo.getClusterName(); String address = elasticsearchTableInfo.getAddress(); - String[] addr = StringUtils.split(address, ","); + String[] addr = address.split(","); esAddressList = Arrays.asList(addr); index = elasticsearchTableInfo.getIndex(); type = elasticsearchTableInfo.getEsType(); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index d41c65723..fd94e0db2 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -281,7 +281,7 @@ public HbaseOutputFormat finish() { String[] columns = keySet.toArray(new String[keySet.size()]); for (int i = 0; i < columns.length; ++i) { String col = columns[i]; - String[] part = StringUtils.split(col, ":"); + String[] part = col.split(":"); families[i] = part[0]; qualifiers[i] = part[1]; } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index 064b13e69..f1ae72b6b 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -26,7 +26,6 @@ import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.MathUtil; -import org.apache.commons.lang3.StringUtils; import java.util.LinkedHashMap; import java.util.List; @@ -69,7 +68,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); + super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java index 78eabde09..2fa0d16b6 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java @@ -31,7 +31,6 @@ import org.apache.flink.util.SerializedValue; import com.dtstack.flink.sql.format.DeserializationMetricWrapper; -import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.Map; @@ -51,7 +50,7 @@ public class KafkaConsumer09 extends FlinkKafkaConsumer09 { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer09(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); + super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java index b78f47cd5..74f45581c 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java @@ -30,7 +30,6 @@ import org.apache.flink.util.SerializedValue; import com.dtstack.flink.sql.format.DeserializationMetricWrapper; -import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.Map; @@ -50,7 +49,7 @@ public class KafkaConsumer010 extends FlinkKafkaConsumer010 { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer010(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); + super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java index 276ce211f..a88fb1f9c 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java @@ -30,7 +30,6 @@ import org.apache.flink.util.SerializedValue; import com.dtstack.flink.sql.format.DeserializationMetricWrapper; -import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.Map; @@ -50,7 +49,7 @@ public class KafkaConsumer011 extends FlinkKafkaConsumer011 { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer011(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); + super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 63ff31c50..374261d40 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -18,7 +18,6 @@ import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.KuduClient; @@ -161,7 +160,7 @@ private void loadData(Map>> tmpCache) { } //load data from table assert scanner != null; - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); + String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); while (scanner.hasMoreRows()) { @@ -264,7 +263,7 @@ private KuduScanner buildScanner(KuduScanner.KuduScannerBuilder builder, Schema Long limitNum = tableInfo.getLimitNum(); Boolean isFaultTolerant = tableInfo.getFaultTolerant(); //查询需要的字段 - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); + String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); //主键过滤条件 主键最小值 String lowerBoundPrimaryKey = tableInfo.getLowerBoundPrimaryKey(); //主键过滤条件 主键最大值 @@ -318,7 +317,7 @@ private KuduScanner buildScanner(KuduScanner.KuduScannerBuilder builder, Schema } private String[] splitString(String data) { - return StringUtils.split(data, ","); + return data.split(","); } @Override diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index e1705d796..588bd00ca 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -23,7 +23,6 @@ import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; import io.vertx.core.json.JsonArray; -import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.AsyncKuduClient; @@ -114,7 +113,7 @@ private void connKuDu() throws KuduException { Long limitNum = kuduSideTableInfo.getLimitNum(); Boolean isFaultTolerant = kuduSideTableInfo.getFaultTolerant(); //查询需要的字段 - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); + String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); if (null == limitNum || limitNum <= 0) { scannerBuilder.limit(FETCH_SIZE); @@ -281,7 +280,7 @@ public GetListRowCB() { public Deferred> call(RowResultIterator results) throws Exception { for (RowResult result : results) { Map oneRow = Maps.newHashMap(); - for (String sideFieldName1 : StringUtils.split(sideInfo.getSideSelectFields(), ",")) { + for (String sideFieldName1 : sideInfo.getSideSelectFields().split(",")) { String sideFieldName = sideFieldName1.trim(); ColumnSchema columnSchema = table.getSchema().getColumn(sideFieldName); if (null != columnSchema) { diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java index b750e2d4a..c5cd8cdb2 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java @@ -20,7 +20,6 @@ import com.dtstack.flink.sql.side.PredicateInfo; import org.apache.commons.lang.math.NumberUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.Type; @@ -164,7 +163,7 @@ public static KuduPredicate buildKuduPredicate(Schema schema, PredicateInfo info case "IN": case "NOT_IN": case "BETWEEN": - value = Arrays.asList(StringUtils.split(info.getCondition(), ",")).stream() + value = Arrays.asList(info.getCondition().split(",")).stream() .map(val -> KuduUtil.getValue(val.trim(), column.getType())).collect(Collectors.toList()); break; case "IS_NOT_NULL": diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index 780319d82..1ef22ffd9 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -18,11 +18,12 @@ package com.dtstack.flink.sql.sink.kudu; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; + +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import org.apache.kudu.client.AsyncKuduClient; import org.apache.kudu.client.AsyncKuduSession; import org.apache.kudu.client.KuduClient; @@ -123,7 +124,7 @@ public void writeRecord(Tuple2 record) throws IOException { Row row = tupleTrans.getField(1); if (row.getArity() != fieldNames.length) { if(outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0) { - LOG.error("record insert failed ..{}", row.toString()); + LOG.error("record insert failed:{}", row.toString()); LOG.error("cause by row.getArity() != fieldNames.length"); } outDirtyRecords.inc(); @@ -142,7 +143,7 @@ public void writeRecord(Tuple2 record) throws IOException { outRecords.inc(); } catch (KuduException e) { if(outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0){ - LOG.error("record insert failed ..{}", row.toString().substring(0, 100)); + LOG.error("record insert failed, total dirty record:{} current row:{}", outDirtyRecords.getCount(), row.toString()); LOG.error("", e); } outDirtyRecords.inc(); diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index 29cc4890a..918451f7a 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -83,7 +83,7 @@ private static List getJarPaths(String addjarPath) { if (addjarPath.length() > 2) { addjarPath = addjarPath.substring(1,addjarPath.length()-1).replace("\"",""); } - List paths = Arrays.asList(StringUtils.split(addjarPath, ",")); + List paths = Arrays.asList(addjarPath.split(",")); return paths; } diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index f90cfd973..1a0565268 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -185,13 +185,13 @@ private MongoCollection getConn(String address, String userName, String password MongoCollection dbCollection; try { MongoCredential credential; - String[] servers = StringUtils.split(address, ","); + String[] servers = address.split(","); String host; Integer port; String[] hostAndPort; List lists = new ArrayList<>(); for (String server : servers) { - hostAndPort = StringUtils.split(server, ":"); + hostAndPort = server.split(":"); host = hostAndPort[0]; port = Integer.parseInt(hostAndPort[1]); lists.add(new ServerAddress(host, port)); @@ -237,7 +237,7 @@ private void loadData(Map>> tmpCache) throws SQ } //load data from table - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); + String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); BasicDBObject basicDBObject = new BasicDBObject(); for (String selectField : sideFieldNames) { basicDBObject.append(selectField, 1); diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 42749e935..ac6b4120c 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -92,13 +92,13 @@ public void open(Configuration parameters) throws Exception { public void connMongoDB() throws Exception { MongoCredential mongoCredential; - String[] servers = StringUtils.split(MongoSideTableInfo.getAddress(), ","); + String[] servers = MongoSideTableInfo.getAddress().split(","); String host; Integer port; String[] hostAndPort; List lists = new ArrayList<>(); for (String server : servers) { - hostAndPort = StringUtils.split(server, ":"); + hostAndPort = server.split(":"); host = hostAndPort[0]; port = Integer.parseInt(hostAndPort[1]); lists.add(new ServerAddress(host, port)); diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java index 2f1a2092d..0b345d947 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java @@ -20,7 +20,6 @@ import com.dtstack.flink.sql.side.PredicateInfo; import com.mongodb.BasicDBObject; -import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.stream.Collectors; @@ -46,11 +45,11 @@ public static BasicDBObject buildFilterObject(PredicateInfo info) { case "<>": return new BasicDBObject("$ne", info.getCondition()); case "IN": - Object[] values = Arrays.stream(StringUtils.split(info.getCondition(), ",")).map(String::trim) + Object[] values = Arrays.stream(info.getCondition().split(",")).map(String::trim) .collect(Collectors.toList()).toArray(); return new BasicDBObject("$in", values); case "NOT IN": - return new BasicDBObject("$nin", StringUtils.split(info.getCondition(), ",")); + return new BasicDBObject("$nin", info.getCondition().split(",")); case "IS NOT NULL": return new BasicDBObject("$exists", true); case "IS NULL": diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index 337fbad59..d5a9b4a82 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -127,13 +127,13 @@ public void close() { private void establishConnection() { try { MongoCredential credential; - String[] servers = StringUtils.split(address, ","); + String[] servers = address.split(","); String host; Integer port; String[] hostAndPort; List lists = new ArrayList<>(); for (String server : servers) { - hostAndPort = StringUtils.split(server, ":"); + hostAndPort = server.split(":"); host = hostAndPort[0]; port = Integer.parseInt(hostAndPort[1]); lists.add(new ServerAddress(host, port)); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java index 7fdeeeca9..c746d1185 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java @@ -31,7 +31,6 @@ import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -199,7 +198,7 @@ private void loadData(Map>> tmpCache) throws SQ Statement statement = connection.createStatement(); statement.setFetchSize(getFetchSize()); ResultSet resultSet = statement.executeQuery(sql); - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); + String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); while (resultSet.next()) { Map oneRow = Maps.newHashMap(); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 3b7444b77..313594050 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -56,7 +56,7 @@ public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List> tmpCache) throws SQLExcep Set keys = ((Jedis) jedis).keys(perKey); List newPerKeys = new LinkedList<>(); for (String key : keys){ - String[] splitKey = StringUtils.split(key, ":"); + String[] splitKey = key.split(":"); String newKey = splitKey[0] + ":" + splitKey[1] + ":" + splitKey[2]; newPerKeys.add(newKey); } List list = newPerKeys.stream().distinct().collect(Collectors.toList()); for(String key : list){ Map kv = Maps.newHashMap(); - String[] primaryKv = StringUtils.split(key, ":"); + String[] primaryKv = key.split(":"); kv.put(primaryKv[1], primaryKv[2]); String pattern = key + "*"; Set realKeys = ((Jedis) jedis).keys(pattern); for (String realKey : realKeys){ - kv.put(StringUtils.split(realKey, ":")[3], jedis.get(realKey)); + kv.put(realKey.split(":")[3], jedis.get(realKey)); } tmpCache.put(key, kv); } @@ -199,19 +198,19 @@ private void loadData(Map> tmpCache) throws SQLExcep Set keys = keys((JedisCluster) jedis, perKey); List newPerKeys = new LinkedList<>(); for (String key : keys){ - String[] splitKey = StringUtils.split(key, ":"); + String[] splitKey = key.split(":"); String newKey = splitKey[0] + ":" + splitKey[1] + ":" + splitKey[2]; newPerKeys.add(newKey); } List list = newPerKeys.stream().distinct().collect(Collectors.toList()); for(String key : list){ Map kv = Maps.newHashMap(); - String[] primaryKv = StringUtils.split(key, ":"); + String[] primaryKv = key.split(":"); kv.put(primaryKv[1], primaryKv[2]); String pattern = key + "*"; Set realKeys = keys((JedisCluster) jedis, pattern); for (String realKey : realKeys){ - kv.put(StringUtils.split(key, ":")[3], jedis.get(realKey)); + kv.put(realKey.split(":")[3], jedis.get(realKey)); } tmpCache.put(key, kv); } @@ -246,15 +245,15 @@ private JedisCommands getJedis(RedisSideTableInfo tableInfo) { timeout = 1000; } - String[] nodes = StringUtils.split(url, ","); - String[] firstIpPort = StringUtils.split(nodes[0], ":"); + String[] nodes = url.split(","); + String[] firstIpPort = nodes[0].split(":"); String firstIp = firstIpPort[0]; String firstPort = firstIpPort[1]; Set addresses = new HashSet<>(); Set ipPorts = new HashSet<>(); for (String ipPort : nodes) { ipPorts.add(ipPort); - String[] ipPortPair = StringUtils.split(ipPort, ":"); + String[] ipPortPair = ipPort.split(":"); addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.valueOf(ipPortPair[1].trim()))); } if (timeout == 0){ diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 41f504bf0..1688b2d2c 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -43,7 +43,6 @@ import io.lettuce.core.api.async.RedisStringAsyncCommands; import io.lettuce.core.cluster.RedisClusterClient; import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; -import org.apache.commons.lang3.StringUtils; import java.util.Collections; import java.util.List; @@ -173,7 +172,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce public void accept(List> keyValues) { if (keyValues.size() != 0) { for (int i = 0; i < keyValues.size(); i++) { - String[] splitKeys = StringUtils.split(keyValues.get(i).getKey(), ":"); + String[] splitKeys = keyValues.get(i).getKey().split(":"); keyValue.put(splitKeys[1], splitKeys[2]); keyValue.put(splitKeys[3], keyValues.get(i).getValue()); } diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index 0bd053473..9460c4a76 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -24,7 +24,6 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,15 +107,15 @@ private GenericObjectPoolConfig setPoolConfig(String maxTotal, String maxIdle, S private void establishConnection() { poolConfig = setPoolConfig(maxTotal, maxIdle, minIdle); - String[] nodes = StringUtils.split(url, ","); - String[] firstIpPort = StringUtils.split(nodes[0], ":"); + String[] nodes = url.split(","); + String[] firstIpPort = nodes[0].split(":"); String firstIp = firstIpPort[0]; String firstPort = firstIpPort[1]; Set addresses = new HashSet<>(); Set ipPorts = new HashSet<>(); for (String ipPort : nodes) { ipPorts.add(ipPort); - String[] ipPortPair = StringUtils.split(ipPort, ":"); + String[] ipPortPair = ipPort.split(":"); addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.valueOf(ipPortPair[1].trim()))); } if (timeout == 0){ diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 0c9b0a64a..f01709874 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -50,7 +50,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map primaryKeysList = Lists.newArrayList(); if (!StringUtils.isEmpty(primaryKeysStr)) { - primaryKeysList = Arrays.asList(StringUtils.split(primaryKeysStr, ",")); + primaryKeysList = Arrays.asList(primaryKeysStr.split(",")); } redisTableInfo.setPrimaryKeys(primaryKeysList); From 1e5a4bf5bdc7dfcb6ee5f47977bed4cc32c21ddf Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Wed, 26 Feb 2020 23:04:34 +0800 Subject: [PATCH 062/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9elasticsearch=206.8.6?= =?UTF-8?q?=E7=9A=84=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/elasticsearch6Side | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/elasticsearch6Side b/docs/elasticsearch6Side index 36b229867..234143cbd 100644 --- a/docs/elasticsearch6Side +++ b/docs/elasticsearch6Side @@ -24,13 +24,13 @@ ``` # 2.支持版本 - mysql-5.6.35 + elasticsearch 6.8.6 ## 3.表结构定义 |参数名称|含义| |----|---| - | tableName | mysql表名称| + | tableName | elasticsearch表名称| | colName | 列名称| | colType | 列类型 [colType支持的类型](colType.md)| | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| From d7ed2add83e4d5e7d2f6556383c299071b2f813d Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Wed, 26 Feb 2020 23:11:20 +0800 Subject: [PATCH 063/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9elasticsearch=206.8.6?= =?UTF-8?q?=E7=9A=84=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/elasticsearch6Side | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/elasticsearch6Side b/docs/elasticsearch6Side index 36b229867..234143cbd 100644 --- a/docs/elasticsearch6Side +++ b/docs/elasticsearch6Side @@ -24,13 +24,13 @@ ``` # 2.支持版本 - mysql-5.6.35 + elasticsearch 6.8.6 ## 3.表结构定义 |参数名称|含义| |----|---| - | tableName | mysql表名称| + | tableName | elasticsearch表名称| | colName | 列名称| | colType | 列类型 [colType支持的类型](colType.md)| | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| From 19562c3074d70fdd14d748e455744fa05c8b92b6 Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Thu, 27 Feb 2020 09:55:39 +0800 Subject: [PATCH 064/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0sendOutputRow?= =?UTF-8?q?=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/AllReqRow.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java index a956137b0..11d941cf5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java @@ -23,8 +23,11 @@ import org.apache.flink.api.common.functions.RichFlatMapFunction; import org.apache.flink.configuration.Configuration; import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; import com.dtstack.flink.sql.factory.DTThreadFactory; +import org.apache.calcite.sql.JoinType; import java.sql.SQLException; import java.util.concurrent.Executors; @@ -65,4 +68,13 @@ public void open(Configuration parameters) throws Exception { es.scheduleAtFixedRate(() -> reloadCache(), sideTableInfo.getCacheTimeout(), sideTableInfo.getCacheTimeout(), TimeUnit.MILLISECONDS); } + protected void sendOutputRow(CRow value, Object sideInput, Collector out){ + if(sideInput == null && sideInfo.getJoinType() != JoinType.LEFT){ + return; + } + + Row row = fillData(value.row(), sideInput); + out.collect(new CRow(row, value.change())); + } + } From 9a57115f69070f7941ce2a5d44bec728cf6bc671 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 27 Feb 2020 14:39:45 +0800 Subject: [PATCH 065/523] rdb all code opt --- .../com/dtstack/flink/sql/side/AllReqRow.java | 10 +- .../flink/sql/side/rdb/all/RdbAllReqRow.java | 187 +++++++++--------- 2 files changed, 103 insertions(+), 94 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java index b56c5d7f7..a8ffaa51d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java @@ -67,8 +67,8 @@ public void open(Configuration parameters) throws Exception { es.scheduleAtFixedRate(() -> reloadCache(), sideTableInfo.getCacheTimeout(), sideTableInfo.getCacheTimeout(), TimeUnit.MILLISECONDS); } - protected void sendOutputRow(CRow value, Object sideInput, Collector out){ - if(sideInput == null && sideInfo.getJoinType() != JoinType.LEFT){ + protected void sendOutputRow(CRow value, Object sideInput, Collector out) { + if (sideInput == null && sideInfo.getJoinType() != JoinType.LEFT) { return; } @@ -76,4 +76,10 @@ protected void sendOutputRow(CRow value, Object sideInput, Collector out){ out.collect(new CRow(row, value.change())); } + @Override + public void close() throws Exception { + if (null != es && !es.isShutdown()) { + es.shutdown(); + } + } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java index 7fdeeeca9..60e3d817f 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.rdb.all; +import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; @@ -40,10 +41,12 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; +import java.util.ArrayList; import java.util.Calendar; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; /** * side operator with cache for all(period reload) @@ -61,39 +64,14 @@ public abstract class RdbAllReqRow extends AllReqRow { private static final int CONN_RETRY_NUM = 3; + private static final int DEFAULT_FETCH_SIZE = 1000; + private AtomicReference>>> cacheRef = new AtomicReference<>(); public RdbAllReqRow(SideInfo sideInfo) { super(sideInfo); } - @Override - public Row fillData(Row input, Object sideInput) { - Map cacheInfo = (Map) sideInput; - Row row = new Row(sideInfo.getOutFieldInfoList().size()); - for (Map.Entry entry : sideInfo.getInFieldIndex().entrySet()) { - Object obj = input.getField(entry.getValue()); - boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); - - //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. - if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); - } - - row.setField(entry.getKey(), obj); - } - - for (Map.Entry entry : sideInfo.getSideFieldNameIndex().entrySet()) { - if (cacheInfo == null) { - row.setField(entry.getKey(), null); - } else { - row.setField(entry.getKey(), cacheInfo.get(entry.getValue())); - } - } - - return row; - } - @Override protected void initCache() throws SQLException { Map>> newCache = Maps.newConcurrentMap(); @@ -105,70 +83,78 @@ protected void initCache() throws SQLException { protected void reloadCache() { //reload cacheRef and replace to old cacheRef Map>> newCache = Maps.newConcurrentMap(); + cacheRef.set(newCache); try { loadData(newCache); } catch (SQLException e) { - LOG.error("", e); + throw new RuntimeException(e); } - - cacheRef.set(newCache); LOG.info("----- rdb all cacheRef reload end:{}", Calendar.getInstance()); } - @Override public void flatMap(CRow value, Collector out) throws Exception { - List inputParams = Lists.newArrayList(); - for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = value.row().getField(conValIndex); - if (equalObj == null) { - if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value.row(), null); - out.collect(new CRow(row, value.change())); - } - return; - } - inputParams.add(equalObj); + List equalValIndex = sideInfo.getEqualValIndex(); + ArrayList inputParams = equalValIndex.stream() + .map(value.row()::getField) + .filter(object -> null != object) + .collect(Collectors.toCollection(ArrayList::new)); + + if (inputParams.size() != equalValIndex.size() && sideInfo.getJoinType() == JoinType.LEFT) { + out.collect(new CRow(fillData(value.row(), null), value.change())); + return; } - String key = buildKey(inputParams); - List> cacheList = cacheRef.get().get(key); - if (CollectionUtils.isEmpty(cacheList)) { - if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value.row(), null); - out.collect(new CRow(row, value.change())); - } else { - return; - } + String cacheKey = inputParams.stream() + .map(Object::toString) + .collect(Collectors.joining("_")); - return; + List> cacheList = cacheRef.get().get(cacheKey); + if (CollectionUtils.isEmpty(cacheList) && sideInfo.getJoinType() == JoinType.LEFT) { + out.collect(new CRow(fillData(value.row(), null), value.change())); } - for (Map one : cacheList) { - out.collect(new CRow(fillData(value.row(), one), value.change())); - } + cacheList.stream().forEach(one -> out.collect(new CRow(fillData(value.row(), one), value.change()))); } - private String buildKey(List equalValList) { - StringBuilder sb = new StringBuilder(""); - for (Object equalVal : equalValList) { - sb.append(equalVal).append("_"); + @Override + public Row fillData(Row input, Object sideInput) { + Map cacheInfo = (Map) sideInput; + Row row = new Row(sideInfo.getOutFieldInfoList().size()); + + for (Map.Entry entry : sideInfo.getInFieldIndex().entrySet()) { + // origin value + Object obj = input.getField(entry.getValue()); + obj = dealTimeAttributeType(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass(), obj); + row.setField(entry.getKey(), obj); } - return sb.toString(); - } + for (Map.Entry entry : sideInfo.getSideFieldNameIndex().entrySet()) { + if (cacheInfo == null) { + row.setField(entry.getKey(), null); + } else { + row.setField(entry.getKey(), cacheInfo.get(entry.getValue())); + } - private String buildKey(Map val, List equalFieldList) { - StringBuilder sb = new StringBuilder(""); - for (String equalField : equalFieldList) { - sb.append(val.get(equalField)).append("_"); } - - return sb.toString(); + return row; } - public abstract Connection getConn(String dbURL, String userName, String password); - + /** + * covert flink time attribute.Type information for indicating event or processing time. + * However, it behaves like a regular SQL timestamp but is serialized as Long. + * + * @param entry + * @param obj + * @return + */ + protected Object dealTimeAttributeType(Class entry, Object obj) { + boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(entry); + if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { + obj = ((Timestamp) obj).getTime(); + } + return obj; + } private void loadData(Map>> tmpCache) throws SQLException { RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); @@ -191,31 +177,11 @@ private void loadData(Map>> tmpCache) throws SQ LOG.error("", e1); } } - - } - - //load data from table - String sql = sideInfo.getSqlCondition(); - Statement statement = connection.createStatement(); - statement.setFetchSize(getFetchSize()); - ResultSet resultSet = statement.executeQuery(sql); - String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); - String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); - while (resultSet.next()) { - Map oneRow = Maps.newHashMap(); - for (String fieldName : sideFieldNames) { - Object object = resultSet.getObject(fieldName.trim()); - int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); - object = SwitchUtil.getTarget(object, fields[fieldIndex]); - oneRow.put(fieldName.trim(), object); - } - - String cacheKey = buildKey(oneRow, sideInfo.getEqualFieldList()); - List> list = tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()); - list.add(oneRow); } + queryAndFillData(tmpCache, connection); } catch (Exception e) { LOG.error("", e); + throw new SQLException(e); } finally { if (connection != null) { connection.close(); @@ -223,8 +189,45 @@ private void loadData(Map>> tmpCache) throws SQ } } + private void queryAndFillData(Map>> tmpCache, Connection connection) throws SQLException { + //load data from table + String sql = sideInfo.getSqlCondition(); + Statement statement = connection.createStatement(); + statement.setFetchSize(getFetchSize()); + ResultSet resultSet = statement.executeQuery(sql); + + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); + String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); + while (resultSet.next()) { + Map oneRow = Maps.newHashMap(); + for (String fieldName : sideFieldNames) { + Object object = resultSet.getObject(fieldName.trim()); + int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); + object = SwitchUtil.getTarget(object, fields[fieldIndex]); + oneRow.put(fieldName.trim(), object); + } + + String cacheKey = sideInfo.getEqualFieldList().stream() + .map(equalField -> oneRow.get(equalField)) + .map(Object::toString) + .collect(Collectors.joining("_")); + + tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()) + .add(oneRow); + } + } + public int getFetchSize() { - return 1000; + return DEFAULT_FETCH_SIZE; } + /** + * get jdbc connection + * @param dbURL + * @param userName + * @param password + * @return + */ + public abstract Connection getConn(String dbURL, String userName, String password); + } From 4f0d3592c92385ddf838e50f57bc883d3b6561e7 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 28 Feb 2020 11:33:37 +0800 Subject: [PATCH 066/523] jdbc connection check --- .../sink/rdb/format/JDBCUpsertOutputFormat.java | 1 + .../sql/sink/rdb/writer/AppendOnlyWriter.java | 5 +++++ .../flink/sql/sink/rdb/writer/JDBCWriter.java | 8 +++++++- .../flink/sql/sink/rdb/writer/UpsertWriter.java | 15 +++++++++++++++ 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index ff7b24979..e1b2cc6f4 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -170,6 +170,7 @@ private void checkConnectionOpen() { if (connection.isClosed()) { LOG.info("db connection reconnect.."); establishConnection(); + jdbcWriter.prepareStatement(connection); } } catch (SQLException e) { LOG.error("check connection open failed..", e); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 59eee0858..ca2bd4c66 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -58,6 +58,11 @@ public AppendOnlyWriter(String insertSQL, int[] fieldTypes, DtRichOutputFormat m @Override public void open(Connection connection) throws SQLException { this.rows = new ArrayList(); + prepareStatement(connection); + } + + @Override + public void prepareStatement(Connection connection) throws SQLException { this.statement = connection.prepareStatement(insertSQL); } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/JDBCWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/JDBCWriter.java index f9ce36846..6e0c20de5 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/JDBCWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/JDBCWriter.java @@ -33,10 +33,16 @@ public interface JDBCWriter extends Serializable { int DIRTYDATA_PRINT_FREQUENTY = 1000; /** - * Open the writer by JDBC Connection. It can create Statement from Connection. + * Open the writer by JDBC Connection. */ void open(Connection connection) throws SQLException; + /** + * Create Statement from Connection. Used where the connection is created + * @throws SQLException + */ + void prepareStatement(Connection connection) throws SQLException; + /** * Add record to writer, the writer may cache the data. */ diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java index c7074de59..ce3a1c680 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java @@ -110,6 +110,11 @@ private UpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String del @Override public void open(Connection connection) throws SQLException { this.keyToRows = new HashMap<>(); + prepareStatement(connection); + } + + @Override + public void prepareStatement(Connection connection) throws SQLException { this.deleteStatement = connection.prepareStatement(deleteSQL); } @@ -224,6 +229,11 @@ private UpsertWriterUsingUpsertStatement( @Override public void open(Connection connection) throws SQLException { super.open(connection); + } + + @Override + public void prepareStatement(Connection connection) throws SQLException { + super.prepareStatement(connection); upsertStatement = connection.prepareStatement(upsertSQL); } @@ -284,6 +294,11 @@ private UpsertWriterUsingInsertUpdateStatement( @Override public void open(Connection connection) throws SQLException { super.open(connection); + } + + @Override + public void prepareStatement(Connection connection) throws SQLException { + super.prepareStatement(connection); existStatement = connection.prepareStatement(existSQL); insertStatement = connection.prepareStatement(insertSQL); updateStatement = connection.prepareStatement(updateSQL); From e0508c1ea7ded11777691cb55b8125149b3d317b Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Fri, 28 Feb 2020 12:32:49 +0800 Subject: [PATCH 067/523] =?UTF-8?q?=E4=BD=BF=E7=94=A8StringUtils.split()?= =?UTF-8?q?=E6=9B=BF=E4=BB=A3String.split();?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../side/cassandra/CassandraAllReqRow.java | 9 +++++---- .../side/cassandra/CassandraAsyncReqRow.java | 7 ++++--- .../sink/cassandra/CassandraOutputFormat.java | 9 +++++---- .../dtstack/flink/sql/side/SideSqlExec.java | 3 ++- .../flink/sql/table/TableInfoParser.java | 3 ++- .../dtstack/flink/sql/util/DtStringUtil.java | 4 ++-- .../sink/elasticsearch/ElasticsearchSink.java | 4 ++-- .../sql/sink/hbase/HbaseOutputFormat.java | 2 +- .../sql/sink/hbase/table/HbaseSinkParser.java | 5 +++-- .../side/impala/table/ImpalaSideParser.java | 5 +++-- .../sink/impala/table/ImpalaTableInfo.java | 3 ++- .../flink/sql/source/kafka/KafkaConsumer.java | 3 ++- .../sql/source/kafka/KafkaConsumer09.java | 3 ++- .../sql/source/kafka/KafkaConsumer010.java | 3 ++- .../sql/source/kafka/KafkaConsumer011.java | 3 ++- .../flink/sql/side/kudu/KuduAllReqRow.java | 7 ++++--- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 5 +++-- .../flink/sql/side/kudu/utils/KuduUtil.java | 3 ++- .../flink/sql/sink/kudu/KuduOutputFormat.java | 11 +++++++++-- .../sql/launcher/perjob/PerJobSubmitter.java | 2 +- .../flink/sql/side/mongo/MongoAllReqRow.java | 6 +++--- .../sql/side/mongo/MongoAsyncReqRow.java | 4 ++-- .../flink/sql/side/mongo/utils/MongoUtil.java | 5 +++-- .../sql/sink/mongo/MongoOutputFormat.java | 4 ++-- .../flink/sql/side/rdb/all/RdbAllReqRow.java | 3 ++- .../sql/side/rdb/all/RdbAllSideInfo.java | 2 +- .../sql/side/rdb/async/RdbAsyncSideInfo.java | 3 ++- .../flink/sql/side/redis/RedisAllReqRow.java | 19 ++++++++++--------- .../sql/side/redis/RedisAsyncReqRow.java | 3 ++- .../sql/sink/redis/RedisOutputFormat.java | 7 ++++--- .../sql/sink/redis/table/RedisSinkParser.java | 3 +-- 31 files changed, 90 insertions(+), 63 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index 4439c2b7a..bc0a0f8cd 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -43,6 +43,7 @@ import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -223,9 +224,9 @@ private Session getConn(CassandraSideTableInfo tableInfo) { //重试策略 RetryPolicy retryPolicy = DowngradingConsistencyRetryPolicy.INSTANCE; - for (String server : address.split(",")) { - cassandraPort = Integer.parseInt(server.split(":")[1]); - serversList.add(InetAddress.getByName(server.split(":")[0])); + for (String server : StringUtils.split(address, ",")) { + cassandraPort = Integer.parseInt(StringUtils.split(server, ":")[1]); + serversList.add(InetAddress.getByName(StringUtils.split(server, ":")[0])); } if (userName == null || userName.isEmpty() || password == null || password.isEmpty()) { @@ -279,7 +280,7 @@ private void loadData(Map>> tmpCache) throws SQ //load data from table String sql = sideInfo.getSqlCondition() + " limit " + FETCH_SIZE; ResultSet resultSet = session.execute(sql); - String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); for (com.datastax.driver.core.Row row : resultSet) { Map oneRow = Maps.newHashMap(); for (String fieldName : sideFieldNames) { diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index 7a209f30e..0173f2d6f 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -51,6 +51,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import io.vertx.core.json.JsonArray; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -134,9 +135,9 @@ private void connCassandraDB(CassandraSideTableInfo tableInfo) { //重试策略 RetryPolicy retryPolicy = DowngradingConsistencyRetryPolicy.INSTANCE; - for (String server : address.split(",")) { - cassandraPort = Integer.parseInt(server.split(":")[1]); - serversList.add(InetAddress.getByName(server.split(":")[0])); + for (String server : StringUtils.split(address, ",")) { + cassandraPort = Integer.parseInt(StringUtils.split(server, ":")[1]); + serversList.add(InetAddress.getByName(StringUtils.split(server, ":")[0])); } if (userName == null || userName.isEmpty() || password == null || password.isEmpty()) { diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java index d8f4eb451..08a2358cb 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java @@ -55,6 +55,7 @@ import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -147,9 +148,9 @@ public void open(int taskNumber, int numTasks) { //重试策略 RetryPolicy retryPolicy = DowngradingConsistencyRetryPolicy.INSTANCE; - for (String server : address.split(",")) { - cassandraPort = Integer.parseInt(server.split(":")[1]); - serversList.add(InetAddress.getByName(server.split(":")[0])); + for (String server : StringUtils.split(address, ",")) { + cassandraPort = Integer.parseInt(StringUtils.split(server, ":")[1]); + serversList.add(InetAddress.getByName(StringUtils.split(server, ":")[0])); } if (userName == null || userName.isEmpty() || password == null || password.isEmpty()) { @@ -218,7 +219,7 @@ private void insertWrite(Row row) { } } catch (Exception e) { if(outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0){ - LOG.error("record insert failed, total dirty num:{}, current record:{}", outDirtyRecords.getCount(), row.toString()); + LOG.error("record insert failed ..", row.toString().substring(0, 100)); LOG.error("", e); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 4e718460e..2d769f5a8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -63,6 +63,7 @@ import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -892,7 +893,7 @@ private TypeInformation projectedTypeInfo(int[] fields, TableSchema schema) private boolean checkFieldsInfo(CreateTmpTableParser.SqlParserResult result, Table table) { List fieldNames = new LinkedList<>(); String fieldsInfo = result.getFieldsInfoStr(); - String[] fields = fieldsInfo.split(","); + String[] fields = StringUtils.split(fieldsInfo, ","); for (int i = 0; i < fields.length; i++) { String[] filed = fields[i].split("\\s"); if (filed.length < 2 || fields.length != table.getSchema().getColumnNames().length){ diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java index ae98d90ae..f4c638381 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java @@ -29,6 +29,7 @@ import com.dtstack.flink.sql.util.MathUtil; import com.google.common.base.Strings; import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; import java.util.Map; import java.util.regex.Matcher; @@ -110,7 +111,7 @@ public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserRe * @return */ private static boolean checkIsSideTable(String tableField){ - String[] fieldInfos = tableField.split(","); + String[] fieldInfos = StringUtils.split(tableField, ","); for(String field : fieldInfos){ Matcher matcher = SIDE_PATTERN.matcher(field.trim()); if(matcher.find()){ diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index 862b0700e..b2486b1ab 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -185,9 +185,9 @@ public static String addJdbcParam(String dbUrl, Map addParams, b Map params = Maps.newHashMap(); if(splits.length > 1){ String existsParamStr = splits[1]; - String[] existsParams = existsParamStr.split("&"); + String[] existsParams = StringUtils.split(existsParamStr, "&"); for(String oneParam : existsParams){ - String[] kv = oneParam.split("="); + String[] kv = StringUtils.split(oneParam, "="); if(kv.length != 2){ throw new RuntimeException("illegal dbUrl:" + dbUrl); } diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 0bfc338e3..491d1af27 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -118,7 +118,7 @@ private RichSinkFunction createEsSinkFunction(){ List transports = new ArrayList<>(); for(String address : esAddressList){ - String[] infoArray = address.split(":"); + String[] infoArray = StringUtils.split(address, ":"); int port = 9300; String host = infoArray[0]; if(infoArray.length > 1){ @@ -169,7 +169,7 @@ public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { esTableInfo = elasticsearchTableInfo; clusterName = elasticsearchTableInfo.getClusterName(); String address = elasticsearchTableInfo.getAddress(); - String[] addr = address.split(","); + String[] addr = StringUtils.split(address, ","); esAddressList = Arrays.asList(addr); index = elasticsearchTableInfo.getIndex(); type = elasticsearchTableInfo.getEsType(); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index fd94e0db2..d41c65723 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -281,7 +281,7 @@ public HbaseOutputFormat finish() { String[] columns = keySet.toArray(new String[keySet.size()]); for (int i = 0; i < columns.length; ++i) { String col = columns[i]; - String[] part = col.split(":"); + String[] part = StringUtils.split(col, ":"); families[i] = part[0]; qualifiers[i] = part[1]; } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index f1ae72b6b..064b13e69 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -26,6 +26,7 @@ import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.MathUtil; +import org.apache.commons.lang3.StringUtils; import java.util.LinkedHashMap; import java.util.List; @@ -68,7 +69,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); + super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java index 2fa0d16b6..78eabde09 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer09.java @@ -31,6 +31,7 @@ import org.apache.flink.util.SerializedValue; import com.dtstack.flink.sql.format.DeserializationMetricWrapper; +import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.Map; @@ -50,7 +51,7 @@ public class KafkaConsumer09 extends FlinkKafkaConsumer09 { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer09(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); + super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java index 74f45581c..b78f47cd5 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer010.java @@ -30,6 +30,7 @@ import org.apache.flink.util.SerializedValue; import com.dtstack.flink.sql.format.DeserializationMetricWrapper; +import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.Map; @@ -49,7 +50,7 @@ public class KafkaConsumer010 extends FlinkKafkaConsumer010 { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer010(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); + super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java index a88fb1f9c..276ce211f 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011.java @@ -30,6 +30,7 @@ import org.apache.flink.util.SerializedValue; import com.dtstack.flink.sql.format.DeserializationMetricWrapper; +import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.Map; @@ -49,7 +50,7 @@ public class KafkaConsumer011 extends FlinkKafkaConsumer011 { private DeserializationMetricWrapper deserializationMetricWrapper; public KafkaConsumer011(String topic, DeserializationMetricWrapper deserializationMetricWrapper, Properties props) { - super(Arrays.asList(topic.split(",")), deserializationMetricWrapper, props); + super(Arrays.asList(StringUtils.split(topic, ",")), deserializationMetricWrapper, props); this.deserializationMetricWrapper = deserializationMetricWrapper; } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 374261d40..63ff31c50 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -18,6 +18,7 @@ import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.KuduClient; @@ -160,7 +161,7 @@ private void loadData(Map>> tmpCache) { } //load data from table assert scanner != null; - String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); while (scanner.hasMoreRows()) { @@ -263,7 +264,7 @@ private KuduScanner buildScanner(KuduScanner.KuduScannerBuilder builder, Schema Long limitNum = tableInfo.getLimitNum(); Boolean isFaultTolerant = tableInfo.getFaultTolerant(); //查询需要的字段 - String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); //主键过滤条件 主键最小值 String lowerBoundPrimaryKey = tableInfo.getLowerBoundPrimaryKey(); //主键过滤条件 主键最大值 @@ -317,7 +318,7 @@ private KuduScanner buildScanner(KuduScanner.KuduScannerBuilder builder, Schema } private String[] splitString(String data) { - return data.split(","); + return StringUtils.split(data, ","); } @Override diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index 588bd00ca..e1705d796 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -23,6 +23,7 @@ import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; import io.vertx.core.json.JsonArray; +import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.AsyncKuduClient; @@ -113,7 +114,7 @@ private void connKuDu() throws KuduException { Long limitNum = kuduSideTableInfo.getLimitNum(); Boolean isFaultTolerant = kuduSideTableInfo.getFaultTolerant(); //查询需要的字段 - String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); if (null == limitNum || limitNum <= 0) { scannerBuilder.limit(FETCH_SIZE); @@ -280,7 +281,7 @@ public GetListRowCB() { public Deferred> call(RowResultIterator results) throws Exception { for (RowResult result : results) { Map oneRow = Maps.newHashMap(); - for (String sideFieldName1 : sideInfo.getSideSelectFields().split(",")) { + for (String sideFieldName1 : StringUtils.split(sideInfo.getSideSelectFields(), ",")) { String sideFieldName = sideFieldName1.trim(); ColumnSchema columnSchema = table.getSchema().getColumn(sideFieldName); if (null != columnSchema) { diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java index c5cd8cdb2..b750e2d4a 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java @@ -20,6 +20,7 @@ import com.dtstack.flink.sql.side.PredicateInfo; import org.apache.commons.lang.math.NumberUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.Type; @@ -163,7 +164,7 @@ public static KuduPredicate buildKuduPredicate(Schema schema, PredicateInfo info case "IN": case "NOT_IN": case "BETWEEN": - value = Arrays.asList(info.getCondition().split(",")).stream() + value = Arrays.asList(StringUtils.split(info.getCondition(), ",")).stream() .map(val -> KuduUtil.getValue(val.trim(), column.getType())).collect(Collectors.toList()); break; case "IS_NOT_NULL": diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index 60e3e9927..1ef22ffd9 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -18,12 +18,19 @@ package com.dtstack.flink.sql.sink.kudu; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; -import org.apache.kudu.client.*; + +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import org.apache.kudu.client.AsyncKuduClient; +import org.apache.kudu.client.AsyncKuduSession; +import org.apache.kudu.client.KuduClient; +import org.apache.kudu.client.KuduException; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.Operation; +import org.apache.kudu.client.PartialRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index 918451f7a..29cc4890a 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -83,7 +83,7 @@ private static List getJarPaths(String addjarPath) { if (addjarPath.length() > 2) { addjarPath = addjarPath.substring(1,addjarPath.length()-1).replace("\"",""); } - List paths = Arrays.asList(addjarPath.split(",")); + List paths = Arrays.asList(StringUtils.split(addjarPath, ",")); return paths; } diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index 1a0565268..f90cfd973 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -185,13 +185,13 @@ private MongoCollection getConn(String address, String userName, String password MongoCollection dbCollection; try { MongoCredential credential; - String[] servers = address.split(","); + String[] servers = StringUtils.split(address, ","); String host; Integer port; String[] hostAndPort; List lists = new ArrayList<>(); for (String server : servers) { - hostAndPort = server.split(":"); + hostAndPort = StringUtils.split(server, ":"); host = hostAndPort[0]; port = Integer.parseInt(hostAndPort[1]); lists.add(new ServerAddress(host, port)); @@ -237,7 +237,7 @@ private void loadData(Map>> tmpCache) throws SQ } //load data from table - String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); BasicDBObject basicDBObject = new BasicDBObject(); for (String selectField : sideFieldNames) { basicDBObject.append(selectField, 1); diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index ac6b4120c..42749e935 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -92,13 +92,13 @@ public void open(Configuration parameters) throws Exception { public void connMongoDB() throws Exception { MongoCredential mongoCredential; - String[] servers = MongoSideTableInfo.getAddress().split(","); + String[] servers = StringUtils.split(MongoSideTableInfo.getAddress(), ","); String host; Integer port; String[] hostAndPort; List lists = new ArrayList<>(); for (String server : servers) { - hostAndPort = server.split(":"); + hostAndPort = StringUtils.split(server, ":"); host = hostAndPort[0]; port = Integer.parseInt(hostAndPort[1]); lists.add(new ServerAddress(host, port)); diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java index 0b345d947..2f1a2092d 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java @@ -20,6 +20,7 @@ import com.dtstack.flink.sql.side.PredicateInfo; import com.mongodb.BasicDBObject; +import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.stream.Collectors; @@ -45,11 +46,11 @@ public static BasicDBObject buildFilterObject(PredicateInfo info) { case "<>": return new BasicDBObject("$ne", info.getCondition()); case "IN": - Object[] values = Arrays.stream(info.getCondition().split(",")).map(String::trim) + Object[] values = Arrays.stream(StringUtils.split(info.getCondition(), ",")).map(String::trim) .collect(Collectors.toList()).toArray(); return new BasicDBObject("$in", values); case "NOT IN": - return new BasicDBObject("$nin", info.getCondition().split(",")); + return new BasicDBObject("$nin", StringUtils.split(info.getCondition(), ",")); case "IS NOT NULL": return new BasicDBObject("$exists", true); case "IS NULL": diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index d5a9b4a82..337fbad59 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -127,13 +127,13 @@ public void close() { private void establishConnection() { try { MongoCredential credential; - String[] servers = address.split(","); + String[] servers = StringUtils.split(address, ","); String host; Integer port; String[] hostAndPort; List lists = new ArrayList<>(); for (String server : servers) { - hostAndPort = server.split(":"); + hostAndPort = StringUtils.split(server, ":"); host = hostAndPort[0]; port = Integer.parseInt(hostAndPort[1]); lists.add(new ServerAddress(host, port)); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java index c746d1185..7fdeeeca9 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java @@ -31,6 +31,7 @@ import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -198,7 +199,7 @@ private void loadData(Map>> tmpCache) throws SQ Statement statement = connection.createStatement(); statement.setFetchSize(getFetchSize()); ResultSet resultSet = statement.executeQuery(sql); - String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); + String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); while (resultSet.next()) { Map oneRow = Maps.newHashMap(); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 313594050..3b7444b77 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -56,7 +56,7 @@ public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List> tmpCache) throws SQLExcep Set keys = ((Jedis) jedis).keys(perKey); List newPerKeys = new LinkedList<>(); for (String key : keys){ - String[] splitKey = key.split(":"); + String[] splitKey = StringUtils.split(key, ":"); String newKey = splitKey[0] + ":" + splitKey[1] + ":" + splitKey[2]; newPerKeys.add(newKey); } List list = newPerKeys.stream().distinct().collect(Collectors.toList()); for(String key : list){ Map kv = Maps.newHashMap(); - String[] primaryKv = key.split(":"); + String[] primaryKv = StringUtils.split(key, ":"); kv.put(primaryKv[1], primaryKv[2]); String pattern = key + "*"; Set realKeys = ((Jedis) jedis).keys(pattern); for (String realKey : realKeys){ - kv.put(realKey.split(":")[3], jedis.get(realKey)); + kv.put(StringUtils.split(realKey, ":")[3], jedis.get(realKey)); } tmpCache.put(key, kv); } @@ -198,19 +199,19 @@ private void loadData(Map> tmpCache) throws SQLExcep Set keys = keys((JedisCluster) jedis, perKey); List newPerKeys = new LinkedList<>(); for (String key : keys){ - String[] splitKey = key.split(":"); + String[] splitKey = StringUtils.split(key, ":"); String newKey = splitKey[0] + ":" + splitKey[1] + ":" + splitKey[2]; newPerKeys.add(newKey); } List list = newPerKeys.stream().distinct().collect(Collectors.toList()); for(String key : list){ Map kv = Maps.newHashMap(); - String[] primaryKv = key.split(":"); + String[] primaryKv = StringUtils.split(key, ":"); kv.put(primaryKv[1], primaryKv[2]); String pattern = key + "*"; Set realKeys = keys((JedisCluster) jedis, pattern); for (String realKey : realKeys){ - kv.put(realKey.split(":")[3], jedis.get(realKey)); + kv.put(StringUtils.split(key, ":")[3], jedis.get(realKey)); } tmpCache.put(key, kv); } @@ -245,15 +246,15 @@ private JedisCommands getJedis(RedisSideTableInfo tableInfo) { timeout = 1000; } - String[] nodes = url.split(","); - String[] firstIpPort = nodes[0].split(":"); + String[] nodes = StringUtils.split(url, ","); + String[] firstIpPort = StringUtils.split(nodes[0], ":"); String firstIp = firstIpPort[0]; String firstPort = firstIpPort[1]; Set addresses = new HashSet<>(); Set ipPorts = new HashSet<>(); for (String ipPort : nodes) { ipPorts.add(ipPort); - String[] ipPortPair = ipPort.split(":"); + String[] ipPortPair = StringUtils.split(ipPort, ":"); addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.valueOf(ipPortPair[1].trim()))); } if (timeout == 0){ diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 1688b2d2c..41f504bf0 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -43,6 +43,7 @@ import io.lettuce.core.api.async.RedisStringAsyncCommands; import io.lettuce.core.cluster.RedisClusterClient; import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; +import org.apache.commons.lang3.StringUtils; import java.util.Collections; import java.util.List; @@ -172,7 +173,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce public void accept(List> keyValues) { if (keyValues.size() != 0) { for (int i = 0; i < keyValues.size(); i++) { - String[] splitKeys = keyValues.get(i).getKey().split(":"); + String[] splitKeys = StringUtils.split(keyValues.get(i).getKey(), ":"); keyValue.put(splitKeys[1], splitKeys[2]); keyValue.put(splitKeys[3], keyValues.get(i).getValue()); } diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index 9460c4a76..0bd053473 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -24,6 +24,7 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,15 +108,15 @@ private GenericObjectPoolConfig setPoolConfig(String maxTotal, String maxIdle, S private void establishConnection() { poolConfig = setPoolConfig(maxTotal, maxIdle, minIdle); - String[] nodes = url.split(","); - String[] firstIpPort = nodes[0].split(":"); + String[] nodes = StringUtils.split(url, ","); + String[] firstIpPort = StringUtils.split(nodes[0], ":"); String firstIp = firstIpPort[0]; String firstPort = firstIpPort[1]; Set addresses = new HashSet<>(); Set ipPorts = new HashSet<>(); for (String ipPort : nodes) { ipPorts.add(ipPort); - String[] ipPortPair = ipPort.split(":"); + String[] ipPortPair = StringUtils.split(ipPort, ":"); addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.valueOf(ipPortPair[1].trim()))); } if (timeout == 0){ diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 49861bb9a..0c9b0a64a 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -24,7 +24,6 @@ import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -51,7 +50,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map primaryKeysList = Lists.newArrayList(); if (!StringUtils.isEmpty(primaryKeysStr)) { - primaryKeysList = Arrays.asList(primaryKeysStr.split(",")); + primaryKeysList = Arrays.asList(StringUtils.split(primaryKeysStr, ",")); } redisTableInfo.setPrimaryKeys(primaryKeysList); From fdc614bc5f77d8b3d479c094231be8486bfda9fc Mon Sep 17 00:00:00 2001 From: daiyang <1193147603@qq.com> Date: Fri, 28 Feb 2020 12:55:55 +0800 Subject: [PATCH 068/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=8E1.8=5Ftest?= =?UTF-8?q?=5F3.1.0x=E5=88=86=E6=94=AF=E7=9A=84=E4=B8=8D=E5=90=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java | 2 +- .../java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java index 08a2358cb..14c13fc8c 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java @@ -219,7 +219,7 @@ private void insertWrite(Row row) { } } catch (Exception e) { if(outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0){ - LOG.error("record insert failed ..", row.toString().substring(0, 100)); + LOG.error("record insert failed, total dirty num:{}, current record:{}", outDirtyRecords.getCount(), row.toString()); LOG.error("", e); } diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index 1ef22ffd9..99d19cdef 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.kudu; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; From 698094e4511b680c1284c2477407502102d6e51b Mon Sep 17 00:00:00 2001 From: dapeng Date: Sat, 29 Feb 2020 21:37:19 +0800 Subject: [PATCH 069/523] =?UTF-8?q?async=E7=9A=84=E8=B6=85=E6=97=B6?= =?UTF-8?q?=E6=AC=A1=E6=95=B0=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/AsyncReqRow.java | 8 +++++--- .../com/dtstack/flink/sql/side/SideTableInfo.java | 13 +++++++++++++ .../dtstack/flink/sql/table/AbsSideTableParser.java | 6 ++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index aecea5458..2759e29c0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -125,11 +125,13 @@ public void timeout(Row input, ResultFuture resultFuture) throws Exception //TODO 需要添加数据指标 if(timeOutNum % TIMEOUT_LOG_FLUSH_NUM == 0){ - LOG.info("Async function call has timed out. input:" + input.toString()); + LOG.info("Async function call has timed out. input:{}, timeOutNum:{}",input.toString(), timeOutNum); + } + timeOutNum ++; + if(timeOutNum > sideInfo.getSideTableInfo().getAsyncTimeoutNumLimit()){ + resultFuture.complete(null); } - timeOutNum++; - resultFuture.complete(null); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java index 8562d9859..c82489c94 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java @@ -53,6 +53,8 @@ public abstract class SideTableInfo extends TableInfo implements Serializable { public static final String ASYNC_TIMEOUT_KEY = "asyncTimeout"; + public static final String ASYNC_TIMEOUT_NUM_KEY = "asyncTimeoutNum"; + private String cacheType = "none";//None or LRU or ALL private int cacheSize = 10000; @@ -63,6 +65,8 @@ public abstract class SideTableInfo extends TableInfo implements Serializable { private int asyncTimeout=10000; + private int asyncTimeoutNumLimit = 0; + private boolean partitionedJoin = false; private String cacheMode="ordered"; @@ -143,4 +147,13 @@ public void setPredicateInfoes(List predicateInfoes) { public List getPredicateInfoes() { return predicateInfoes; } + + public int getAsyncTimeoutNumLimit() { + return asyncTimeoutNumLimit; + } + + public void setAsyncTimeoutNumLimit(int asyncTimeoutNumLimit) { + this.asyncTimeoutNumLimit = asyncTimeoutNumLimit; + } + } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java index 520ca2138..4e584cbe8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java @@ -108,6 +108,12 @@ protected void parseCacheProp(SideTableInfo sideTableInfo, Map p } sideTableInfo.setAsyncTimeout(asyncTimeout); } + if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())){ + Integer asyncTimeoutNum = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())); + if (asyncTimeoutNum > 0){ + sideTableInfo.setAsyncTimeout(asyncTimeoutNum); + } + } } } } From 48dd0c0fc268265e5d556d7f8ba0880bb570e87b Mon Sep 17 00:00:00 2001 From: dapeng Date: Sun, 1 Mar 2020 14:07:32 +0800 Subject: [PATCH 070/523] fix completeExceptionally desc --- core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index 2759e29c0..2d14867a2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -129,9 +129,8 @@ public void timeout(Row input, ResultFuture resultFuture) throws Exception } timeOutNum ++; if(timeOutNum > sideInfo.getSideTableInfo().getAsyncTimeoutNumLimit()){ - resultFuture.complete(null); + resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); } - } From 87c26d71fecc1fd5aeb8d1b8c66d3077c8bcb45c Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Sun, 1 Mar 2020 14:44:42 +0800 Subject: [PATCH 071/523] =?UTF-8?q?=E5=90=88=E5=B9=B6=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/AsyncReqRow.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index 066708d6c..cdd4a1610 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -122,13 +122,14 @@ protected void dealCacheData(String key, CacheObj missKeyObj) { @Override public void timeout(CRow input, ResultFuture resultFuture) throws Exception { - StreamRecordQueueEntry future = (StreamRecordQueueEntry)resultFuture; - try { - if (null == future.get()) { - resultFuture.completeExceptionally(new TimeoutException("Async function call has timed out.")); - } - } catch (Exception e) { - resultFuture.completeExceptionally(new Exception(e)); + + if(timeOutNum % TIMEOUT_LOG_FLUSH_NUM == 0){ + LOG.info("Async function call has timed out. input:{}, timeOutNum:{}",input.toString(), timeOutNum); + } + + timeOutNum ++; + if(timeOutNum > sideInfo.getSideTableInfo().getAsyncTimeoutNumLimit()){ + resultFuture.complete(null); } } From 28294ff5fad1f061f43f12d8ce37e9954e26da4e Mon Sep 17 00:00:00 2001 From: dapeng Date: Sun, 1 Mar 2020 20:14:51 +0800 Subject: [PATCH 072/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=80=BC=E5=92=8C=E5=88=A4=E6=96=AD=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java | 2 ++ .../src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index 2d14867a2..635ff5dd8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -130,6 +130,8 @@ public void timeout(Row input, ResultFuture resultFuture) throws Exception timeOutNum ++; if(timeOutNum > sideInfo.getSideTableInfo().getAsyncTimeoutNumLimit()){ resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); + } else { + resultFuture.complete(null); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java index c82489c94..8c78578fb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java @@ -65,7 +65,7 @@ public abstract class SideTableInfo extends TableInfo implements Serializable { private int asyncTimeout=10000; - private int asyncTimeoutNumLimit = 0; + private int asyncTimeoutNumLimit = Integer.MAX_VALUE; private boolean partitionedJoin = false; From 1248591145ceffe9f59c4bd6908677283798d402 Mon Sep 17 00:00:00 2001 From: dapeng Date: Sun, 1 Mar 2020 14:07:32 +0800 Subject: [PATCH 073/523] fix completeExceptionally desc --- core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index cdd4a1610..cd811e48f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -129,7 +129,7 @@ public void timeout(CRow input, ResultFuture resultFuture) throws Exceptio timeOutNum ++; if(timeOutNum > sideInfo.getSideTableInfo().getAsyncTimeoutNumLimit()){ - resultFuture.complete(null); + resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); } } From d445fcac42fbc1b03aae9d1562ea3dd64b95fe21 Mon Sep 17 00:00:00 2001 From: dapeng Date: Sun, 1 Mar 2020 20:47:05 +0800 Subject: [PATCH 074/523] =?UTF-8?q?=E5=8F=82=E6=95=B0=E8=B5=8B=E5=80=BC?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java | 2 +- .../java/com/dtstack/flink/sql/table/AbsSideTableParser.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java index 8c78578fb..c82489c94 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java @@ -65,7 +65,7 @@ public abstract class SideTableInfo extends TableInfo implements Serializable { private int asyncTimeout=10000; - private int asyncTimeoutNumLimit = Integer.MAX_VALUE; + private int asyncTimeoutNumLimit = 0; private boolean partitionedJoin = false; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java index 4e584cbe8..bf0b8a7a4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java @@ -111,7 +111,7 @@ protected void parseCacheProp(SideTableInfo sideTableInfo, Map p if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())){ Integer asyncTimeoutNum = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())); if (asyncTimeoutNum > 0){ - sideTableInfo.setAsyncTimeout(asyncTimeoutNum); + sideTableInfo.setAsyncTimeoutNumLimit(asyncTimeoutNum); } } } From 48efb8b55fd1815de8573b21ac3c685efa8be7a4 Mon Sep 17 00:00:00 2001 From: dapeng Date: Sun, 1 Mar 2020 20:55:27 +0800 Subject: [PATCH 075/523] =?UTF-8?q?=E5=8F=82=E6=95=B0=E8=B5=8B=E5=80=BC?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java | 2 +- .../java/com/dtstack/flink/sql/table/AbsSideTableParser.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java index 8c78578fb..c82489c94 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java @@ -65,7 +65,7 @@ public abstract class SideTableInfo extends TableInfo implements Serializable { private int asyncTimeout=10000; - private int asyncTimeoutNumLimit = Integer.MAX_VALUE; + private int asyncTimeoutNumLimit = 0; private boolean partitionedJoin = false; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java index 4e584cbe8..bf0b8a7a4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java @@ -111,7 +111,7 @@ protected void parseCacheProp(SideTableInfo sideTableInfo, Map p if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())){ Integer asyncTimeoutNum = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())); if (asyncTimeoutNum > 0){ - sideTableInfo.setAsyncTimeout(asyncTimeoutNum); + sideTableInfo.setAsyncTimeoutNumLimit(asyncTimeoutNum); } } } From 565834eabc4ec92d6f2efd79741dc66704117dfd Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 2 Mar 2020 10:22:23 +0800 Subject: [PATCH 076/523] =?UTF-8?q?=E9=BB=98=E8=AE=A4=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=80=BC=E4=B8=8D=E9=99=90=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java index c82489c94..8c78578fb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java @@ -65,7 +65,7 @@ public abstract class SideTableInfo extends TableInfo implements Serializable { private int asyncTimeout=10000; - private int asyncTimeoutNumLimit = 0; + private int asyncTimeoutNumLimit = Integer.MAX_VALUE; private boolean partitionedJoin = false; From c47553672b5f5158315b62c96fca4947dbc34856 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 2 Mar 2020 10:22:24 +0800 Subject: [PATCH 077/523] remove log level and modify objectmapper class --- core/pom.xml | 5 ----- .../src/main/java/com/dtstack/flink/sql/Main.java | 9 --------- .../com/dtstack/flink/sql/exec/ApiResult.java | 2 +- .../flink/sql/exec/ExecuteProcessHelper.java | 11 ----------- .../com/dtstack/flink/sql/exec/ParamsInfo.java | 15 ++------------- 5 files changed, 3 insertions(+), 39 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 4b5a3e67b..677f9d5ee 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -121,11 +121,6 @@ junit 4.12 - - ch.qos.logback - logback-classic - 1.1.7 - diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index 4ff296f1f..5d7528869 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -27,9 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.LoggerContext; - /** * Date: 2018/6/26 * Company: www.dtstack.com @@ -45,10 +42,4 @@ public static void main(String[] args) throws Exception { env.execute(paramsInfo.getName()); LOG.info("program {} execution success", paramsInfo.getName()); } - private static void setLogLevel(String level){ - LoggerContext loggerContext= (LoggerContext) LoggerFactory.getILoggerFactory(); - //设置全局日志级别 - ch.qos.logback.classic.Logger logger = loggerContext.getLogger("root"); - logger.setLevel(Level.toLevel(level, Level.INFO)); - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java b/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java index d6ca857c3..d67a39a40 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java @@ -1,6 +1,6 @@ package com.dtstack.flink.sql.exec; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 2c6d74df0..d2c2926ef 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -18,8 +18,6 @@ package com.dtstack.flink.sql.exec; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.LoggerContext; import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.config.CalciteConfig; import com.dtstack.flink.sql.enums.ClusterMode; @@ -128,7 +126,6 @@ public static ParamsInfo parseParams(String[] args) throws Exception { .setDeployMode(deployMode) .setConfProp(confProperties) .setJarUrlList(jarURList) - .setLogLevel(logLevel) .build(); } @@ -154,8 +151,6 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); StreamQueryConfig streamQueryConfig = StreamEnvConfigManager.getStreamQueryConfig(tableEnv, paramsInfo.getConfProp()); - setLogLevel(paramsInfo.getLogLevel()); - SqlParser.setLocalSqlPluginRoot(paramsInfo.getLocalSqlPluginPath()); SqlTree sqlTree = SqlParser.parseSql(paramsInfo.getSql()); @@ -348,11 +343,5 @@ public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperti return env; } - private static void setLogLevel(String level){ - LoggerContext loggerContext= (LoggerContext) LoggerFactory.getILoggerFactory(); - //设置全局日志级别 - ch.qos.logback.classic.Logger logger = loggerContext.getLogger("root"); - logger.setLevel(Level.toLevel(level, Level.INFO)); - } } \ No newline at end of file diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java index 59249827c..27cc7702d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java @@ -38,11 +38,10 @@ public class ParamsInfo { private String remoteSqlPluginPath; private String pluginLoadMode; private String deployMode; - private String logLevel; private Properties confProp; public ParamsInfo(String sql, String name, List jarUrlList, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, String deployMode, String logLevel, Properties confProp) { + String remoteSqlPluginPath, String pluginLoadMode, String deployMode, Properties confProp) { this.sql = sql; this.name = name; this.jarUrlList = jarUrlList; @@ -50,7 +49,6 @@ public ParamsInfo(String sql, String name, List jarUrlList, String localSql this.remoteSqlPluginPath = remoteSqlPluginPath; this.pluginLoadMode = pluginLoadMode; this.deployMode = deployMode; - this.logLevel = logLevel; this.confProp = confProp; } @@ -86,10 +84,6 @@ public Properties getConfProp() { return confProp; } - public String getLogLevel() { - return logLevel; - } - @Override public String toString() { return "ParamsInfo{" + @@ -100,7 +94,6 @@ public String toString() { ", remoteSqlPluginPath='" + remoteSqlPluginPath + '\'' + ", pluginLoadMode='" + pluginLoadMode + '\'' + ", deployMode='" + deployMode + '\'' + - ", logLevel='" + logLevel + '\'' + ", confProp=" + confProp + '}'; } @@ -160,10 +153,6 @@ public ParamsInfo.Builder setDeployMode(String deployMode) { return this; } - public ParamsInfo.Builder setLogLevel(String logLevel) { - this.logLevel = logLevel; - return this; - } public ParamsInfo.Builder setConfProp(Properties confProp) { this.confProp = confProp; @@ -172,7 +161,7 @@ public ParamsInfo.Builder setConfProp(Properties confProp) { public ParamsInfo build() { return new ParamsInfo(sql, name, jarUrlList, localSqlPluginPath, - remoteSqlPluginPath, pluginLoadMode, deployMode, logLevel, confProp); + remoteSqlPluginPath, pluginLoadMode, deployMode, confProp); } } } From 1a165d245443191af98a628ac85d5407634db3a6 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 2 Mar 2020 10:47:43 +0800 Subject: [PATCH 078/523] modify apiResult objectMapper --- core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java b/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java index 2da763988..d67a39a40 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java @@ -1,11 +1,9 @@ package com.dtstack.flink.sql.exec; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.UUID; - /** * API调用结果返回 * Date: 2020/2/24 From fbf268e37a8dd78290641b02d159c228bccd1ea9 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 2 Mar 2020 16:28:51 +0800 Subject: [PATCH 079/523] set max batch size --- .../com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java index d43104fcb..4554bb798 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java @@ -29,6 +29,8 @@ */ public class RdbTableInfo extends TargetTableInfo { + public static final int MAX_BATCH_SIZE = 10000; + public static final String URL_KEY = "url"; public static final String TABLE_NAME_KEY = "tableName"; @@ -144,6 +146,10 @@ public boolean check() { Preconditions.checkNotNull(tableName, "rdb field of tableName is required"); Preconditions.checkNotNull(userName, "rdb field of userName is required"); Preconditions.checkNotNull(password, "rdb field of password is required"); + + if (null != batchSize) { + Preconditions.checkArgument(batchSize <= MAX_BATCH_SIZE, "batchSize must be less than " + MAX_BATCH_SIZE); + } return true; } From 286a796e96d0c3e9276a856559c178df05f21a33 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 12:26:08 +0800 Subject: [PATCH 080/523] keyParititon function --- .../format/SerializationMetricWrapper.java | 4 ++ .../kafka/AbstractKafkaProducerFactory.java | 2 +- .../sink/kafka/CustomerFlinkPartition.java | 19 +++++ .../CustomerKeyedSerializationSchema.java | 69 +++++++++++++++++++ .../sql/sink/kafka/table/KafkaSinkParser.java | 3 + .../sink/kafka/table/KafkaSinkTableInfo.java | 24 +++++++ .../flink/sql/sink/kafka/KafkaProducer.java | 4 +- .../sql/sink/kafka/KafkaProducerFactory.java | 4 +- .../flink/sql/sink/kafka/KafkaSink.java | 21 +++++- .../flink/sql/sink/kafka/KafkaProducer09.java | 4 +- .../sink/kafka/KafkaProducer09Factory.java | 4 +- .../flink/sql/sink/kafka/KafkaSink.java | 22 +++++- .../sql/sink/kafka/KafkaProducer010.java | 4 +- .../sink/kafka/KafkaProducer010Factory.java | 4 +- .../flink/sql/sink/kafka/KafkaSink.java | 22 +++++- .../sql/sink/kafka/KafkaProducer011.java | 4 +- .../sink/kafka/KafkaProducer011Factory.java | 5 +- .../flink/sql/sink/kafka/KafkaSink.java | 22 +++++- 18 files changed, 216 insertions(+), 25 deletions(-) create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java diff --git a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java index bfcf476fc..5f547ac64 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java @@ -77,4 +77,8 @@ public void setRuntimeContext(RuntimeContext runtimeContext) { this.runtimeContext = runtimeContext; } + public SerializationSchema getSerializationSchema() { + return serializationSchema; + } + } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java index 30d6cea16..88c2ca939 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java @@ -51,7 +51,7 @@ public abstract class AbstractKafkaProducerFactory { * @param partitioner * @return */ - public abstract RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner); + public abstract RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys); protected SerializationMetricWrapper createSerializationMetricWrapper(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { return new SerializationMetricWrapper(createSerializationSchema(kafkaSinkTableInfo, typeInformation)); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java new file mode 100644 index 000000000..1d456a887 --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -0,0 +1,19 @@ +package com.dtstack.flink.sql.sink.kafka; + + +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; + +import java.util.Random; +import org.apache.flink.util.Preconditions; + +public class CustomerFlinkPartition extends FlinkFixedPartitioner { + @Override + public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { + Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); + if(key == null){ + Random random = new Random(); + return partitions[random.nextInt(1000) % partitions.length]; + } + return partitions[key.hashCode() % partitions.length]; + } +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java new file mode 100644 index 000000000..8952391b6 --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -0,0 +1,69 @@ +package com.dtstack.flink.sql.sink.kafka; + + +import com.dtstack.flink.sql.format.SerializationMetricWrapper; +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.formats.json.JsonRowSchemaConverter; +import org.apache.flink.formats.json.JsonRowSerializationSchema; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.Meter; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema; +import org.apache.flink.types.Row; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { + + private Logger log = LoggerFactory.getLogger(getClass()); + + private static final long serialVersionUID = 1L; + private final SerializationMetricWrapper serializationMetricWrapper; + private String[] partitionKeys; + private ObjectMapper mapper = null; + + public CustomerKeyedSerializationSchema(SerializationMetricWrapper serializationMetricWrapper, String[] partitionKeys) { + this.serializationMetricWrapper = serializationMetricWrapper; + this.partitionKeys = partitionKeys; + this.mapper = new ObjectMapper(); + } + + public byte[] serializeKey(Row element) { + if(partitionKeys == null || partitionKeys.length <=0){ + return null; + } + SerializationSchema serializationSchema = serializationMetricWrapper.getSerializationSchema(); + if(serializationSchema instanceof JsonRowSerializationSchema){ + return serializeJsonKey((JsonRowSerializationSchema) serializationSchema, element); + } + return null; + } + + public byte[] serializeValue(Row element) { + return this.serializationMetricWrapper.serialize(element); + } + + public String getTargetTopic(Row element) { + return null; + } + + private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationSchema, Row element) { + try { + byte[] data = jsonRowSerializationSchema.serialize(element); + ObjectNode objectNode = mapper.readValue(data, ObjectNode.class); + StringBuilder sb = new StringBuilder(); + for(String key : partitionKeys){ + if(objectNode.has(key)){ + sb.append(objectNode.get(key)); + } + } + return sb.toString().getBytes(); + }catch (Exception e){ + log.error("serializeJsonKey error", e); + } + return null; + + } +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index 4fb567f82..8ba21b953 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -49,6 +49,9 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map kafkaParam = new HashMap(); @@ -51,6 +55,10 @@ public class KafkaSinkTableInfo extends TargetTableInfo { private String fieldDelimiter; + private String enableKeyPartition; + + private String partitionKeys; + public void addKafkaParam(String key, String value) { kafkaParam.put(key, value); } @@ -104,4 +112,20 @@ public boolean check() { return false; } + public String getEnableKeyPartition() { + return enableKeyPartition; + } + + public void setEnableKeyPartition(String enableKeyPartition) { + this.enableKeyPartition = enableKeyPartition; + } + + public String getPartitionKeys() { + return partitionKeys; + } + + public void setPartitionKeys(String partitionKeys) { + this.partitionKeys = partitionKeys; + } + } \ No newline at end of file diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java index 41163c9d1..f818ba9ca 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java @@ -39,8 +39,8 @@ public class KafkaProducer extends FlinkKafkaProducer { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner) { - super(topicId, serializationSchema, producerConfig, customPartitioner); + public KafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] parititonKeys) { + super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, parititonKeys), producerConfig, customPartitioner); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java index 358c64440..f3a2f40f5 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java @@ -36,7 +36,7 @@ public class KafkaProducerFactory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner) { - return new KafkaProducer(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner); + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + return new KafkaProducer(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index ba924d474..8d7c4585b 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.TargetTableInfo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -63,6 +64,8 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< /** Partitioner to select Kafka partition for each item. */ protected Optional> partitioner; + private String[] partitionKeys; + @Override public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafkaSinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; @@ -74,7 +77,8 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { for (String key : kafkaSinkTableInfo.getKafkaParamKeys()) { properties.setProperty(key, kafkaSinkTableInfo.getKafkaParam(key)); } - this.partitioner = Optional.of(new FlinkFixedPartitioner<>()); + this.partitioner = Optional.of(getFlinkPartitioner(kafkaSinkTableInfo)); + this.partitionKeys = getPartitionKeys(kafkaSinkTableInfo); this.fieldNames = kafkaSinkTableInfo.getFields(); TypeInformation[] types = new TypeInformation[kafkaSinkTableInfo.getFields().length]; for (int i = 0; i < kafkaSinkTableInfo.getFieldClasses().length; i++) { @@ -93,7 +97,7 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { this.parallelism = parallelism; } - this.flinkKafkaProducer = (FlinkKafkaProducer) new KafkaProducerFactory().createKafkaProducer(kafkaSinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner); + this.flinkKafkaProducer = (FlinkKafkaProducer) new KafkaProducerFactory().createKafkaProducer(kafkaSinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); return this; } @@ -133,4 +137,17 @@ public TableSink> configure(String[] fieldNames, TypeInform this.fieldTypes = fieldTypes; return this; } + private FlinkKafkaPartitioner getFlinkPartitioner(KafkaSinkTableInfo kafkaSinkTableInfo){ + if("true".equalsIgnoreCase(kafkaSinkTableInfo.getEnableKeyPartition())){ + return new CustomerFlinkPartition<>(); + } + return new FlinkFixedPartitioner<>(); + } + + private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ + if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java index 55457e368..7b569d98d 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java @@ -39,8 +39,8 @@ public class KafkaProducer09 extends FlinkKafkaProducer09 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer09(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner) { - super(topicId, serializationSchema, producerConfig, customPartitioner.orElse(null)); + public KafkaProducer09(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner,String[] partitionKeys) { + super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.orElse(null)); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java index 10620b36d..7fb3909ee 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java @@ -36,7 +36,7 @@ public class KafkaProducer09Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner) { - return new KafkaProducer09(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner); + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner,String[] partitionKeys) { + return new KafkaProducer09(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 6d39536a5..7ec1b4556 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.TargetTableInfo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -63,6 +64,8 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< /** Partitioner to select Kafka partition for each item. */ protected Optional> partitioner; + private String[] partitionKeys; + protected int parallelism; @@ -78,7 +81,8 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { properties.setProperty(key, kafka09SinkTableInfo.getKafkaParam(key)); } - this.partitioner = Optional.of(new FlinkFixedPartitioner<>()); + this.partitioner = Optional.of(getFlinkPartitioner(kafka09SinkTableInfo)); + this.partitionKeys = getPartitionKeys(kafka09SinkTableInfo); this.fieldNames = kafka09SinkTableInfo.getFields(); TypeInformation[] types = new TypeInformation[kafka09SinkTableInfo.getFields().length]; for (int i = 0; i < kafka09SinkTableInfo.getFieldClasses().length; i++) { @@ -98,7 +102,7 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { } this.kafkaProducer09 = (FlinkKafkaProducer09) new KafkaProducer09Factory() - .createKafkaProducer(kafka09SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner); + .createKafkaProducer(kafka09SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); return this; } @@ -140,4 +144,18 @@ public TableSink> configure(String[] fieldNames, TypeInform return this; } + private FlinkKafkaPartitioner getFlinkPartitioner(KafkaSinkTableInfo kafkaSinkTableInfo){ + if("true".equalsIgnoreCase(kafkaSinkTableInfo.getEnableKeyPartition())){ + return new CustomerFlinkPartition<>(); + } + return new FlinkFixedPartitioner<>(); + } + + private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ + if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } + } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java index 35f9f0072..3e93bdf21 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java @@ -39,8 +39,8 @@ public class KafkaProducer010 extends FlinkKafkaProducer010 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner) { - super(topicId, serializationSchema, producerConfig, customPartitioner.orElse(null)); + public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.orElse(null)); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java index 59a0f90d9..c44a9fe86 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java @@ -36,7 +36,7 @@ public class KafkaProducer010Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner) { - return new KafkaProducer010(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner); + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + return new KafkaProducer010(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index dee05ab08..022b3d409 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.TargetTableInfo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -67,6 +68,7 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< /** Partitioner to select Kafka partition for each item. */ protected Optional> partitioner; + private String[] partitionKeys; @Override public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { @@ -80,7 +82,8 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { properties.setProperty(key, kafka10SinkTableInfo.getKafkaParam(key)); } - this.partitioner = Optional.of(new FlinkFixedPartitioner<>()); + this.partitioner = Optional.of(getFlinkPartitioner(kafka10SinkTableInfo)); + this.partitionKeys = getPartitionKeys(kafka10SinkTableInfo); this.fieldNames = kafka10SinkTableInfo.getFields(); TypeInformation[] types = new TypeInformation[kafka10SinkTableInfo.getFields().length]; for (int i = 0; i < kafka10SinkTableInfo.getFieldClasses().length; i++) { @@ -100,7 +103,7 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { this.parallelism = parallelism; } - this.kafkaProducer010 = (FlinkKafkaProducer010) new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner); + this.kafkaProducer010 = (FlinkKafkaProducer010) new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); return this; } @@ -140,4 +143,19 @@ public TableSink> configure(String[] fieldNames, TypeInform this.fieldTypes = fieldTypes; return this; } + + private FlinkKafkaPartitioner getFlinkPartitioner(KafkaSinkTableInfo kafkaSinkTableInfo){ + if("true".equalsIgnoreCase(kafkaSinkTableInfo.getEnableKeyPartition())){ + return new CustomerFlinkPartition<>(); + } + return new FlinkFixedPartitioner<>(); + } + + private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ + if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } + } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java index 47b977f41..52385f89a 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java @@ -40,8 +40,8 @@ public class KafkaProducer011 extends FlinkKafkaProducer011 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer011(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner) { - super(topicId, serializationSchema, producerConfig, customPartitioner); + public KafkaProducer011(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java index 68e9b3d3f..e2272b16e 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java @@ -36,7 +36,8 @@ public class KafkaProducer011Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner) { - return new KafkaProducer011(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner); + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, + String[] partitionKeys) { + return new KafkaProducer011(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index c2ef7d35a..5f3426233 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.TargetTableInfo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -67,6 +68,7 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener /** Partitioner to select Kafka partition for each item. */ protected Optional> partitioner; + private String[] partitionKeys; @Override @@ -80,7 +82,8 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { for (String key : kafka11SinkTableInfo.getKafkaParamKeys()) { properties.setProperty(key, kafka11SinkTableInfo.getKafkaParam(key)); } - this.partitioner = Optional.of(new FlinkFixedPartitioner<>()); + this.partitioner = Optional.of(getFlinkPartitioner(kafka11SinkTableInfo)); + this.partitionKeys = getPartitionKeys(kafka11SinkTableInfo); this.fieldNames = kafka11SinkTableInfo.getFields(); TypeInformation[] types = new TypeInformation[kafka11SinkTableInfo.getFields().length]; for (int i = 0; i < kafka11SinkTableInfo.getFieldClasses().length; i++) { @@ -100,7 +103,7 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { } this.kafkaProducer011 = (FlinkKafkaProducer011) new KafkaProducer011Factory() - .createKafkaProducer(kafka11SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner); + .createKafkaProducer(kafka11SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); return this; } @@ -140,4 +143,19 @@ public TypeInformation[] getFieldTypes() { this.fieldTypes = fieldTypes; return this; } + + private FlinkKafkaPartitioner getFlinkPartitioner(KafkaSinkTableInfo kafkaSinkTableInfo){ + if("true".equalsIgnoreCase(kafkaSinkTableInfo.getEnableKeyPartition())){ + return new CustomerFlinkPartition<>(); + } + return new FlinkFixedPartitioner<>(); + } + + private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ + if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } + } From c0c7a23cdeb0653dbf65c22cd6b7764744e0dcdd Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 12:41:57 +0800 Subject: [PATCH 081/523] remove logger --- .../sink/kafka/CustomerKeyedSerializationSchema.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index 8952391b6..dcf1de8dd 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -2,23 +2,15 @@ import com.dtstack.flink.sql.format.SerializationMetricWrapper; -import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.formats.json.JsonRowSchemaConverter; import org.apache.flink.formats.json.JsonRowSerializationSchema; -import org.apache.flink.metrics.Counter; -import org.apache.flink.metrics.Meter; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema; import org.apache.flink.types.Row; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { - private Logger log = LoggerFactory.getLogger(getClass()); - private static final long serialVersionUID = 1L; private final SerializationMetricWrapper serializationMetricWrapper; private String[] partitionKeys; @@ -61,7 +53,7 @@ private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationS } return sb.toString().getBytes(); }catch (Exception e){ - log.error("serializeJsonKey error", e); + } return null; From f51bf15a33eabf17af514d800b292a555dfa1b90 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 13:18:32 +0800 Subject: [PATCH 082/523] fix filed --- .../dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java index 061d44a64..bd2fff811 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java @@ -41,7 +41,7 @@ public class KafkaSinkTableInfo extends TargetTableInfo { public static final String TYPE_KEY = "type"; - public static final String ENABLE_KEY_PARTITION_KEY = "enableKeyPartition"; + public static final String ENABLE_KEY_PARTITION_KEY = "enableKeyPartitions"; public static final String PARTITION_KEY = "partitionKeys"; From 174f70fe8a9e25e25a39c040557338276f4e77fd Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 13:37:49 +0800 Subject: [PATCH 083/523] test log --- .../flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index dcf1de8dd..6317e01b4 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -51,7 +51,7 @@ private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationS sb.append(objectNode.get(key)); } } - return sb.toString().getBytes(); + return null; }catch (Exception e){ } From 214562d1719c58417e6c413aff2b03f4c5f6e308 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 15:02:48 +0800 Subject: [PATCH 084/523] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java | 1 + .../flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java | 1 + 2 files changed, 2 insertions(+) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index 1d456a887..055882761 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -9,6 +9,7 @@ public class CustomerFlinkPartition extends FlinkFixedPartitioner { @Override public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { + System.out.println("key="+new String(key)); Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); if(key == null){ Random random = new Random(); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index 6317e01b4..fad8bfc0c 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -23,6 +23,7 @@ public CustomerKeyedSerializationSchema(SerializationMetricWrapper serialization } public byte[] serializeKey(Row element) { + System.out.println("element = " + element+"|partitionKeys=" + partitionKeys); if(partitionKeys == null || partitionKeys.length <=0){ return null; } From 0628f06d5f6a91e7f04687552a85bb5eed3c146b Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 15:32:41 +0800 Subject: [PATCH 085/523] test system.out --- .../sql/sink/kafka/CustomerKeyedSerializationSchema.java | 8 +++++--- .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index fad8bfc0c..22b283a99 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -23,12 +23,12 @@ public CustomerKeyedSerializationSchema(SerializationMetricWrapper serialization } public byte[] serializeKey(Row element) { - System.out.println("element = " + element+"|partitionKeys=" + partitionKeys); if(partitionKeys == null || partitionKeys.length <=0){ return null; - } + } SerializationSchema serializationSchema = serializationMetricWrapper.getSerializationSchema(); if(serializationSchema instanceof JsonRowSerializationSchema){ + System.out.println("serial start, element = " + element); return serializeJsonKey((JsonRowSerializationSchema) serializationSchema, element); } return null; @@ -46,13 +46,15 @@ private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationS try { byte[] data = jsonRowSerializationSchema.serialize(element); ObjectNode objectNode = mapper.readValue(data, ObjectNode.class); + System.out.println("objectNode = " + objectNode); StringBuilder sb = new StringBuilder(); for(String key : partitionKeys){ if(objectNode.has(key)){ + System.out.println("key = " + key+ ", value = " + objectNode.get(key)); sb.append(objectNode.get(key)); } } - return null; + return sb.toString().getBytes(); }catch (Exception e){ } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 022b3d409..f5050ab69 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -102,7 +102,6 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { if (parallelism != null) { this.parallelism = parallelism; } - this.kafkaProducer010 = (FlinkKafkaProducer010) new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); return this; } @@ -145,6 +144,7 @@ public TableSink> configure(String[] fieldNames, TypeInform } private FlinkKafkaPartitioner getFlinkPartitioner(KafkaSinkTableInfo kafkaSinkTableInfo){ + System.out.println("enablePartition =" + kafkaSinkTableInfo.getEnableKeyPartition()); if("true".equalsIgnoreCase(kafkaSinkTableInfo.getEnableKeyPartition())){ return new CustomerFlinkPartition<>(); } @@ -152,6 +152,7 @@ private FlinkKafkaPartitioner getFlinkPartitioner(KafkaSinkTableInfo kafkaSinkTa } private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ + System.out.println("partitionKeys =" + kafkaSinkTableInfo.getPartitionKeys()); if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); } From ff67b88aa25f672339b36fa82f40679041629a23 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 16:03:23 +0800 Subject: [PATCH 086/523] last commit --- .../flink/sql/sink/kafka/CustomerFlinkPartition.java | 4 +--- .../sink/kafka/CustomerKeyedSerializationSchema.java | 3 --- .../com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 12 +----------- 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index 055882761..a0be560bb 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -9,11 +9,9 @@ public class CustomerFlinkPartition extends FlinkFixedPartitioner { @Override public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { - System.out.println("key="+new String(key)); Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); if(key == null){ - Random random = new Random(); - return partitions[random.nextInt(1000) % partitions.length]; + super.partition(record, key, value, targetTopic, partitions); } return partitions[key.hashCode() % partitions.length]; } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index 22b283a99..de8cfbcd9 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -28,7 +28,6 @@ public byte[] serializeKey(Row element) { } SerializationSchema serializationSchema = serializationMetricWrapper.getSerializationSchema(); if(serializationSchema instanceof JsonRowSerializationSchema){ - System.out.println("serial start, element = " + element); return serializeJsonKey((JsonRowSerializationSchema) serializationSchema, element); } return null; @@ -46,11 +45,9 @@ private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationS try { byte[] data = jsonRowSerializationSchema.serialize(element); ObjectNode objectNode = mapper.readValue(data, ObjectNode.class); - System.out.println("objectNode = " + objectNode); StringBuilder sb = new StringBuilder(); for(String key : partitionKeys){ if(objectNode.has(key)){ - System.out.println("key = " + key+ ", value = " + objectNode.get(key)); sb.append(objectNode.get(key)); } } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index f5050ab69..72e90404c 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -82,7 +82,7 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { properties.setProperty(key, kafka10SinkTableInfo.getKafkaParam(key)); } - this.partitioner = Optional.of(getFlinkPartitioner(kafka10SinkTableInfo)); + this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka10SinkTableInfo); this.fieldNames = kafka10SinkTableInfo.getFields(); TypeInformation[] types = new TypeInformation[kafka10SinkTableInfo.getFields().length]; @@ -142,17 +142,7 @@ public TableSink> configure(String[] fieldNames, TypeInform this.fieldTypes = fieldTypes; return this; } - - private FlinkKafkaPartitioner getFlinkPartitioner(KafkaSinkTableInfo kafkaSinkTableInfo){ - System.out.println("enablePartition =" + kafkaSinkTableInfo.getEnableKeyPartition()); - if("true".equalsIgnoreCase(kafkaSinkTableInfo.getEnableKeyPartition())){ - return new CustomerFlinkPartition<>(); - } - return new FlinkFixedPartitioner<>(); - } - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - System.out.println("partitionKeys =" + kafkaSinkTableInfo.getPartitionKeys()); if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); } From f0687cb83922cf13f621b13652050216ef6604dc Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 16:16:45 +0800 Subject: [PATCH 087/523] log test again --- .../dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index a0be560bb..d892e75df 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -3,6 +3,7 @@ import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; +import java.util.Arrays; import java.util.Random; import org.apache.flink.util.Preconditions; @@ -10,6 +11,7 @@ public class CustomerFlinkPartition extends FlinkFixedPartitioner { @Override public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); + System.out.println("key = " + key+ ", value = " + value+ ", targetTopic = " + targetTopic + ", partitions = " + Arrays.toString(partitions)); if(key == null){ super.partition(record, key, value, targetTopic, partitions); } From f2fef32f1ab1fce05c1db9ff4545bbbca5fcd5c8 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 16:43:36 +0800 Subject: [PATCH 088/523] fix last one --- .../dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java | 3 +-- .../main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 2 +- .../main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 2 +- .../main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index d892e75df..3667fbea6 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -11,9 +11,8 @@ public class CustomerFlinkPartition extends FlinkFixedPartitioner { @Override public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); - System.out.println("key = " + key+ ", value = " + value+ ", targetTopic = " + targetTopic + ", partitions = " + Arrays.toString(partitions)); if(key == null){ - super.partition(record, key, value, targetTopic, partitions); + return super.partition(record, key, value, targetTopic, partitions); } return partitions[key.hashCode() % partitions.length]; } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 8d7c4585b..9b12bf09c 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -77,7 +77,7 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { for (String key : kafkaSinkTableInfo.getKafkaParamKeys()) { properties.setProperty(key, kafkaSinkTableInfo.getKafkaParam(key)); } - this.partitioner = Optional.of(getFlinkPartitioner(kafkaSinkTableInfo)); + this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafkaSinkTableInfo); this.fieldNames = kafkaSinkTableInfo.getFields(); TypeInformation[] types = new TypeInformation[kafkaSinkTableInfo.getFields().length]; diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 7ec1b4556..68255e057 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -81,7 +81,7 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { properties.setProperty(key, kafka09SinkTableInfo.getKafkaParam(key)); } - this.partitioner = Optional.of(getFlinkPartitioner(kafka09SinkTableInfo)); + this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka09SinkTableInfo); this.fieldNames = kafka09SinkTableInfo.getFields(); TypeInformation[] types = new TypeInformation[kafka09SinkTableInfo.getFields().length]; diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 5f3426233..a5af15f4f 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -82,7 +82,7 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { for (String key : kafka11SinkTableInfo.getKafkaParamKeys()) { properties.setProperty(key, kafka11SinkTableInfo.getKafkaParam(key)); } - this.partitioner = Optional.of(getFlinkPartitioner(kafka11SinkTableInfo)); + this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka11SinkTableInfo); this.fieldNames = kafka11SinkTableInfo.getFields(); TypeInformation[] types = new TypeInformation[kafka11SinkTableInfo.getFields().length]; From c64518ee5ce0ab9eca111d76bb1bc29e507d2183 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 17:11:01 +0800 Subject: [PATCH 089/523] out test --- .../dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java | 4 +++- .../sql/sink/kafka/CustomerKeyedSerializationSchema.java | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index 3667fbea6..47448a3ff 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -14,6 +14,8 @@ public int partition(T record, byte[] key, byte[] value, String targetTopic, int if(key == null){ return super.partition(record, key, value, targetTopic, partitions); } - return partitions[key.hashCode() % partitions.length]; + System.out.println("hashcode=" + key.hashCode()); + Random random = new Random(); + return partitions[random.nextInt() % partitions.length]; } } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index de8cfbcd9..7642c88ce 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -51,6 +51,7 @@ private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationS sb.append(objectNode.get(key)); } } + System.out.println("serialKey=" + sb.toString()); return sb.toString().getBytes(); }catch (Exception e){ From ea131b6fc44aa83a8c0833a3eb1598f039d6ba6f Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 17:28:44 +0800 Subject: [PATCH 090/523] so --- .../dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java | 4 +--- .../sql/sink/kafka/CustomerKeyedSerializationSchema.java | 1 - .../com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index 47448a3ff..3667fbea6 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -14,8 +14,6 @@ public int partition(T record, byte[] key, byte[] value, String targetTopic, int if(key == null){ return super.partition(record, key, value, targetTopic, partitions); } - System.out.println("hashcode=" + key.hashCode()); - Random random = new Random(); - return partitions[random.nextInt() % partitions.length]; + return partitions[key.hashCode() % partitions.length]; } } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index 7642c88ce..de8cfbcd9 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -51,7 +51,6 @@ private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationS sb.append(objectNode.get(key)); } } - System.out.println("serialKey=" + sb.toString()); return sb.toString().getBytes(); }catch (Exception e){ diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java index 3e93bdf21..20d831d8b 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java @@ -40,7 +40,7 @@ public class KafkaProducer010 extends FlinkKafkaProducer010 { private SerializationMetricWrapper serializationMetricWrapper; public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { - super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.orElse(null)); + super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.get()); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } From aa5e8c5f177301bac9136eed036260d66754b870 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 18:15:14 +0800 Subject: [PATCH 091/523] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E7=9A=84=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/kafka/CustomerFlinkPartition.java | 2 -- .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 6 ------ .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 7 ------- .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 7 ------- 4 files changed, 22 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index 3667fbea6..83489969e 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -3,8 +3,6 @@ import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import java.util.Arrays; -import java.util.Random; import org.apache.flink.util.Preconditions; public class CustomerFlinkPartition extends FlinkFixedPartitioner { diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 9b12bf09c..d0d477f4e 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -137,12 +137,6 @@ public TableSink> configure(String[] fieldNames, TypeInform this.fieldTypes = fieldTypes; return this; } - private FlinkKafkaPartitioner getFlinkPartitioner(KafkaSinkTableInfo kafkaSinkTableInfo){ - if("true".equalsIgnoreCase(kafkaSinkTableInfo.getEnableKeyPartition())){ - return new CustomerFlinkPartition<>(); - } - return new FlinkFixedPartitioner<>(); - } private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 68255e057..52caa7bd2 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -144,13 +144,6 @@ public TableSink> configure(String[] fieldNames, TypeInform return this; } - private FlinkKafkaPartitioner getFlinkPartitioner(KafkaSinkTableInfo kafkaSinkTableInfo){ - if("true".equalsIgnoreCase(kafkaSinkTableInfo.getEnableKeyPartition())){ - return new CustomerFlinkPartition<>(); - } - return new FlinkFixedPartitioner<>(); - } - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index a5af15f4f..c5da782d0 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -144,13 +144,6 @@ public TypeInformation[] getFieldTypes() { return this; } - private FlinkKafkaPartitioner getFlinkPartitioner(KafkaSinkTableInfo kafkaSinkTableInfo){ - if("true".equalsIgnoreCase(kafkaSinkTableInfo.getEnableKeyPartition())){ - return new CustomerFlinkPartition<>(); - } - return new FlinkFixedPartitioner<>(); - } - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); From 9995f5823c12826a851277d60968f7107b0440fb Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 19:47:50 +0800 Subject: [PATCH 092/523] test partition function --- .../sink/kafka/CustomerFlinkPartition.java | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index 83489969e..ac0b5483d 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -3,15 +3,36 @@ import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.util.Preconditions; -public class CustomerFlinkPartition extends FlinkFixedPartitioner { - @Override +public class CustomerFlinkPartition extends FlinkKafkaPartitioner { + private static final long serialVersionUID = 1L; + private int parallelInstanceId; + + public CustomerFlinkPartition() { + + } + + public void open(int parallelInstanceId, int parallelInstances) { + Preconditions.checkArgument(parallelInstanceId >= 0, "Id of this subtask cannot be negative."); + Preconditions.checkArgument(parallelInstances > 0, "Number of subtasks must be larger than 0."); + this.parallelInstanceId = parallelInstanceId; + } + public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); if(key == null){ - return super.partition(record, key, value, targetTopic, partitions); + return partitions[this.parallelInstanceId % partitions.length]; } return partitions[key.hashCode() % partitions.length]; } + + public boolean equals(Object o) { + return this == o || o instanceof CustomerFlinkPartition; + } + + public int hashCode() { + return CustomerFlinkPartition.class.hashCode(); + } } From 68af363f95649f4bd43863acb2bec525c0297db7 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 21:13:02 +0800 Subject: [PATCH 093/523] =?UTF-8?q?=E6=B5=8B=E8=AF=95partition?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 72e90404c..3391bc93b 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -66,8 +66,6 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< /** The schema of the table. */ private TableSchema schema; - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; private String[] partitionKeys; @Override @@ -82,7 +80,6 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { properties.setProperty(key, kafka10SinkTableInfo.getKafkaParam(key)); } - this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka10SinkTableInfo); this.fieldNames = kafka10SinkTableInfo.getFields(); TypeInformation[] types = new TypeInformation[kafka10SinkTableInfo.getFields().length]; @@ -102,7 +99,8 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { if (parallelism != null) { this.parallelism = parallelism; } - this.kafkaProducer010 = (FlinkKafkaProducer010) new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); + this.kafkaProducer010 = (FlinkKafkaProducer010) new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, + Optional.of(new CustomerFlinkPartition<>()), partitionKeys); return this; } From c3d8ed4f324c046d0d383fad8d32f828696c5936 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 21:27:59 +0800 Subject: [PATCH 094/523] test partition kafka sink --- .../com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 3391bc93b..f89e83176 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -61,7 +61,7 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< protected int parallelism; - protected FlinkKafkaProducer010 kafkaProducer010; + protected KafkaSinkTableInfo kafka10SinkTableInfo; /** The schema of the table. */ private TableSchema schema; @@ -70,7 +70,7 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< @Override public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { - KafkaSinkTableInfo kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; + this.kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; this.topic = kafka10SinkTableInfo.getTopic(); properties = new Properties(); @@ -99,8 +99,6 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { if (parallelism != null) { this.parallelism = parallelism; } - this.kafkaProducer010 = (FlinkKafkaProducer010) new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, - Optional.of(new CustomerFlinkPartition<>()), partitionKeys); return this; } @@ -111,6 +109,10 @@ public TypeInformation getRecordType() { @Override public void emitDataStream(DataStream> dataStream) { + + FlinkKafkaProducer010 kafkaProducer010 = (FlinkKafkaProducer010) new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, + Optional.of(new CustomerFlinkPartition<>()), partitionKeys); + DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) .map((Tuple2 record) -> record.f1) .returns(getOutputType().getTypeAt(1)) From e0b29ca79303ac8a078100affa70dc4248f3dc23 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 22:13:37 +0800 Subject: [PATCH 095/523] fix blank and hashcode question --- .../dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java | 3 ++- .../flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index ac0b5483d..6b5297bb6 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -22,10 +22,11 @@ public void open(int parallelInstanceId, int parallelInstances) { public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); + System.out.println("partitionKey=" + new String(key)); if(key == null){ return partitions[this.parallelInstanceId % partitions.length]; } - return partitions[key.hashCode() % partitions.length]; + return partitions[new String(key).hashCode() % partitions.length]; } public boolean equals(Object o) { diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index de8cfbcd9..0dfb003bf 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -48,7 +48,7 @@ private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationS StringBuilder sb = new StringBuilder(); for(String key : partitionKeys){ if(objectNode.has(key)){ - sb.append(objectNode.get(key)); + sb.append(objectNode.get(key.trim())); } } return sb.toString().getBytes(); From 3db8088d41680ae7371a0dae06ac58fa3ac61203 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 3 Mar 2020 22:20:57 +0800 Subject: [PATCH 096/523] log fix and abs hashcode --- .../dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index 6b5297bb6..b9fcfa98b 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -22,11 +22,12 @@ public void open(int parallelInstanceId, int parallelInstances) { public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); - System.out.println("partitionKey=" + new String(key)); if(key == null){ + System.out.println("partitionKey=null"); return partitions[this.parallelInstanceId % partitions.length]; } - return partitions[new String(key).hashCode() % partitions.length]; + System.out.println("partitionKey=" + new String(key)); + return partitions[Math.abs(new String(key).hashCode()) % partitions.length]; } public boolean equals(Object o) { From e4eeebb7f9b1a21d9067280b644838d14ba87074 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 4 Mar 2020 00:07:44 +0800 Subject: [PATCH 097/523] remove unuse import --- .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index f89e83176..fd39a8af6 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -27,9 +27,7 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; @@ -110,8 +108,8 @@ public TypeInformation getRecordType() { @Override public void emitDataStream(DataStream> dataStream) { - FlinkKafkaProducer010 kafkaProducer010 = (FlinkKafkaProducer010) new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, - Optional.of(new CustomerFlinkPartition<>()), partitionKeys); + RichSinkFunction kafkaProducer010 = new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, + Optional.of(new CustomerFlinkPartition()), partitionKeys); DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) .map((Tuple2 record) -> record.f1) From 586c74a117b02aed822dbe311ca79419856137f3 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 4 Mar 2020 10:22:35 +0800 Subject: [PATCH 098/523] es id number check --- ...asticsearch6Side => elasticsearch6Side.md} | 0 .../table/ElasticsearchTableInfo.java | 8 +++ .../sink/elasticsearch/CustomerSinkFunc.java | 38 ++++++------ .../sink/elasticsearch/ElasticsearchSink.java | 62 +++++++------------ .../table/ElasticsearchTableInfo.java | 8 +++ pom.xml | 2 +- 6 files changed, 56 insertions(+), 62 deletions(-) rename docs/{elasticsearch6Side => elasticsearch6Side.md} (100%) diff --git a/docs/elasticsearch6Side b/docs/elasticsearch6Side.md similarity index 100% rename from docs/elasticsearch6Side rename to docs/elasticsearch6Side.md diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index ac1d712ce..b0439978e 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -23,6 +23,10 @@ import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; + +import java.util.Arrays; /** * @date 2018/09/12 @@ -131,6 +135,10 @@ public boolean check() { Preconditions.checkNotNull(id, "elasticsearch type of id is required"); Preconditions.checkNotNull(clusterName, "elasticsearch type of clusterName is required"); + Arrays.stream(StringUtils.split(id, ",")).forEach(number -> { + Preconditions.checkArgument(NumberUtils.isNumber(number), "id must be a numeric type"); + }); + if (isAuthMesh()) { Preconditions.checkNotNull(userName, "elasticsearch type of userName is required"); Preconditions.checkNotNull(password, "elasticsearch type of password is required"); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index d6d207c9e..b1c8a5e97 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -31,9 +31,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * @author yinxi @@ -42,6 +42,8 @@ public class CustomerSinkFunc implements ElasticsearchSinkFunction { private final Logger logger = LoggerFactory.getLogger(CustomerSinkFunc.class); + /** 用作ID的属性值连接符号 */ + private static final String ID_VALUE_SPLIT = "_"; private String index; @@ -57,10 +59,7 @@ public class CustomerSinkFunc implements ElasticsearchSinkFunction { private transient Counter outDirtyRecords; - /** 默认分隔符为'_' */ - private char sp = '_'; - - public CustomerSinkFunc(String index, String type, List fieldNames, List fieldTypes, List idFieldIndexes){ + public CustomerSinkFunc(String index, String type, List fieldNames, List fieldTypes, List idFieldIndexes) { this.index = index; this.type = type; this.fieldNames = fieldNames; @@ -96,31 +95,30 @@ public void setOutDirtyRecords(Counter outDirtyRecords) { } private IndexRequest createIndexRequest(Row element) { + // index start at 1, + String idFieldStr = idFieldIndexList.stream() + .filter(index -> index > 0 && index <= element.getArity()) + .map(index -> element.getField(index - 1).toString()) + .collect(Collectors.joining(ID_VALUE_SPLIT)); - List idFieldList = new ArrayList<>(); - for(int index : idFieldIndexList){ - if(index >= element.getArity()){ - continue; - } - - idFieldList.add(element.getField(index).toString()); - } - - Map dataMap = Es6Util.rowToJsonMap(element,fieldNames,fieldTypes); + Map dataMap = Es6Util.rowToJsonMap(element, fieldNames, fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); - for(int i=0; i, IStreamSinkGener { - private final Logger logger = LoggerFactory.getLogger(ElasticsearchSink.class); + private final int ES_DEFAULT_PORT = 9200; + private final String ES_DEFAULT_SCHEMA = "http"; private String clusterName; @@ -103,32 +101,20 @@ public TypeInformation[] getFieldTypes() { private RichSinkFunction createEsSinkFunction() { - - Map userConfig = Maps.newHashMap(); userConfig.put("cluster.name", clusterName); // This instructs the sink to emit after every element, otherwise they would be buffered userConfig.put(org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "" + bulkFlushMaxActions); - List transports = new ArrayList<>(); - - for (String address : esAddressList) { - String[] infoArray = address.split(":"); - int port = 9200; - String host = infoArray[0]; - if (infoArray.length > 1) { - port = Integer.valueOf(infoArray[1].trim()); - } - - try { - transports.add(new HttpHost(host.trim(), port, "http")); - } catch (Exception e) { - logger.error("", e); - throw new RuntimeException(e); - } - } - CustomerSinkFunc customerSinkFunc = new CustomerSinkFunc(index, type, Arrays.asList(fieldNames), Arrays.asList(columnTypes), idIndexList); + List transports = esAddressList.stream() + .map(address -> address.split(":")) + .map(addressArray -> { + String host = addressArray[0].trim(); + int port = addressArray.length > 1 ? Integer.valueOf(addressArray[1].trim()) : ES_DEFAULT_PORT; + return new HttpHost(host.trim(), port, ES_DEFAULT_SCHEMA); + }).collect(Collectors.toList()); + CustomerSinkFunc customerSinkFunc = new CustomerSinkFunc(index, type, Arrays.asList(fieldNames), Arrays.asList(columnTypes), idIndexList); return new MetricElasticsearch6Sink(userConfig, transports, customerSinkFunc, esTableInfo); } @@ -151,23 +137,17 @@ public void setBulkFlushMaxActions(int bulkFlushMaxActions) { @Override public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { - ElasticsearchTableInfo elasticsearchTableInfo = (ElasticsearchTableInfo) targetTableInfo; - esTableInfo = elasticsearchTableInfo; - clusterName = elasticsearchTableInfo.getClusterName(); - String address = elasticsearchTableInfo.getAddress(); - String[] addr = address.split(","); - esAddressList = Arrays.asList(addr); - index = elasticsearchTableInfo.getIndex(); - type = elasticsearchTableInfo.getEsType(); - String id = elasticsearchTableInfo.getId(); - String[] idField = StringUtils.split(id, ","); - idIndexList = new ArrayList<>(); - - for (int i = 0; i < idField.length; ++i) { - idIndexList.add(Integer.valueOf(idField[i])); + esTableInfo = (ElasticsearchTableInfo) targetTableInfo; + clusterName = esTableInfo.getClusterName(); + index = esTableInfo.getIndex(); + type = esTableInfo.getEsType(); + columnTypes = esTableInfo.getFieldTypes(); + esAddressList = Arrays.asList(esTableInfo.getAddress().split(",")); + + String id = esTableInfo.getId(); + if (!StringUtils.isEmpty(id)) { + idIndexList = Arrays.stream(StringUtils.split(id, ",")).map(Integer::valueOf).collect(Collectors.toList()); } - - columnTypes = elasticsearchTableInfo.getFieldTypes(); return this; } } diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index b0156b660..5709d609c 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -20,6 +20,10 @@ import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; + +import java.util.Arrays; /** * @author yinxi @@ -127,6 +131,10 @@ public boolean check() { Preconditions.checkNotNull(id, "elasticsearch6 type of id is required"); Preconditions.checkNotNull(clusterName, "elasticsearch6 type of clusterName is required"); + Arrays.stream(StringUtils.split(id, ",")).forEach(number ->{ + Preconditions.checkArgument(NumberUtils.isNumber(number),"id must be a numeric type"); + }); + if (isAuthMesh()) { Preconditions.checkNotNull(userName, "elasticsearch6 type of userName is required"); Preconditions.checkNotNull(password, "elasticsearch6 type of password is required"); diff --git a/pom.xml b/pom.xml index 2794f9004..5493645e9 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,7 @@ mysql hbase elasticsearch5 + elasticsearch6 mongo redis5 launcher @@ -33,7 +34,6 @@ impala db2 polardb - elasticsearch6 From 3df78d7ec37de341c5abef90d10a1abd2bd348f5 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 4 Mar 2020 10:37:02 +0800 Subject: [PATCH 099/523] topic is pattern default is false --- .../dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 10049cbbe..005bfb4c2 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -51,7 +51,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Wed, 4 Mar 2020 11:14:03 +0800 Subject: [PATCH 100/523] =?UTF-8?q?fix=20=E7=BC=A9=E8=BF=9B=E5=92=8C?= =?UTF-8?q?=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/kafka/CustomerFlinkPartition.java | 2 -- .../sql/sink/kafka/CustomerKeyedSerializationSchema.java | 9 ++++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index b9fcfa98b..fb2252cdc 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -23,10 +23,8 @@ public void open(int parallelInstanceId, int parallelInstances) { public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); if(key == null){ - System.out.println("partitionKey=null"); return partitions[this.parallelInstanceId % partitions.length]; } - System.out.println("partitionKey=" + new String(key)); return partitions[Math.abs(new String(key).hashCode()) % partitions.length]; } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index 0dfb003bf..09e7e0507 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -8,9 +8,13 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema; import org.apache.flink.types.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { + private static final Logger LOG = LoggerFactory.getLogger(CustomerKeyedSerializationSchema.class); + private static final long serialVersionUID = 1L; private final SerializationMetricWrapper serializationMetricWrapper; private String[] partitionKeys; @@ -52,10 +56,9 @@ private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationS } } return sb.toString().getBytes(); - }catch (Exception e){ - + } catch (Exception e){ + LOG.error("serializeJsonKey error", e); } return null; - } } From 284d52d4ecb5dbcf51a1b379731f9b664586d183 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 4 Mar 2020 13:23:54 +0800 Subject: [PATCH 101/523] id allow as null --- .../sink/elasticsearch/CustomerSinkFunc.java | 27 +++++++++++-------- .../table/ElasticsearchTableInfo.java | 9 ++++--- .../sink/elasticsearch/ElasticsearchSink.java | 10 +------ .../table/ElasticsearchTableInfo.java | 9 ++++--- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index a95d477de..1f3efb8d7 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Reason: @@ -47,6 +48,8 @@ public class CustomerSinkFunc implements ElasticsearchSinkFunction { private final Logger logger = LoggerFactory.getLogger(CustomerSinkFunc.class); + private static final String ID_VALUE_SPLIT = "_"; + private String index; private String type; @@ -93,15 +96,11 @@ public void setOutRecords(Counter outRecords) { } private IndexRequest createIndexRequest(Row element) { - - List idFieldList = new ArrayList<>(); - for(int index : idFieldIndexList){ - if(index >= element.getArity()){ - continue; - } - - idFieldList.add(element.getField(index).toString()); - } + // index start at 1, + String idFieldStr = idFieldIndexList.stream() + .filter(index -> index > 0 && index <= element.getArity()) + .map(index -> element.getField(index - 1).toString()) + .collect(Collectors.joining(ID_VALUE_SPLIT)); Map dataMap = EsUtil.rowToJsonMap(element,fieldNames,fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); @@ -109,11 +108,17 @@ private IndexRequest createIndexRequest(Row element) { dataMap.put(fieldNames.get(i), element.getField(i)); } - String id = StringUtils.join(idFieldList, sp); + if (StringUtils.isEmpty(idFieldStr)) { + return Requests.indexRequest() + .index(index) + .type(type) + .source(dataMap); + } + return Requests.indexRequest() .index(index) .type(type) - .id(id) + .id(idFieldStr) .source(dataMap); } } diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index b0439978e..9681742c2 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -132,12 +132,13 @@ public boolean check() { Preconditions.checkNotNull(address, "elasticsearch type of address is required"); Preconditions.checkNotNull(index, "elasticsearch type of index is required"); Preconditions.checkNotNull(esType, "elasticsearch type of type is required"); - Preconditions.checkNotNull(id, "elasticsearch type of id is required"); Preconditions.checkNotNull(clusterName, "elasticsearch type of clusterName is required"); - Arrays.stream(StringUtils.split(id, ",")).forEach(number -> { - Preconditions.checkArgument(NumberUtils.isNumber(number), "id must be a numeric type"); - }); + if (!StringUtils.isEmpty(id)) { + Arrays.stream(StringUtils.split(id, ",")).forEach(number -> { + Preconditions.checkArgument(NumberUtils.isNumber(number), "id must be a numeric type"); + }); + } if (isAuthMesh()) { Preconditions.checkNotNull(userName, "elasticsearch type of userName is required"); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index cc650f670..2646c50e9 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -127,14 +127,6 @@ public void emitDataStream(DataStream> dataStream) { } } - public void setParallelism(int parallelism) { - this.parallelism = parallelism; - } - - public void setBulkFlushMaxActions(int bulkFlushMaxActions) { - this.bulkFlushMaxActions = bulkFlushMaxActions; - } - @Override public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { esTableInfo = (ElasticsearchTableInfo) targetTableInfo; @@ -143,8 +135,8 @@ public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { type = esTableInfo.getEsType(); columnTypes = esTableInfo.getFieldTypes(); esAddressList = Arrays.asList(esTableInfo.getAddress().split(",")); - String id = esTableInfo.getId(); + if (!StringUtils.isEmpty(id)) { idIndexList = Arrays.stream(StringUtils.split(id, ",")).map(Integer::valueOf).collect(Collectors.toList()); } diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index 5709d609c..3cc3dd9ff 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -128,12 +128,13 @@ public boolean check() { Preconditions.checkNotNull(address, "elasticsearch6 type of address is required"); Preconditions.checkNotNull(index, "elasticsearch6 type of index is required"); Preconditions.checkNotNull(esType, "elasticsearch6 type of type is required"); - Preconditions.checkNotNull(id, "elasticsearch6 type of id is required"); Preconditions.checkNotNull(clusterName, "elasticsearch6 type of clusterName is required"); - Arrays.stream(StringUtils.split(id, ",")).forEach(number ->{ - Preconditions.checkArgument(NumberUtils.isNumber(number),"id must be a numeric type"); - }); + if (!StringUtils.isEmpty(id)) { + Arrays.stream(StringUtils.split(id, ",")).forEach(number -> { + Preconditions.checkArgument(NumberUtils.isNumber(number), "id must be a numeric type"); + }); + } if (isAuthMesh()) { Preconditions.checkNotNull(userName, "elasticsearch6 type of userName is required"); From 0ad83f421d249b18771e59ccbd9928d608b8954b Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 4 Mar 2020 15:08:03 +0800 Subject: [PATCH 102/523] =?UTF-8?q?=E6=97=A5=E5=BF=97=E7=BA=A7=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 5 +++++ .../java/com/dtstack/flink/sql/option/Options.java | 11 +++++++++++ launcher/pom.xml | 7 +++++-- .../com/dtstack/flink/sql/launcher/LauncherMain.java | 11 +++++++++++ pom.xml | 1 + 5 files changed, 33 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 583958908..8b9e76f68 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,11 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack * 描述：yarn session 模式下指定的运行的一些参数，[可参考](https://ci.apache.org/projects/flink/flink-docs-release-1.8/ops/cli.html),目前只支持指定yid * 必选：否 * 默认值：false + +* **logLevel** + * 描述：设定日志级别 + * 必选：否 + * 默认值：info ## 2 结构 ### 2.1 源表插件 diff --git a/core/src/main/java/com/dtstack/flink/sql/option/Options.java b/core/src/main/java/com/dtstack/flink/sql/option/Options.java index 6adee2a01..9252c699f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/Options.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/Options.java @@ -68,6 +68,9 @@ public class Options { @OptionRequired(description = "plugin load mode, by classpath or shipfile") private String pluginLoadMode = EPluginLoadMode.CLASSPATH.name(); + + @OptionRequired(description = "log level") + private String logLevel = "info"; public String getMode() { return mode; @@ -172,4 +175,12 @@ public String getPluginLoadMode() { public void setPluginLoadMode(String pluginLoadMode) { this.pluginLoadMode = pluginLoadMode; } + + public String getLogLevel() { + return logLevel; + } + + public void setLogLevel(String logLevel) { + this.logLevel = logLevel; + } } diff --git a/launcher/pom.xml b/launcher/pom.xml index 2f8140fd5..8c1ce53d5 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -10,7 +10,6 @@ 4.0.0 sql.launcher - com.dtstack.flink @@ -35,7 +34,11 @@ fastjson 1.2.7 - + + com.aiweiergou + tools-logger + ${logger.tool.version} + diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index c098bc27e..54eea3657 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.launcher; +import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; import com.dtstack.flink.sql.constrant.ConfigConstrant; import com.google.common.collect.Lists; import com.alibaba.fastjson.JSON; @@ -83,6 +84,8 @@ public static void main(String[] args) throws Exception { confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); + setLogLevel(launcherOptions.getLogLevel()); + if(mode.equals(ClusterMode.local.name())) { String[] localArgs = argList.toArray(new String[argList.size()]); Main.main(localArgs); @@ -146,4 +149,12 @@ private static String[] parseJson(String[] args) { String[] array = list.toArray(new String[list.size()]); return array; } + + private static void setLogLevel(String logLevel){ + if(org.apache.commons.lang3.StringUtils.isBlank(logLevel)){ + return; + } + ChangeLogLevelProcess logLevelProcess = new ChangeLogLevelProcess(); + logLevelProcess.process(logLevel); + } } diff --git a/pom.xml b/pom.xml index 20ad02810..34932dd5b 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ UTF-8 1.8.1 + 1.0.0-SNAPSHOT From 13efd08d605784d740eaab286f9fcf5200a110d2 Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Wed, 4 Mar 2020 16:05:18 +0800 Subject: [PATCH 103/523] =?UTF-8?q?CustomerFlinkPartition=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E5=BA=8F=E5=88=97=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/format/SerializationMetricWrapper.java | 2 ++ .../flink/sql/sink/kafka/CustomerFlinkPartition.java | 2 -- .../com/dtstack/flink/sql/sink/kafka/KafkaProducer.java | 7 +++++++ .../com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java | 7 +++++++ .../com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java | 7 +++++++ .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 4 ++-- .../com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java | 7 +++++++ 7 files changed, 32 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java index 5f547ac64..3a5af18b1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java @@ -36,6 +36,8 @@ */ public class SerializationMetricWrapper implements SerializationSchema { + private static final long serialVersionUID = 1L; + private SerializationSchema serializationSchema; private transient RuntimeContext runtimeContext; diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index fb2252cdc..e212d1f57 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -1,8 +1,6 @@ package com.dtstack.flink.sql.sink.kafka; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; - import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.util.Preconditions; diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java index f818ba9ca..71a9cc386 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java @@ -24,6 +24,8 @@ import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.types.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Optional; import java.util.Properties; @@ -37,6 +39,10 @@ */ public class KafkaProducer extends FlinkKafkaProducer { + private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer.class); + + private static final long serialVersionUID = 1L; + private SerializationMetricWrapper serializationMetricWrapper; public KafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] parititonKeys) { @@ -46,6 +52,7 @@ public KafkaProducer(String topicId, SerializationSchema serializationSchem @Override public void open(Configuration configuration) throws Exception { + LOG.warn("---open KafkaProducer--"); RuntimeContext runtimeContext = getRuntimeContext(); serializationMetricWrapper.setRuntimeContext(runtimeContext); serializationMetricWrapper.initMetric(); diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java index 7b569d98d..c815e134a 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java @@ -24,6 +24,8 @@ import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.types.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Optional; import java.util.Properties; @@ -37,6 +39,10 @@ */ public class KafkaProducer09 extends FlinkKafkaProducer09 { + private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer09.class); + + private static final long serialVersionUID = 1L; + private SerializationMetricWrapper serializationMetricWrapper; public KafkaProducer09(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner,String[] partitionKeys) { @@ -46,6 +52,7 @@ public KafkaProducer09(String topicId, SerializationSchema serializationSch @Override public void open(Configuration configuration) { + LOG.info("----KafkaProducer09 open ---"); RuntimeContext runtimeContext = getRuntimeContext(); serializationMetricWrapper.setRuntimeContext(runtimeContext); serializationMetricWrapper.initMetric(); diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java index 20d831d8b..3cdc45dec 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java @@ -24,6 +24,8 @@ import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.types.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Optional; import java.util.Properties; @@ -37,6 +39,10 @@ */ public class KafkaProducer010 extends FlinkKafkaProducer010 { + private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer010.class); + + private static final long serialVersionUID = 1L; + private SerializationMetricWrapper serializationMetricWrapper; public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { @@ -46,6 +52,7 @@ public KafkaProducer010(String topicId, SerializationSchema serializationSc @Override public void open(Configuration configuration) { + LOG.info("----open KafkaProducer010 --"); RuntimeContext runtimeContext = getRuntimeContext(); serializationMetricWrapper.setRuntimeContext(runtimeContext); serializationMetricWrapper.initMetric(); diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index fd39a8af6..42f1045bf 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -108,8 +108,8 @@ public TypeInformation getRecordType() { @Override public void emitDataStream(DataStream> dataStream) { - RichSinkFunction kafkaProducer010 = new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, - Optional.of(new CustomerFlinkPartition()), partitionKeys); + RichSinkFunction kafkaProducer010 = new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, + Optional.of(new CustomerFlinkPartition<>()), partitionKeys); DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) .map((Tuple2 record) -> record.f1) diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java index 52385f89a..7880dd419 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java @@ -25,6 +25,8 @@ import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.types.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Optional; import java.util.Properties; @@ -38,6 +40,10 @@ */ public class KafkaProducer011 extends FlinkKafkaProducer011 { + private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer011.class); + + private static final long serialVersionUID = 1L; + private SerializationMetricWrapper serializationMetricWrapper; public KafkaProducer011(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { @@ -47,6 +53,7 @@ public KafkaProducer011(String topicId, SerializationSchema serializationSc @Override public void open(Configuration configuration) throws Exception { + LOG.info("--KafkaProducer011 open--"); RuntimeContext runtimeContext = getRuntimeContext(); serializationMetricWrapper.setRuntimeContext(runtimeContext); serializationMetricWrapper.initMetric(); From 126a29564b43c6fafca7b0236e1e83228e42cccc Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 4 Mar 2020 16:14:19 +0800 Subject: [PATCH 104/523] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../environment/StreamEnvConfigManager.java | 18 ++++++----- .../flink/sql/option/OptionParser.java | 4 +-- .../flink/sql/parser/CreateFuncParser.java | 8 ++--- .../flink/sql/side/SidePredicatesParser.java | 2 ++ .../dtstack/flink/sql/side/SideSQLParser.java | 4 +++ .../dtstack/flink/sql/side/SideSqlExec.java | 22 +++++++------ .../flink/sql/table/AbsSideTableParser.java | 2 +- .../com/dtstack/flink/sql/util/ClassUtil.java | 3 +- .../com/dtstack/flink/sql/util/DateUtil.java | 32 +++++++++---------- .../com/dtstack/flink/sql/util/JDBCUtils.java | 4 +-- 10 files changed, 56 insertions(+), 43 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index ab74edabc..b453c9414 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -258,6 +258,8 @@ private static Optional createStateBackend(String backendType, Str checkpointDataUriEmptyCheck(checkpointDataUri, backendType); stateBackend = new RocksDBStateBackend(checkpointDataUri, BooleanUtils.toBoolean(backendIncremental)); break; + default: + break; } return stateBackend == null ? Optional.empty() : Optional.of(stateBackend); } @@ -317,14 +319,14 @@ private static void verityTtl(String ttlMintimeStr, String ttlMaxtimeStr) { * @return */ private static Long getTtlTime(Integer timeNumber, String timeUnit) { - if (timeUnit.equalsIgnoreCase("d")) { - return timeNumber * 1000l * 60 * 60 * 24; - } else if (timeUnit.equalsIgnoreCase("h")) { - return timeNumber * 1000l * 60 * 60; - } else if (timeUnit.equalsIgnoreCase("m")) { - return timeNumber * 1000l * 60; - } else if (timeUnit.equalsIgnoreCase("s")) { - return timeNumber * 1000l; + if ("d".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L * 60 * 60 * 24; + } else if ("h".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L * 60 * 60; + } else if ("m".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L * 60; + } else if ("s".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L; } else { throw new RuntimeException("not support " + timeNumber + timeUnit); } diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java index 43f599d14..0e8bda1fe 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java @@ -114,7 +114,7 @@ public List getProgramExeArgList() throws Exception { } public static void main(String[] args) throws Exception { - OptionParser OptionParser = new OptionParser(args); - System.out.println(OptionParser.getOptions()); + OptionParser optionParser = new OptionParser(args); + System.out.println(optionParser.getOptions()); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java index 670d98a7e..fc6a7e16c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java @@ -32,18 +32,18 @@ public class CreateFuncParser implements IParser { - private static final String funcPatternStr = "(?i)\\s*create\\s+(scala|table|aggregate)\\s+function\\s+(\\S+)\\s+WITH\\s+(\\S+)"; + private static final String FUNC_PATTERN_STR = "(?i)\\s*create\\s+(scala|table|aggregate)\\s+function\\s+(\\S+)\\s+WITH\\s+(\\S+)"; - private static final Pattern funcPattern = Pattern.compile(funcPatternStr); + private static final Pattern FUNC_PATTERN = Pattern.compile(FUNC_PATTERN_STR); @Override public boolean verify(String sql) { - return funcPattern.matcher(sql).find(); + return FUNC_PATTERN.matcher(sql).find(); } @Override public void parseSql(String sql, SqlTree sqlTree) { - Matcher matcher = funcPattern.matcher(sql); + Matcher matcher = FUNC_PATTERN.matcher(sql); if(matcher.find()){ String type = matcher.group(1); String funcName = matcher.group(2); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java index 50103a9f5..64f9e646b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java @@ -101,6 +101,8 @@ private void parseSql(SqlNode sqlNode, Map sideTableMap, parseSql(unionLeft, sideTableMap, tabMapping); parseSql(unionRight, sideTableMap, tabMapping); break; + default: + break; } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index c12bc981e..b7d886566 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -131,6 +131,8 @@ private void checkAndReplaceMultiJoin(SqlNode sqlNode, Set sideTableSet) checkAndReplaceMultiJoin(unionLeft, sideTableSet); checkAndReplaceMultiJoin(unionRight, sideTableSet); break; + default: + break; } } @@ -204,6 +206,8 @@ private Object parseSql(SqlNode sqlNode, Set sideTableSet, Queue case ORDER_BY: SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode; parseSql(sqlOrderBy.query, sideTableSet, queueInfo, parentWhere, parentSelectList); + default: + break; } return ""; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index d54c87c17..1689949ca 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -90,7 +90,7 @@ public class SideSqlExec { private String tmpFields = null; - private SideSQLParser sideSQLParser = new SideSQLParser(); + private SideSQLParser sideSqlParser = new SideSQLParser(); private SidePredicatesParser sidePredicatesParser = new SidePredicatesParser(); private Map localTableCache = Maps.newHashMap(); @@ -109,8 +109,8 @@ public void exec(String sql, Map sideTableMap, StreamTabl LOG.error("fill predicates for sideTable fail ", e); } - sideSQLParser.setLocalTableCache(localTableCache); - Queue exeQueue = sideSQLParser.getExeQueue(sql, sideTableMap.keySet()); + sideSqlParser.setLocalTableCache(localTableCache); + Queue exeQueue = sideSqlParser.getExeQueue(sql, sideTableMap.keySet()); Object pollObj = null; //need clean @@ -140,7 +140,7 @@ public void exec(String sql, Map sideTableMap, StreamTabl LOG.info("exec sql: " + pollSqlNode.toString()); } }else if(pollSqlNode.getKind() == AS){ - AliasInfo aliasInfo = parseASNode(pollSqlNode); + AliasInfo aliasInfo = parseAsNode(pollSqlNode); Table table = tableEnv.sqlQuery(aliasInfo.getName()); tableEnv.registerTable(aliasInfo.getAlias(), table); localTableCache.put(aliasInfo.getAlias(), table); @@ -151,9 +151,9 @@ public void exec(String sql, Map sideTableMap, StreamTabl } } else if (pollSqlNode.getKind() == WITH_ITEM) { SqlWithItem sqlWithItem = (SqlWithItem) pollSqlNode; - String TableAlias = sqlWithItem.name.toString(); + String tableAlias = sqlWithItem.name.toString(); Table table = tableEnv.sqlQuery(sqlWithItem.query.toString()); - tableEnv.registerTable(TableAlias, table); + tableEnv.registerTable(tableAlias, table); } }else if (pollObj instanceof JoinInfo){ @@ -253,11 +253,13 @@ private void addAliasForFieldNode(SqlNode pollSqlNode, List fieldList, H } } break; + default: + break; } } - public AliasInfo parseASNode(SqlNode sqlNode) throws SqlParseException { + public AliasInfo parseAsNode(SqlNode sqlNode) throws SqlParseException { SqlKind sqlKind = sqlNode.getKind(); if(sqlKind != AS){ throw new RuntimeException(sqlNode + " is not 'as' operator"); @@ -487,6 +489,8 @@ public SqlNode filterNodeWithTargetName(SqlNode sqlNode, String targetTableName) }else{ return null; } + default: + break; } return null; @@ -709,7 +713,7 @@ public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, } localTableCache.putAll(tableCache); - Queue exeQueue = sideSQLParser.getExeQueue(result.getExecSql(), sideTableMap.keySet()); + Queue exeQueue = sideSqlParser.getExeQueue(result.getExecSql(), sideTableMap.keySet()); Object pollObj = null; //need clean @@ -759,7 +763,7 @@ public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, } protected void dealAsSourceTable(StreamTableEnvironment tableEnv, SqlNode pollSqlNode) throws SqlParseException { - AliasInfo aliasInfo = parseASNode(pollSqlNode); + AliasInfo aliasInfo = parseAsNode(pollSqlNode); if (localTableCache.containsKey(aliasInfo.getName())) { return; } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java index 520ca2138..49c5629bc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java @@ -87,7 +87,7 @@ protected void parseCacheProp(SideTableInfo sideTableInfo, Map p if(props.containsKey(SideTableInfo.CACHE_MODE_KEY.toLowerCase())){ String cachemode = MathUtil.getString(props.get(SideTableInfo.CACHE_MODE_KEY.toLowerCase())); - if(!cachemode.equalsIgnoreCase("ordered") && !cachemode.equalsIgnoreCase("unordered")){ + if(!"ordered".equalsIgnoreCase(cachemode) && !"unordered".equalsIgnoreCase(cachemode)){ throw new RuntimeException("cachemode must ordered or unordered!"); } sideTableInfo.setCacheMode(cachemode.toLowerCase()); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index feebbe3f1..4bb4ff0d8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -86,7 +86,8 @@ public static Class stringConvertClass(String str) { case "decimal": case "decimalunsigned": return BigDecimal.class; - + default: + break; } throw new RuntimeException("不支持 " + str + " 类型"); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index 5bfa2f203..da5a42b57 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -116,9 +116,9 @@ public static long getTodayStart(long day) { * @return */ public static long getTodayStart(long day,String scope) { - if(scope.equals("MS")){ + if("MS".equals(scope)){ return getTodayStart(day)*1000; - }else if(scope.equals("S")){ + }else if("S".equals(scope)){ return getTodayStart(day); }else{ return getTodayStart(day); @@ -154,9 +154,9 @@ public static long getNextDayStart(long day) { * @return */ public static long getNextDayStart(long day,String scope) { - if(scope.equals("MS")){ + if("MS".equals(scope)){ return getNextDayStart(day)*1000; - }else if(scope.equals("S")){ + }else if("S".equals(scope)){ return getNextDayStart(day); }else{ return getNextDayStart(day); @@ -335,7 +335,7 @@ public static String get30DaysLaterByString(String day, String inFormat, String * @return String * @throws ParseException */ - public static String getDateStrTOFormat(String day, String inFormat, String outFormat) throws ParseException { + public static String getDateStrToFormat(String day, String inFormat, String outFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -344,7 +344,7 @@ public static String getDateStrTOFormat(String day, String inFormat, String outF return dayBefore; } - public static long getDateMillTOFormat(String day, String inFormat) throws ParseException { + public static long getDateMillToFormat(String day, String inFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -470,11 +470,11 @@ public static long getMillByDay(int severalDays,String condition) { if(condition==null){ return getMillToDay(cal,dateT); } - if(condition.equals("-")){ + if("-".equals(condition)){ dateT = (cal.get(Calendar.DATE) - severalDays); return getMillToDay(cal,dateT); } - if(condition.equals("+")){ + if("+".equals(condition)){ dateT = (cal.get(Calendar.DATE) + severalDays); return getMillToDay(cal,dateT); } @@ -490,11 +490,11 @@ public static long getStampByDay(int severalDays,String condition) { if(condition==null){ return getStampToDay(cal,dateT); } - if(condition.equals("-")){ + if("-".equals(condition)){ dateT = (cal.get(Calendar.DATE) - severalDays); return getStampToDay(cal,dateT); } - if(condition.equals("+")){ + if("+".equals(condition)){ dateT = (cal.get(Calendar.DATE) + severalDays); return getStampToDay(cal,dateT); } @@ -575,8 +575,8 @@ public static String getDate(Date date, String format) { */ public static long stringToLong(String day, String format) throws ParseException { SimpleDateFormat dateFormat = new SimpleDateFormat(format); - long Date = dateFormat.parse(day).getTime(); - return Date; + long date = dateFormat.parse(day).getTime(); + return date; } /** @@ -588,8 +588,8 @@ public static long stringToLong(String day, String format) throws ParseException public static Date stringToDate(String day, String format) { try { SimpleDateFormat dateFormat = new SimpleDateFormat(format); - Date Date = dateFormat.parse(day); - return Date; + Date date = dateFormat.parse(day); + return date; } catch (ParseException e) { return new Date(); } @@ -608,8 +608,8 @@ public static String longToString(long day, String format) throws ParseException day=day*1000; } SimpleDateFormat dateFormat = new SimpleDateFormat(format); - String Date = dateFormat.format(day); - return Date; + String date = dateFormat.format(day); + return date; } /** diff --git a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java index fde2f166e..3d66d8a6d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java @@ -28,10 +28,10 @@ public class JDBCUtils { private static final Logger LOG = LoggerFactory.getLogger(ClassUtil.class); - public final static String lock_str = "jdbc_lock_str"; + public final static String LOCK_STR = "jdbc_lock_str"; public static void forName(String clazz, ClassLoader classLoader) { - synchronized (lock_str){ + synchronized (LOCK_STR){ try { Class.forName(clazz, true, classLoader); DriverManager.setLoginTimeout(10); From c40ac170b52ee22d239baddffbe40ec0907117f9 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 4 Mar 2020 16:24:01 +0800 Subject: [PATCH 105/523] =?UTF-8?q?=E6=97=A5=E5=BF=97=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/kafka/CustomerKeyedSerializationSchema.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index 09e7e0507..f81f7b0fc 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -11,10 +11,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.concurrent.atomic.AtomicLong; + public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { private static final Logger LOG = LoggerFactory.getLogger(CustomerKeyedSerializationSchema.class); + private static final AtomicLong counter = new AtomicLong(0L); + private static final long serialVersionUID = 1L; private final SerializationMetricWrapper serializationMetricWrapper; private String[] partitionKeys; @@ -57,7 +61,9 @@ private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationS } return sb.toString().getBytes(); } catch (Exception e){ - LOG.error("serializeJsonKey error", e); + if(counter.getAndIncrement() % 1000 == 0){ + LOG.error("serializeJsonKey error", e); + } } return null; } From a41d72ed39d870260e2b79b3979e852d0f9f348a Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 4 Mar 2020 16:27:09 +0800 Subject: [PATCH 106/523] =?UTF-8?q?=E5=90=8D=E7=A7=B0=E5=A4=A7=E5=86=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/kafka/CustomerKeyedSerializationSchema.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index f81f7b0fc..498766564 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -17,7 +17,7 @@ public class CustomerKeyedSerializationSchema implements KeyedSerializationSchem private static final Logger LOG = LoggerFactory.getLogger(CustomerKeyedSerializationSchema.class); - private static final AtomicLong counter = new AtomicLong(0L); + private static final AtomicLong COUNTER = new AtomicLong(0L); private static final long serialVersionUID = 1L; private final SerializationMetricWrapper serializationMetricWrapper; @@ -61,7 +61,7 @@ private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationS } return sb.toString().getBytes(); } catch (Exception e){ - if(counter.getAndIncrement() % 1000 == 0){ + if(COUNTER.getAndIncrement() % 1000 == 0){ LOG.error("serializeJsonKey error", e); } } From e0f289be82541f7f630e91f9506f8be08ef63ae4 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 4 Mar 2020 16:47:04 +0800 Subject: [PATCH 107/523] codeview fix --- .../flink/sql/sink/console/table/TablePrintUtil.java | 12 +++++++++--- .../flink/sql/launcher/ClusterClientFactory.java | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java index 8813da619..39abb81b4 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java @@ -58,7 +58,9 @@ public static TablePrintUtil build(String[][] data) { public static TablePrintUtil build(List data) { TablePrintUtil self = new TablePrintUtil(); self.data = new ArrayList<>(); - if (data.size() <= 0) throw new RuntimeException("数据源至少得有一行吧"); + if (data.size() <= 0) { + throw new RuntimeException("数据源至少得有一行吧"); + } Object obj = data.get(0); @@ -70,7 +72,9 @@ public static TablePrintUtil build(List data) { int length = ((List) obj).size(); for (Object item : data) { List col = (List) item; - if (col.size() != length) throw new RuntimeException("数据源每列长度必须一致"); + if (col.size() != length) { + throw new RuntimeException("数据源每列长度必须一致"); + } self.data.add(col.toArray(new String[length])); } } else { @@ -163,7 +167,9 @@ private int[] getColLengths() { if (equilong) {//如果等宽表格 int max = 0; for (int len : result) { - if (len > max) max = len; + if (len > max) { + max = len; + } } for (int i = 0; i < result.length; i++) { result[i] = max; diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 14cd847b5..7f4ded520 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -89,7 +89,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { if (StringUtils.isNotBlank(yarnConfDir)) { try { - config.setString(ConfigConstants.PATH_HADOOP_CONFIG, yarnConfDir); + config.setString("fs.hdfs.hadoopconf", yarnConfDir); FileSystem.initialize(config); YarnConfiguration yarnConf = YarnConfLoader.getYarnConf(yarnConfDir); @@ -166,7 +166,7 @@ private static ApplicationId getYarnClusterApplicationId(YarnClient yarnClient) private static ApplicationId toApplicationId(String appIdStr) { Iterator it = StringHelper._split(appIdStr).iterator(); - if (!(it.next()).equals("application")) { + if (!"application".equals(it.next())) { throw new IllegalArgumentException("Invalid ApplicationId prefix: " + appIdStr + ". The valid ApplicationId should start with prefix " + "application"); } else { try { From d636b1743d40989472b23341634070e78d8a7735 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 4 Mar 2020 17:04:11 +0800 Subject: [PATCH 108/523] allow es is is null --- ...asticsearch6Side => elasticsearch6Side.md} | 0 docs/elasticsearch6Sink.md | 2 +- .../sink/elasticsearch/CustomerSinkFunc.java | 27 ++++--- .../table/ElasticsearchTableInfo.java | 11 ++- .../sink/elasticsearch/CustomerSinkFunc.java | 38 +++++----- .../sink/elasticsearch/ElasticsearchSink.java | 70 ++++++------------- .../table/ElasticsearchTableInfo.java | 11 ++- 7 files changed, 76 insertions(+), 83 deletions(-) rename docs/{elasticsearch6Side => elasticsearch6Side.md} (100%) diff --git a/docs/elasticsearch6Side b/docs/elasticsearch6Side.md similarity index 100% rename from docs/elasticsearch6Side rename to docs/elasticsearch6Side.md diff --git a/docs/elasticsearch6Sink.md b/docs/elasticsearch6Sink.md index 47514b20c..59a4ab53a 100644 --- a/docs/elasticsearch6Sink.md +++ b/docs/elasticsearch6Sink.md @@ -35,7 +35,7 @@ CREATE TABLE tableName( |cluster | ES 集群名称 |是|| |index | 选择的ES上的index名称|是|| |esType | 选择ES上的type名称|是|| -|id | 生成id的规则(当前是根据指定的字段pos获取字段信息,拼接生成id;|是|| +|id | 生成id的规则(当前是根据指定的字段pos获取字段信息,拼接生成id;|否|| | |若id为空字符串或索引都超出范围，则随机生成id值)||| |authMesh | 是否进行用户名密码认证 | 否 | false| |userName | 用户名 | 否，authMesh='true'时为必填 || diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index a95d477de..1f3efb8d7 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Reason: @@ -47,6 +48,8 @@ public class CustomerSinkFunc implements ElasticsearchSinkFunction { private final Logger logger = LoggerFactory.getLogger(CustomerSinkFunc.class); + private static final String ID_VALUE_SPLIT = "_"; + private String index; private String type; @@ -93,15 +96,11 @@ public void setOutRecords(Counter outRecords) { } private IndexRequest createIndexRequest(Row element) { - - List idFieldList = new ArrayList<>(); - for(int index : idFieldIndexList){ - if(index >= element.getArity()){ - continue; - } - - idFieldList.add(element.getField(index).toString()); - } + // index start at 1, + String idFieldStr = idFieldIndexList.stream() + .filter(index -> index > 0 && index <= element.getArity()) + .map(index -> element.getField(index - 1).toString()) + .collect(Collectors.joining(ID_VALUE_SPLIT)); Map dataMap = EsUtil.rowToJsonMap(element,fieldNames,fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); @@ -109,11 +108,17 @@ private IndexRequest createIndexRequest(Row element) { dataMap.put(fieldNames.get(i), element.getField(i)); } - String id = StringUtils.join(idFieldList, sp); + if (StringUtils.isEmpty(idFieldStr)) { + return Requests.indexRequest() + .index(index) + .type(type) + .source(dataMap); + } + return Requests.indexRequest() .index(index) .type(type) - .id(id) + .id(idFieldStr) .source(dataMap); } } diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index ac1d712ce..9681742c2 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -23,6 +23,10 @@ import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; + +import java.util.Arrays; /** * @date 2018/09/12 @@ -128,9 +132,14 @@ public boolean check() { Preconditions.checkNotNull(address, "elasticsearch type of address is required"); Preconditions.checkNotNull(index, "elasticsearch type of index is required"); Preconditions.checkNotNull(esType, "elasticsearch type of type is required"); - Preconditions.checkNotNull(id, "elasticsearch type of id is required"); Preconditions.checkNotNull(clusterName, "elasticsearch type of clusterName is required"); + if (!StringUtils.isEmpty(id)) { + Arrays.stream(StringUtils.split(id, ",")).forEach(number -> { + Preconditions.checkArgument(NumberUtils.isNumber(number), "id must be a numeric type"); + }); + } + if (isAuthMesh()) { Preconditions.checkNotNull(userName, "elasticsearch type of userName is required"); Preconditions.checkNotNull(password, "elasticsearch type of password is required"); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index d6d207c9e..b1c8a5e97 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -31,9 +31,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * @author yinxi @@ -42,6 +42,8 @@ public class CustomerSinkFunc implements ElasticsearchSinkFunction { private final Logger logger = LoggerFactory.getLogger(CustomerSinkFunc.class); + /** 用作ID的属性值连接符号 */ + private static final String ID_VALUE_SPLIT = "_"; private String index; @@ -57,10 +59,7 @@ public class CustomerSinkFunc implements ElasticsearchSinkFunction { private transient Counter outDirtyRecords; - /** 默认分隔符为'_' */ - private char sp = '_'; - - public CustomerSinkFunc(String index, String type, List fieldNames, List fieldTypes, List idFieldIndexes){ + public CustomerSinkFunc(String index, String type, List fieldNames, List fieldTypes, List idFieldIndexes) { this.index = index; this.type = type; this.fieldNames = fieldNames; @@ -96,31 +95,30 @@ public void setOutDirtyRecords(Counter outDirtyRecords) { } private IndexRequest createIndexRequest(Row element) { + // index start at 1, + String idFieldStr = idFieldIndexList.stream() + .filter(index -> index > 0 && index <= element.getArity()) + .map(index -> element.getField(index - 1).toString()) + .collect(Collectors.joining(ID_VALUE_SPLIT)); - List idFieldList = new ArrayList<>(); - for(int index : idFieldIndexList){ - if(index >= element.getArity()){ - continue; - } - - idFieldList.add(element.getField(index).toString()); - } - - Map dataMap = Es6Util.rowToJsonMap(element,fieldNames,fieldTypes); + Map dataMap = Es6Util.rowToJsonMap(element, fieldNames, fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); - for(int i=0; i, IStreamSinkGener { - private final Logger logger = LoggerFactory.getLogger(ElasticsearchSink.class); + private final int ES_DEFAULT_PORT = 9200; + private final String ES_DEFAULT_SCHEMA = "http"; private String clusterName; @@ -103,32 +101,20 @@ public TypeInformation[] getFieldTypes() { private RichSinkFunction createEsSinkFunction() { - - Map userConfig = Maps.newHashMap(); userConfig.put("cluster.name", clusterName); // This instructs the sink to emit after every element, otherwise they would be buffered userConfig.put(org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "" + bulkFlushMaxActions); - List transports = new ArrayList<>(); - - for (String address : esAddressList) { - String[] infoArray = address.split(":"); - int port = 9200; - String host = infoArray[0]; - if (infoArray.length > 1) { - port = Integer.valueOf(infoArray[1].trim()); - } - - try { - transports.add(new HttpHost(host.trim(), port, "http")); - } catch (Exception e) { - logger.error("", e); - throw new RuntimeException(e); - } - } - CustomerSinkFunc customerSinkFunc = new CustomerSinkFunc(index, type, Arrays.asList(fieldNames), Arrays.asList(columnTypes), idIndexList); + List transports = esAddressList.stream() + .map(address -> address.split(":")) + .map(addressArray -> { + String host = addressArray[0].trim(); + int port = addressArray.length > 1 ? Integer.valueOf(addressArray[1].trim()) : ES_DEFAULT_PORT; + return new HttpHost(host.trim(), port, ES_DEFAULT_SCHEMA); + }).collect(Collectors.toList()); + CustomerSinkFunc customerSinkFunc = new CustomerSinkFunc(index, type, Arrays.asList(fieldNames), Arrays.asList(columnTypes), idIndexList); return new MetricElasticsearch6Sink(userConfig, transports, customerSinkFunc, esTableInfo); } @@ -141,33 +127,19 @@ public void emitDataStream(DataStream> dataStream) { } } - public void setParallelism(int parallelism) { - this.parallelism = parallelism; - } - - public void setBulkFlushMaxActions(int bulkFlushMaxActions) { - this.bulkFlushMaxActions = bulkFlushMaxActions; - } - @Override public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { - ElasticsearchTableInfo elasticsearchTableInfo = (ElasticsearchTableInfo) targetTableInfo; - esTableInfo = elasticsearchTableInfo; - clusterName = elasticsearchTableInfo.getClusterName(); - String address = elasticsearchTableInfo.getAddress(); - String[] addr = address.split(","); - esAddressList = Arrays.asList(addr); - index = elasticsearchTableInfo.getIndex(); - type = elasticsearchTableInfo.getEsType(); - String id = elasticsearchTableInfo.getId(); - String[] idField = StringUtils.split(id, ","); - idIndexList = new ArrayList<>(); - - for (int i = 0; i < idField.length; ++i) { - idIndexList.add(Integer.valueOf(idField[i])); + esTableInfo = (ElasticsearchTableInfo) targetTableInfo; + clusterName = esTableInfo.getClusterName(); + index = esTableInfo.getIndex(); + type = esTableInfo.getEsType(); + columnTypes = esTableInfo.getFieldTypes(); + esAddressList = Arrays.asList(esTableInfo.getAddress().split(",")); + String id = esTableInfo.getId(); + + if (!StringUtils.isEmpty(id)) { + idIndexList = Arrays.stream(StringUtils.split(id, ",")).map(Integer::valueOf).collect(Collectors.toList()); } - - columnTypes = elasticsearchTableInfo.getFieldTypes(); return this; } } diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index b0156b660..3cc3dd9ff 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -20,6 +20,10 @@ import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; + +import java.util.Arrays; /** * @author yinxi @@ -124,9 +128,14 @@ public boolean check() { Preconditions.checkNotNull(address, "elasticsearch6 type of address is required"); Preconditions.checkNotNull(index, "elasticsearch6 type of index is required"); Preconditions.checkNotNull(esType, "elasticsearch6 type of type is required"); - Preconditions.checkNotNull(id, "elasticsearch6 type of id is required"); Preconditions.checkNotNull(clusterName, "elasticsearch6 type of clusterName is required"); + if (!StringUtils.isEmpty(id)) { + Arrays.stream(StringUtils.split(id, ",")).forEach(number -> { + Preconditions.checkArgument(NumberUtils.isNumber(number), "id must be a numeric type"); + }); + } + if (isAuthMesh()) { Preconditions.checkNotNull(userName, "elasticsearch6 type of userName is required"); Preconditions.checkNotNull(password, "elasticsearch6 type of password is required"); From e51f1cfdf5ccf41410998242c8f78a22d0875028 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 4 Mar 2020 17:04:12 +0800 Subject: [PATCH 109/523] codereview --- .../flink/sql/side/cassandra/table/CassandraSideParser.java | 3 +++ .../flink/sql/side/clickhouse/ClickhouseAllReqRow.java | 6 +++--- .../flink/sql/sink/console/table/TablePrintUtil.java | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java index 62dd753b9..d2b93563b 100644 --- a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java +++ b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java @@ -99,6 +99,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map getColList(Object obj) { Method[] methods = obj.getClass().getMethods(); for (Method m : methods) { StringBuilder getMethodName = new StringBuilder(m.getName()); - if (getMethodName.substring(0, 3).equals("get") && !m.getName().equals("getClass")) { + if ("get".equals(getMethodName.substring(0, 3)) && !"getClass".equals(m.getName())) { Col col = new Col(); col.getMethodName = getMethodName.toString(); char first = Character.toLowerCase(getMethodName.delete(0, 3).charAt(0)); From 841c1fe931d0e1bdb838b4eea9034123a5b61e81 Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Wed, 4 Mar 2020 18:12:19 +0800 Subject: [PATCH 110/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E5=86=B2=E7=AA=81=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/exec/ExecuteProcessHelper.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index d2c2926ef..5cd1248e2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -188,17 +188,23 @@ public static List getExternalJarUrls(String addJarListStr) throws java.io. return jarUrlList; } - public static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironment tableEnv, SqlTree sqlTree, Map sideTableMap, Map registerTableCache, StreamQueryConfig queryConfig) throws Exception { + private static void sqlTranslation(String localSqlPluginPath, + StreamTableEnvironment tableEnv, + SqlTree sqlTree,Map sideTableMap, + Map registerTableCache, + StreamQueryConfig queryConfig) throws Exception { + SideSqlExec sideSqlExec = new SideSqlExec(); sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { - sideSqlExec.registerTmpTable(result, sideTableMap, tableEnv, registerTableCache); + sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, result); } for (InsertSqlParser.SqlParseResult result : sqlTree.getExecSqlList()) { if (LOG.isInfoEnabled()) { LOG.info("exe-sql:\n" + result.getExecSql()); } + boolean isSide = false; for (String tableName : result.getTargetTableList()) { if (sqlTree.getTmpTableMap().containsKey(tableName)) { @@ -208,7 +214,7 @@ public static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironm SqlNode sqlNode = org.apache.calcite.sql.parser.SqlParser.create(realSql, CalciteConfig.MYSQL_LEX_CONFIG).parseStmt(); String tmpSql = ((SqlInsert) sqlNode).getSource().toString(); tmp.setExecSql(tmpSql); - sideSqlExec.registerTmpTable(tmp, sideTableMap, tableEnv, registerTableCache); + sideSqlExec.exec(tmp.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, tmp); } else { for (String sourceTable : result.getSourceTableList()) { if (sideTableMap.containsKey(sourceTable)) { @@ -218,10 +224,14 @@ public static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironm } if (isSide) { //sql-dimensional table contains the dimension table of execution - sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig); + sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null); } else { + System.out.println("----------exec sql without dimension join-----------"); + System.out.println("----------real sql exec is--------------------------"); + System.out.println(result.getExecSql()); FlinkSQLExec.sqlUpdate(tableEnv, result.getExecSql(), queryConfig); if (LOG.isInfoEnabled()) { + System.out.println(); LOG.info("exec sql: " + result.getExecSql()); } } From aa1b250485b8c1a3b0e7768df199eff1b7aeccba Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 4 Mar 2020 18:16:03 +0800 Subject: [PATCH 111/523] =?UTF-8?q?hbase=20=E8=A7=A3=E6=9E=90PRIMARY=20KEY?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/hbase/ReplaceInfo.java | 15 ++++- .../flink/sql/side/hbase/RowKeyBuilder.java | 59 +++++++++---------- .../sql/side/hbase/enums/EReplaceType.java | 1 + 3 files changed, 43 insertions(+), 32 deletions(-) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceInfo.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceInfo.java index 92d8f6d45..c2d36b5c1 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceInfo.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceInfo.java @@ -20,9 +20,10 @@ package com.dtstack.flink.sql.side.hbase; -import com.dtstack.flink.sql.side.hbase.enums.EReplaceType; - import java.io.Serializable; +import java.util.List; + +import com.dtstack.flink.sql.side.hbase.enums.EReplaceType; /** * Reason: @@ -39,6 +40,8 @@ public class ReplaceInfo implements Serializable { private EReplaceType type; private String param; + + private List subReplaceInfos; public ReplaceInfo(EReplaceType type){ this.type = type; @@ -59,4 +62,12 @@ public String getParam() { public void setParam(String param) { this.param = param; } + + public List getSubReplaceInfos() { + return subReplaceInfos; + } + + public void setSubReplaceInfos(List subReplaceInfos) { + this.subReplaceInfos = subReplaceInfos; + } } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java index 114b7fa6a..b8530f224 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java @@ -21,7 +21,9 @@ package com.dtstack.flink.sql.side.hbase; import com.dtstack.flink.sql.side.hbase.enums.EReplaceType; +import com.dtstack.flink.sql.util.MD5Utils; import com.google.common.collect.Lists; +import org.apache.commons.collections.CollectionUtils; import java.io.Serializable; import java.util.List; @@ -42,31 +44,10 @@ public class RowKeyBuilder implements Serializable{ private static Pattern Md5Operator = Pattern.compile("(?i)^md5\\(\\s*(.*)\\s*\\)$"); - private List> operatorChain = Lists.newArrayList(); + private List operatorChain = Lists.newArrayList(); public void init(String rowKeyTempl){ - - String[] strArr = splitIgnoreQuotaBrackets(rowKeyTempl, "\\+"); - - for(String infoAlias : strArr){ - infoAlias = infoAlias.trim(); - Matcher matcher = Md5Operator.matcher(infoAlias); - String fieldCols = null; - if(matcher.find()){ - fieldCols = matcher.group(1); - }else{ - fieldCols = infoAlias; - } - - String[] fieldArr = fieldCols.split("\\+"); - List fieldList = Lists.newArrayList(); - for(String oneField : fieldArr){ - ReplaceInfo replaceInfo = getReplaceInfo(oneField); - fieldList.add(replaceInfo); - } - - operatorChain.add(fieldList); - } + operatorChain.addAll(makeFormula(rowKeyTempl)); } /** @@ -75,18 +56,15 @@ public void init(String rowKeyTempl){ * @return */ public String getRowKey(Map refData){ - - StringBuilder sb = new StringBuilder(""); - for(List fieldList : operatorChain){ - sb.append(buildStr(fieldList, refData)); - } - - return sb.toString(); + return buildStr(operatorChain, refData); } private String buildStr(List fieldList, Map refData){ + if(CollectionUtils.isEmpty(fieldList)){ + return ""; + } StringBuffer sb = new StringBuffer(""); for(ReplaceInfo replaceInfo : fieldList){ @@ -95,6 +73,10 @@ private String buildStr(List fieldList, Map refData continue; } + if(replaceInfo.getType() == EReplaceType.FUNC){ + sb.append(MD5Utils.getMD5String(buildStr(replaceInfo.getSubReplaceInfos(), refData))); + continue; + } String replaceName = replaceInfo.getParam(); if(!refData.containsKey(replaceName)){ throw new RuntimeException(String.format("build rowKey with field %s which value not found.", replaceName)); @@ -137,4 +119,21 @@ public ReplaceInfo getReplaceInfo(String field){ return replaceInfo; } + private List makeFormula(String formula){ + if(formula == null || formula.length() <= 0){ + Lists.newArrayList(); + } + List result = Lists.newArrayList(); + for(String meta: splitIgnoreQuotaBrackets(formula, "\\+")){ + Matcher matcher = Md5Operator.matcher(meta.trim()); + if(matcher.find()){ + ReplaceInfo replaceInfo = new ReplaceInfo(EReplaceType.FUNC); + replaceInfo.setSubReplaceInfos(makeFormula(matcher.group(1))); + result.add(replaceInfo); + } else { + result.add(getReplaceInfo(meta)); + } + } + return result; + } } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java index 3768d7c5e..f4b039ef8 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java @@ -29,5 +29,6 @@ */ public enum EReplaceType { PARAM, + FUNC, CONSTANT; } From eaae4a193050e725c6cc427a305b9b17bbc2fbcf Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 4 Mar 2020 21:10:43 +0800 Subject: [PATCH 112/523] =?UTF-8?q?=E4=B8=8Etest=E5=88=86=E6=94=AF?= =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/exec/ExecuteProcessHelper.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index ed02e9687..11c85f132 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -187,12 +187,17 @@ public static List getExternalJarUrls(String addJarListStr) throws java.io. } return jarUrlList; } - - public static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironment tableEnv, SqlTree sqlTree, Map sideTableMap, Map registerTableCache, StreamQueryConfig queryConfig) throws Exception { - SideSqlExec sideSqlExec = new SideSqlExec(); + + private static void sqlTranslation(String localSqlPluginPath, + StreamTableEnvironment tableEnv, + SqlTree sqlTree,Map sideTableMap, + Map registerTableCache, + StreamQueryConfig queryConfig) throws Exception { + + SideSqlExec sideSqlExec = new SideSqlExec(); sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { - sideSqlExec.registerTmpTable(result, sideTableMap, tableEnv, registerTableCache); + sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, result); } for (InsertSqlParser.SqlParseResult result : sqlTree.getExecSqlList()) { @@ -208,7 +213,7 @@ public static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironm SqlNode sqlNode = org.apache.calcite.sql.parser.SqlParser.create(realSql, CalciteConfig.MYSQL_LEX_CONFIG).parseStmt(); String tmpSql = ((SqlInsert) sqlNode).getSource().toString(); tmp.setExecSql(tmpSql); - sideSqlExec.registerTmpTable(tmp, sideTableMap, tableEnv, registerTableCache); + sideSqlExec.exec(tmp.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, tmp); } else { for (String sourceTable : result.getSourceTableList()) { if (sideTableMap.containsKey(sourceTable)) { From cf5e1eefc3676c842b3951d05c47d09e24ca52b2 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 4 Mar 2020 21:11:39 +0800 Subject: [PATCH 113/523] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/exec/ExecuteProcessHelper.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 11c85f132..058ef6097 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -223,8 +223,11 @@ private static void sqlTranslation(String localSqlPluginPath, } if (isSide) { //sql-dimensional table contains the dimension table of execution - sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig); + sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null); } else { + System.out.println("----------exec sql without dimension join-----------"); + System.out.println("----------real sql exec is--------------------------"); + System.out.println(result.getExecSql()); FlinkSQLExec.sqlUpdate(tableEnv, result.getExecSql(), queryConfig); if (LOG.isInfoEnabled()) { LOG.info("exec sql: " + result.getExecSql()); From f56801c3180bef5347adced6a352dd6e190c7f2d Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 4 Mar 2020 22:41:53 +0800 Subject: [PATCH 114/523] code review --- .../side/cassandra/CassandraAllReqRow.java | 10 ++- .../side/cassandra/CassandraAllSideInfo.java | 10 +-- .../side/cassandra/CassandraAsyncReqRow.java | 9 ++- .../cassandra/CassandraAsyncSideInfo.java | 10 +-- .../cassandra/table/CassandraSideParser.java | 14 ++--- .../table/CassandraSideTableInfo.java | 4 +- .../sink/cassandra/CassandraOutputFormat.java | 4 +- .../sql/sink/cassandra/CassandraSink.java | 4 +- .../cassandra/table/CassandraSinkParser.java | 10 +-- .../cassandra/table/CassandraTableInfo.java | 4 +- .../side/clickhouse/ClickhouseAllReqRow.java | 4 +- .../clickhouse/ClickhouseAllSideInfo.java | 4 +- .../clickhouse/ClickhouseAsyncReqRow.java | 4 +- .../clickhouse/ClickhouseAsyncSideInfo.java | 4 +- .../table/ClickhouseSideParser.java | 6 +- .../table/ClickhouseSinkParser.java | 6 +- .../sql/sink/console/ConsoleOutputFormat.java | 4 +- .../flink/sql/sink/console/ConsoleSink.java | 4 +- .../sink/console/table/ConsoleSinkParser.java | 10 +-- .../sink/console/table/ConsoleTableInfo.java | 4 +- .../flink/sql/constrant/ConfigConstrant.java | 6 +- .../dtstack/flink/sql/enums/ColumnType.java | 63 +++++++++++++++++-- .../flink/sql/enums/ECacheContentType.java | 9 +++ .../dtstack/flink/sql/enums/ECacheType.java | 13 +++- .../flink/sql/enums/EDatabaseType.java | 9 +++ .../flink/sql/enums/EPluginLoadMode.java | 6 ++ .../flink/sql/enums/EStateBackend.java | 9 +++ .../environment/StreamEnvConfigManager.java | 6 +- .../flink/sql/exec/ExecuteProcessHelper.java | 36 +++++------ .../DtNestRowDeserializationSchema.java | 8 +-- .../flink/sql/option/OptionParser.java | 2 +- ...t.java => AbstractDtRichOutputFormat.java} | 2 +- .../dtstack/flink/sql/parser/SqlParser.java | 12 ++-- .../com/dtstack/flink/sql/parser/SqlTree.java | 8 +-- ...leInfo.java => AbstractSideTableInfo.java} | 4 +- .../{AllReqRow.java => BaseAllReqRow.java} | 8 +-- ...{AsyncReqRow.java => BaseAsyncReqRow.java} | 14 ++--- .../side/{SideInfo.java => BaseSideInfo.java} | 22 +++---- .../flink/sql/side/SidePredicatesParser.java | 7 +-- .../dtstack/flink/sql/side/SideSqlExec.java | 14 ++--- .../flink/sql/side/StreamSideFactory.java | 10 +-- ...sSideCache.java => AbstractSideCache.java} | 8 +-- .../flink/sql/side/cache/LRUSideCache.java | 6 +- .../sql/side/operator/SideAsyncOperator.java | 16 ++--- .../operator/SideWithAllCacheOperator.java | 18 +++--- .../flink/sql/sink/IStreamSinkGener.java | 4 +- .../flink/sql/sink/StreamSinkFactory.java | 12 ++-- .../flink/sql/source/IStreamSourceGener.java | 4 +- .../flink/sql/source/StreamSourceFactory.java | 12 ++-- ...rser.java => AbstractSideTableParser.java} | 38 +++++------ ...eParser.java => AbstractSourceParser.java} | 16 ++--- ...Info.java => AbstractSourceTableInfo.java} | 2 +- ...{TableInfo.java => AbstractTableInfo.java} | 2 +- ...rser.java => AbstractTableInfoParser.java} | 18 +++--- ...leParser.java => AbstractTableParser.java} | 16 ++--- ...Info.java => AbstractTargetTableInfo.java} | 2 +- .../sql/table/ITableFieldDealHandler.java | 2 +- .../com/dtstack/flink/sql/util/DateUtil.java | 1 - .../dtstack/flink/sql/util/PluginUtil.java | 10 +-- ....java => AbstractCustomerWaterMarker.java} | 4 +- .../CustomerWaterMarkerForLong.java | 2 +- .../CustomerWaterMarkerForTimeStamp.java | 3 +- .../sql/watermarker/WaterMarkerAssigner.java | 8 +-- .../sql/side/SidePredicatesParserTest.java | 4 +- .../flink/sql/side/db2/Db2AllReqRow.java | 8 +-- .../flink/sql/side/db2/Db2AllSideInfo.java | 4 +- .../flink/sql/side/db2/Db2AsyncReqRow.java | 4 +- .../flink/sql/side/db2/Db2AsyncSideInfo.java | 4 +- .../sql/side/db2/table/Db2SideParser.java | 6 +- .../flink/sql/sink/db/table/DbSinkParser.java | 6 +- .../sink/elasticsearch/ElasticsearchSink.java | 4 +- .../table/ElasticsearchSinkParser.java | 8 +-- .../table/ElasticsearchTableInfo.java | 4 +- .../flink/sql/side/hbase/HbaseAllReqRow.java | 8 +-- .../sql/side/hbase/HbaseAllSideInfo.java | 10 +-- .../sql/side/hbase/HbaseAsyncReqRow.java | 14 ++--- .../sql/side/hbase/HbaseAsyncSideInfo.java | 10 +-- ...ler.java => AbstractRowKeyModeDealer.java} | 12 ++-- .../PreRowKeyModeDealerDealer.java | 9 ++- .../rowkeydealer/RowKeyEqualModeDealer.java | 6 +- ...ator.java => AbstractReplaceOperator.java} | 4 +- .../sql/side/hbase/Md5ReplaceOperator.java | 2 +- .../sql/side/hbase/enums/EReplaceOpType.java | 9 ++- .../sql/side/hbase/enums/EReplaceType.java | 6 ++ .../sql/side/hbase/table/HbaseSideParser.java | 12 ++-- .../side/hbase/table/HbaseSideTableInfo.java | 4 +- .../sql/side/hbase/utils/HbaseUtils.java | 2 + .../sql/sink/hbase/HbaseOutputFormat.java | 4 +- .../flink/sql/sink/hbase/HbaseSink.java | 5 +- .../sql/sink/hbase/table/HbaseSinkParser.java | 10 +-- .../sql/sink/hbase/table/HbaseTableInfo.java | 4 +- hbase/pom.xml | 7 +++ .../sql/side/impala/ImpalaAllReqRow.java | 6 +- .../sql/side/impala/ImpalaAllSideInfo.java | 7 +-- .../sql/side/impala/ImpalaAsyncReqRow.java | 4 +- .../sql/side/impala/ImpalaAsyncSideInfo.java | 6 +- .../side/impala/table/ImpalaSideParser.java | 7 ++- .../flink/sql/sink/impala/ImpalaSink.java | 4 +- .../sink/impala/table/ImpalaSinkParser.java | 7 ++- .../sql/sink/kafka/table/KafkaSinkParser.java | 8 +-- .../sink/kafka/table/KafkaSinkTableInfo.java | 5 +- .../source/kafka/table/KafkaSourceParser.java | 10 +-- .../kafka/table/KafkaSourceTableInfo.java | 4 +- .../flink/sql/sink/kafka/KafkaSink.java | 4 +- .../flink/sql/source/kafka/KafkaSource.java | 6 +- .../flink/sql/sink/kafka/KafkaSink.java | 4 +- .../flink/sql/source/kafka/KafkaSource.java | 6 +- .../flink/sql/sink/kafka/KafkaSink.java | 4 +- .../flink/sql/source/kafka/KafkaSource.java | 6 +- .../flink/sql/sink/kafka/KafkaSink.java | 4 +- .../flink/sql/source/kafka/KafkaSource.java | 6 +- .../flink/sql/side/kudu/KuduAllReqRow.java | 8 +-- .../flink/sql/side/kudu/KuduAllSideInfo.java | 10 +-- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 4 +- .../sql/side/kudu/KuduAsyncSideInfo.java | 10 +-- .../sql/side/kudu/table/KuduSideParser.java | 10 +-- .../side/kudu/table/KuduSideTableInfo.java | 4 +- .../flink/sql/sink/kudu/KuduOutputFormat.java | 4 +- .../dtstack/flink/sql/sink/kudu/KuduSink.java | 4 +- .../sql/sink/kudu/table/KuduSinkParser.java | 10 +-- .../sql/sink/kudu/table/KuduTableInfo.java | 4 +- .../flink/sql/side/mongo/MongoAllReqRow.java | 8 +-- .../sql/side/mongo/MongoAllSideInfo.java | 10 +-- .../sql/side/mongo/MongoAsyncReqRow.java | 14 ++--- .../sql/side/mongo/MongoAsyncSideInfo.java | 13 ++-- .../sql/side/mongo/table/MongoSideParser.java | 12 ++-- .../side/mongo/table/MongoSideTableInfo.java | 4 +- .../sql/sink/mongo/MongoOutputFormat.java | 4 +- .../flink/sql/sink/mongo/MongoSink.java | 4 +- .../sql/sink/mongo/table/MongoSinkParser.java | 10 +-- .../sql/sink/mongo/table/MongoTableInfo.java | 4 +- .../flink/sql/side/mysql/MysqlAllReqRow.java | 4 +- .../sql/side/mysql/MysqlAllSideInfo.java | 4 +- .../sql/side/mysql/MysqlAsyncReqRow.java | 4 +- .../sql/side/mysql/MysqlAsyncSideInfo.java | 4 +- .../sql/side/mysql/table/MysqlSideParser.java | 6 +- .../sql/sink/mysql/table/MysqlSinkParser.java | 6 +- .../sql/side/oracle/OracleAllReqRow.java | 4 +- .../sql/side/oracle/OracleAllSideInfo.java | 5 +- .../sql/side/oracle/OracleAsyncReqRow.java | 4 +- .../sql/side/oracle/OracleAsyncSideInfo.java | 8 +-- .../side/oracle/table/OracleSideParser.java | 6 +- .../sink/oracle/table/OracleSinkParser.java | 6 +- .../sql/side/polardb/PolardbAllReqRow.java | 4 +- .../sql/side/polardb/PolardbAllSideInfo.java | 4 +- .../sql/side/polardb/PolardbAsyncReqRow.java | 4 +- .../side/polardb/PolardbAsyncSideInfo.java | 4 +- .../side/polardb/table/PolardbSideParser.java | 6 +- .../sink/polardb/table/PolardbSinkParser.java | 6 +- .../side/postgresql/PostgresqlAllReqRow.java | 4 +- .../postgresql/PostgresqlAllSideInfo.java | 4 +- .../postgresql/PostgresqlAsyncReqRow.java | 4 +- .../postgresql/PostgresqlAsyncSideInfo.java | 4 +- .../table/PostgresqlSideParser.java | 6 +- .../table/PostgresqlSinkParser.java | 6 +- .../side/rdb/all/AbstractRdbAllReqRow.java | 8 +-- .../sql/side/rdb/all/RdbAllSideInfo.java | 10 +-- .../sql/side/rdb/async/RdbAsyncReqRow.java | 8 +-- .../sql/side/rdb/async/RdbAsyncSideInfo.java | 11 ++-- .../sql/side/rdb/table/RdbSideParser.java | 8 +-- .../sql/side/rdb/table/RdbSideTableInfo.java | 4 +- .../flink/sql/sink/rdb/AbstractRdbSink.java | 4 +- .../rdb/format/AbstractJDBCOutputFormat.java | 4 +- .../sql/sink/rdb/table/RdbSinkParser.java | 9 ++- .../sql/sink/rdb/table/RdbTableInfo.java | 6 +- .../sink/rdb/writer/AbstractUpsertWriter.java | 12 ++-- .../sql/sink/rdb/writer/AppendOnlyWriter.java | 6 +- .../flink/sql/side/redis/RedisAllReqRow.java | 4 +- .../sql/side/redis/RedisAllSideInfo.java | 10 +-- .../sql/side/redis/RedisAsyncReqRow.java | 4 +- .../sql/side/redis/RedisAsyncSideInfo.java | 10 +-- .../sql/side/redis/table/RedisSideParser.java | 8 +-- .../sql/side/redis/table/RedisSideReqRow.java | 8 +-- .../side/redis/table/RedisSideTableInfo.java | 4 +- .../sql/sink/redis/RedisOutputFormat.java | 4 +- .../flink/sql/sink/redis/RedisSink.java | 4 +- .../sql/sink/redis/table/RedisSinkParser.java | 8 +-- .../sql/sink/redis/table/RedisTableInfo.java | 4 +- .../CustomerSocketTextStreamFunction.java | 4 +- .../serversocket/ServersocketSource.java | 4 +- .../table/ServersocketSourceParser.java | 8 +-- .../table/ServersocketSourceTableInfo.java | 4 +- .../side/sqlserver/SqlserverAllReqRow.java | 4 +- .../side/sqlserver/SqlserverAllSideInfo.java | 4 +- .../side/sqlserver/SqlserverAsyncReqRow.java | 2 +- .../sqlserver/SqlserverAsyncSideInfo.java | 4 +- .../sqlserver/table/SqlserverSideParser.java | 6 +- .../sqlserver/table/SqlserverSinkParser.java | 6 +- 188 files changed, 731 insertions(+), 640 deletions(-) rename core/src/main/java/com/dtstack/flink/sql/outputformat/{DtRichOutputFormat.java => AbstractDtRichOutputFormat.java} (95%) rename core/src/main/java/com/dtstack/flink/sql/side/{SideTableInfo.java => AbstractSideTableInfo.java} (96%) rename core/src/main/java/com/dtstack/flink/sql/side/{AllReqRow.java => BaseAllReqRow.java} (88%) rename core/src/main/java/com/dtstack/flink/sql/side/{AsyncReqRow.java => BaseAsyncReqRow.java} (91%) rename core/src/main/java/com/dtstack/flink/sql/side/{SideInfo.java => BaseSideInfo.java} (91%) rename core/src/main/java/com/dtstack/flink/sql/side/cache/{AbsSideCache.java => AbstractSideCache.java} (84%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbsSideTableParser.java => AbstractSideTableParser.java} (71%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbsSourceParser.java => AbstractSourceParser.java} (81%) rename core/src/main/java/com/dtstack/flink/sql/table/{SourceTableInfo.java => AbstractSourceTableInfo.java} (97%) rename core/src/main/java/com/dtstack/flink/sql/table/{TableInfo.java => AbstractTableInfo.java} (98%) rename core/src/main/java/com/dtstack/flink/sql/table/{TableInfoParser.java => AbstractTableInfoParser.java} (85%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbsTableParser.java => AbstractTableParser.java} (90%) rename core/src/main/java/com/dtstack/flink/sql/table/{TargetTableInfo.java => AbstractTargetTableInfo.java} (94%) rename core/src/main/java/com/dtstack/flink/sql/watermarker/{AbsCustomerWaterMarker.java => AbstractCustomerWaterMarker.java} (94%) rename hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/{AbsRowKeyModeDealer.java => AbstractRowKeyModeDealer.java} (87%) rename hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/{ReplaceOperator.java => AbstractReplaceOperator.java} (93%) diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index e30b6dfc8..6d1953bfc 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -28,10 +28,10 @@ import com.datastax.driver.core.SocketOptions; import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; @@ -60,14 +60,12 @@ * * @author xuqianjin */ -public class CassandraAllReqRow extends AllReqRow { +public class CassandraAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 54015343561288219L; private static final Logger LOG = LoggerFactory.getLogger(CassandraAllReqRow.class); - private static final String cassandra_DRIVER = "com.cassandra.jdbc.Driver"; - private static final int CONN_RETRY_NUM = 3; private static final int FETCH_SIZE = 1000; @@ -77,7 +75,7 @@ public class CassandraAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public CassandraAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public CassandraAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new com.dtstack.flink.sql.side.cassandra.CassandraAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java index fa665f9a0..5d95dc9bf 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; @@ -37,16 +37,16 @@ * * @author xuqianjin */ -public class CassandraAllSideInfo extends SideInfo { +public class CassandraAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = -8690814317653033557L; - public CassandraAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public CassandraAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { CassandraSideTableInfo cassandraSideTableInfo = (CassandraSideTableInfo) sideTableInfo; sqlCondition = "select ${selectField} from ${tableName} "; diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index a60116e34..3020a73d3 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -30,11 +30,11 @@ import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.google.common.base.Function; @@ -56,7 +56,6 @@ import java.net.InetAddress; import java.sql.Timestamp; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -66,7 +65,7 @@ * * @author xuqianjin */ -public class CassandraAsyncReqRow extends AsyncReqRow { +public class CassandraAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 6631584128079864735L; @@ -82,7 +81,7 @@ public class CassandraAsyncReqRow extends AsyncReqRow { private transient ListenableFuture session; private transient CassandraSideTableInfo cassandraSideTableInfo; - public CassandraAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public CassandraAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new com.dtstack.flink.sql.side.cassandra.CassandraAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java index 3557f0f73..82055b94c 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlBasicCall; @@ -39,16 +39,16 @@ * * @author xuqianjin */ -public class CassandraAsyncSideInfo extends SideInfo { +public class CassandraAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = -4403313049809013362L; - public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { CassandraSideTableInfo cassandraSideTableInfo = (CassandraSideTableInfo) sideTableInfo; String sideTableName = joinInfo.getSideTableName(); diff --git a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java index d2b93563b..e893e56b0 100644 --- a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java +++ b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java @@ -19,18 +19,16 @@ package com.dtstack.flink.sql.side.cassandra.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; -import java.math.BigDecimal; -import java.sql.Date; import java.sql.Timestamp; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -38,7 +36,7 @@ * * @author xuqianjin */ -public class CassandraSideParser extends AbsSideTableParser { +public class CassandraSideParser extends AbstractSideTableParser { private final static String SIDE_SIGN_KEY = "sideSignKey"; @@ -73,7 +71,7 @@ public CassandraSideParser() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo cassandraSideTableInfo = new com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo(); cassandraSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, cassandraSideTableInfo); @@ -96,7 +94,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map { +public class CassandraOutputFormat extends AbstractDtRichOutputFormat { private static final long serialVersionUID = -7994311331389155692L; private static final Logger LOG = LoggerFactory.getLogger(CassandraOutputFormat.class); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java index eb7b23b53..26152a7d3 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.cassandra.table.CassandraTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -63,7 +63,7 @@ public CassandraSink() { } @Override - public CassandraSink genStreamSink(TargetTableInfo targetTableInfo) { + public CassandraSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { CassandraTableInfo cassandraTableInfo = (CassandraTableInfo) targetTableInfo; this.address = cassandraTableInfo.getAddress(); this.tableName = cassandraTableInfo.getTableName(); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java index 4c68e71ae..9ef8639ba 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java @@ -19,13 +19,13 @@ package com.dtstack.flink.sql.sink.cassandra.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -33,7 +33,7 @@ * * @author xuqianjin */ -public class CassandraSinkParser extends AbsTableParser { +public class CassandraSinkParser extends AbstractTableParser { public static final String ADDRESS_KEY = "address"; @@ -60,7 +60,7 @@ public class CassandraSinkParser extends AbsTableParser { public static final String POOL_TIMEOUT_MILLIS_KEY = "poolTimeoutMillis"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { CassandraTableInfo cassandraTableInfo = new CassandraTableInfo(); cassandraTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, cassandraTableInfo); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java index c6626c42a..ffb5fa876 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.cassandra.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; /** @@ -28,7 +28,7 @@ * * @author xuqianjin */ -public class CassandraTableInfo extends TargetTableInfo { +public class CassandraTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "cassandra"; diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java index bd3d033ff..c9a0c447b 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -37,7 +37,7 @@ public class ClickhouseAllReqRow extends AbstractRdbAllReqRow { private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - public ClickhouseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ClickhouseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java index 973c069b9..43fbeaa56 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -28,7 +28,7 @@ public class ClickhouseAllSideInfo extends RdbAllSideInfo { - public ClickhouseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index 3733d7350..e6f008d63 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -37,7 +37,7 @@ public class ClickhouseAsyncReqRow extends RdbAsyncReqRow { private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - public ClickhouseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ClickhouseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java index 254561de0..eec5fbe74 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -29,7 +29,7 @@ public class ClickhouseAsyncSideInfo extends RdbAsyncSideInfo { - public ClickhouseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java b/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java index 7be387fd8..ab285c37f 100644 --- a/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java +++ b/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.clickhouse.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import ru.yandex.clickhouse.domain.ClickHouseDataType; import java.util.Map; @@ -38,8 +38,8 @@ public class ClickhouseSideParser extends RdbSideParser { private static final String CURR_TYPE = "clickhouse"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); clickhouseTableInfo.setType(CURR_TYPE); return clickhouseTableInfo; } diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java index 8c3df93d7..5b0f2598f 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.clickhouse.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import ru.yandex.clickhouse.domain.ClickHouseDataType; import java.util.Map; @@ -30,8 +30,8 @@ public class ClickhouseSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "clickhouse"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); clickhouseTableInfo.setType(CURR_TYPE); return clickhouseTableInfo; } diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java index a397036ef..e99c76d03 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.console; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.sink.console.table.TablePrintUtil; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -37,7 +37,7 @@ * * @author xuqianjin */ -public class ConsoleOutputFormat extends DtRichOutputFormat { +public class ConsoleOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(ConsoleOutputFormat.class); diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java index 77a3efea2..6cfdd2d3d 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.console; import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -80,7 +80,7 @@ public void emitDataStream(DataStream> dataStream) { } @Override - public ConsoleSink genStreamSink(TargetTableInfo targetTableInfo) { + public ConsoleSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { return this; } } diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java index e77444bfd..93ed02420 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java @@ -18,13 +18,13 @@ package com.dtstack.flink.sql.sink.console.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -32,9 +32,9 @@ * * @author xuqianjin */ -public class ConsoleSinkParser extends AbsTableParser { +public class ConsoleSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ConsoleTableInfo consoleTableInfo = new ConsoleTableInfo(); consoleTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, consoleTableInfo); diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java index 4b286c667..6fd7063c6 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.console.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; /** * Reason: @@ -26,7 +26,7 @@ * * @author xuqianjin */ -public class ConsoleTableInfo extends TargetTableInfo { +public class ConsoleTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "console"; diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java index dcda4c093..d5b13ce12 100644 --- a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java @@ -65,10 +65,10 @@ public class ConfigConstrant { // restart plocy - public static final int failureRate = 3; + public static final int FAILUEE_RATE = 3; - public static final int failureInterval = 6; //min + public static final int FAILUEE_INTERVAL = 6; //min - public static final int delayInterval = 10; //sec + public static final int DELAY_INTERVAL = 10; //sec } diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java index 749bbc907..7f3f0019c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java @@ -46,11 +46,66 @@ * @author huyifan.zju@163.com */ public enum ColumnType { - STRING, VARCHAR, CHAR, - INT, MEDIUMINT, TINYINT, DATETIME, SMALLINT, BIGINT, - DOUBLE, FLOAT, + /* + * string + */ + STRING, + /** + * varchar + */ + VARCHAR, + /** + * char + */ + CHAR, + /** + * int + */ + INT, + /** + * mediumint + */ + MEDIUMINT, + /** + * tinyint + */ + TINYINT, + /** + * datetime + */ + DATETIME, + /** + * smallint + */ + SMALLINT, + /** + * bigint + */ + BIGINT, + /** + * double + */ + DOUBLE, + /** + * float + */ + FLOAT, + /** + * boolean + */ BOOLEAN, - DATE, TIMESTAMP, DECIMAL; + /** + * date + */ + DATE, + /** + * timestamp + */ + TIMESTAMP, + /** + * decimal + */ + DECIMAL; public static ColumnType fromString(String type) { if(type == null) { diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java index 66160d820..49e352757 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java @@ -30,8 +30,17 @@ public enum ECacheContentType { + /** + * 无 + */ MissVal(0), + /** + * 1行 + */ SingleLine(1), + /** + * 多行 + */ MultiLine(2); int type; diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java index 582148c2c..9d5bb5d11 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java @@ -27,7 +27,18 @@ * @author xuchao */ public enum ECacheType { - NONE, LRU, ALL; + /** + * none + */ + NONE, + /** + * lru + */ + LRU, + /** + * all + */ + ALL; public static boolean isValid(String type){ for(ECacheType tmpType : ECacheType.values()){ diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java b/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java index 9b01bf052..7d2235f2e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java @@ -26,8 +26,17 @@ */ public enum EDatabaseType { + /** + * mysql + */ MYSQL, + /** + * sqlserver + */ SQLSERVER, + /** + * oracle + */ ORACLE, } diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java b/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java index 6cb027ac3..439966dd2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java @@ -26,7 +26,13 @@ */ public enum EPluginLoadMode { + /** + * 0:classpath + */ CLASSPATH(0), + /** + * 1:shipfile + */ SHIPFILE(1); private int type; diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java b/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java index a8f926175..098cb57fe 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java @@ -25,8 +25,17 @@ * @author maqi */ public enum EStateBackend { + /** + * memory + */ MEMORY, + /** + * rockdb + */ ROCKSDB, + /** + * filesystem + */ FILESYSTEM; public static EStateBackend convertFromString(String type) { diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index b453c9414..9db50e65a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -102,9 +102,9 @@ public static void streamExecutionEnvironmentConfig(StreamExecutionEnvironment s }); streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( - ConfigConstrant.failureRate, - Time.of(ConfigConstrant.failureInterval, TimeUnit.MINUTES), - Time.of(ConfigConstrant.delayInterval, TimeUnit.SECONDS) + ConfigConstrant.FAILUEE_RATE, + Time.of(ConfigConstrant.FAILUEE_INTERVAL, TimeUnit.MINUTES), + Time.of(ConfigConstrant.DELAY_INTERVAL, TimeUnit.SECONDS) )); // checkpoint config diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 2c6d74df0..ed54a9c48 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -36,12 +36,12 @@ import com.dtstack.flink.sql.parser.SqlParser; import com.dtstack.flink.sql.parser.SqlTree; import com.dtstack.flink.sql.side.SideSqlExec; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.sink.StreamSinkFactory; import com.dtstack.flink.sql.source.StreamSourceFactory; -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import com.dtstack.flink.sql.watermarker.WaterMarkerAssigner; @@ -159,7 +159,7 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf SqlParser.setLocalSqlPluginRoot(paramsInfo.getLocalSqlPluginPath()); SqlTree sqlTree = SqlParser.parseSql(paramsInfo.getSql()); - Map sideTableMap = Maps.newHashMap(); + Map sideTableMap = Maps.newHashMap(); Map registerTableCache = Maps.newHashMap(); //register udf @@ -193,7 +193,7 @@ public static List getExternalJarUrls(String addJarListStr) throws java.io. return jarUrlList; } - public static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironment tableEnv, SqlTree sqlTree, Map sideTableMap, Map registerTableCache, StreamQueryConfig queryConfig) throws Exception { + public static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironment tableEnv, SqlTree sqlTree, Map sideTableMap, Map registerTableCache, StreamQueryConfig queryConfig) throws Exception { SideSqlExec sideSqlExec = new SideSqlExec(); sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { @@ -264,14 +264,14 @@ public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrl * @throws Exception */ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { + String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { Set pluginClassPatshSets = Sets.newHashSet(); WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); - for (TableInfo tableInfo : sqlTree.getTableInfoMap().values()) { + for (AbstractTableInfo tableInfo : sqlTree.getTableInfoMap().values()) { - if (tableInfo instanceof SourceTableInfo) { + if (tableInfo instanceof AbstractSourceTableInfo) { - SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; + AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; Table table = StreamSourceFactory.getStreamSource(sourceTableInfo, env, tableEnv, localSqlPluginPath); tableEnv.registerTable(sourceTableInfo.getAdaptName(), table); //Note --- parameter conversion function can not be used inside a function of the type of polymerization @@ -302,21 +302,21 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment } registerTableCache.put(tableInfo.getName(), regTable); - URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), SourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractSourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sourceTablePathUrl); - } else if (tableInfo instanceof TargetTableInfo) { + } else if (tableInfo instanceof AbstractTargetTableInfo) { - TableSink tableSink = StreamSinkFactory.getTableSink((TargetTableInfo) tableInfo, localSqlPluginPath); + TableSink tableSink = StreamSinkFactory.getTableSink((AbstractTargetTableInfo) tableInfo, localSqlPluginPath); TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); - URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), TargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sinkTablePathUrl); - } else if (tableInfo instanceof SideTableInfo) { - String sideOperator = ECacheType.ALL.name().equals(((SideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; - sideTableMap.put(tableInfo.getName(), (SideTableInfo) tableInfo); + } else if (tableInfo instanceof AbstractSideTableInfo) { + String sideOperator = ECacheType.ALL.name().equals(((AbstractSideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; + sideTableMap.put(tableInfo.getName(), (AbstractSideTableInfo) tableInfo); - URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, SideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, AbstractSideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sideTablePathUrl); } else { throw new RuntimeException("not support table type:" + tableInfo.getType()); diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 78b579305..4c0d68eb2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.format.dtnest; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.google.common.base.Strings; import com.google.common.collect.Maps; import org.apache.flink.api.common.serialization.AbstractDeserializationSchema; @@ -57,9 +57,9 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final String[] fieldNames; private final TypeInformation[] fieldTypes; - private List fieldExtraInfos; + private List fieldExtraInfos; - public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { + public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); this.rowAndFieldMapping = rowAndFieldMapping; @@ -75,7 +75,7 @@ public Row deserialize(byte[] message) throws IOException { try { for (int i = 0; i < fieldNames.length; i++) { JsonNode node = getIgnoreCase(fieldNames[i]); - TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); if (node == null) { if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java index 0e8bda1fe..e49adfd93 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java @@ -92,7 +92,7 @@ public Options getOptions(){ } public List getProgramExeArgList() throws Exception { - Map mapConf = PluginUtil.ObjectToMap(properties); + Map mapConf = PluginUtil.objectToMap(properties); List args = Lists.newArrayList(); for(Map.Entry one : mapConf.entrySet()){ String key = one.getKey(); diff --git a/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java b/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java similarity index 95% rename from core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java rename to core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java index 1fc40c13b..fbcc86bbd 100644 --- a/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java +++ b/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java @@ -27,7 +27,7 @@ * extend RichOutputFormat with metric 'dtNumRecordsOut', 'dtNumDirtyRecordsOut', 'dtNumRecordsOutRate' * Created by sishu.yss on 2018/11/28. */ -public abstract class DtRichOutputFormat extends RichOutputFormat{ +public abstract class AbstractDtRichOutputFormat extends RichOutputFormat{ public transient Counter outRecords; public transient Counter outDirtyRecords; diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index a76c1b31a..2afc76c48 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.parser; import com.dtstack.flink.sql.enums.ETableType; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.table.TableInfoParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfoParser; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import com.google.common.collect.Lists; @@ -75,7 +75,7 @@ public static SqlTree parseSql(String sql) throws Exception { List sqlArr = DtStringUtil.splitIgnoreQuota(sql, SQL_DELIMITER); SqlTree sqlTree = new SqlTree(); - TableInfoParser tableInfoParser = new TableInfoParser(); + AbstractTableInfoParser tableInfoParser = new AbstractTableInfoParser(); for(String childSql : sqlArr){ if(Strings.isNullOrEmpty(childSql)){ continue; @@ -113,7 +113,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } - TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), + AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } @@ -126,7 +126,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } - TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SINK.getType(), + AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SINK.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } @@ -144,7 +144,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } } else { - TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), + AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java index 1b64b7c68..5252ee022 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.parser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.google.common.collect.Maps; import com.google.common.collect.Lists; @@ -41,7 +41,7 @@ public class SqlTree { private Map preDealTableMap = Maps.newHashMap(); - private Map tableInfoMap = Maps.newLinkedHashMap(); + private Map tableInfoMap = Maps.newLinkedHashMap(); private List execSqlList = Lists.newArrayList(); @@ -89,11 +89,11 @@ public List getTmpSqlList(){ return tmpSqlList; } - public Map getTableInfoMap() { + public Map getTableInfoMap() { return tableInfoMap; } - public void addTableInfo(String tableName, TableInfo tableInfo){ + public void addTableInfo(String tableName, AbstractTableInfo tableInfo){ tableInfoMap.put(tableName, tableInfo); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java similarity index 96% rename from core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index 8562d9859..6de2354a3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.google.common.collect.Lists; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ * @author xuchao */ -public abstract class SideTableInfo extends TableInfo implements Serializable { +public abstract class AbstractSideTableInfo extends AbstractTableInfo implements Serializable { public static final String TARGET_SUFFIX = "Side"; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java similarity index 88% rename from core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java rename to core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java index 7d16ee726..8a6851add 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java @@ -37,13 +37,13 @@ * @author xuchao */ -public abstract class AllReqRow extends RichFlatMapFunction implements ISideReqRow { +public abstract class BaseAllReqRow extends RichFlatMapFunction implements ISideReqRow { - protected SideInfo sideInfo; + protected BaseSideInfo sideInfo; private ScheduledExecutorService es; - public AllReqRow(SideInfo sideInfo){ + public BaseAllReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -59,7 +59,7 @@ public void open(Configuration parameters) throws Exception { System.out.println("----- all cacheRef init end-----"); //start reload cache thread - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); es = Executors.newSingleThreadScheduledExecutor(new DTThreadFactory("cache-all-reload")); es.scheduleAtFixedRate(() -> reloadCache(), sideTableInfo.getCacheTimeout(), sideTableInfo.getCacheTimeout(), TimeUnit.MILLISECONDS); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java similarity index 91% rename from core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java rename to core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 0415102ab..ff08603cb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.metric.MetricConstant; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cache.LRUSideCache; import org.apache.calcite.sql.JoinType; @@ -47,14 +47,14 @@ * @author xuchao */ -public abstract class AsyncReqRow extends RichAsyncFunction implements ISideReqRow { - private static final Logger LOG = LoggerFactory.getLogger(AsyncReqRow.class); +public abstract class BaseAsyncReqRow extends RichAsyncFunction implements ISideReqRow { + private static final Logger LOG = LoggerFactory.getLogger(BaseAsyncReqRow.class); private static final long serialVersionUID = 2098635244857937717L; - protected SideInfo sideInfo; + protected BaseSideInfo sideInfo; protected transient Counter parseErrorRecords; - public AsyncReqRow(SideInfo sideInfo){ + public BaseAsyncReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -66,12 +66,12 @@ public void open(Configuration parameters) throws Exception { } private void initCache(){ - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); if(sideTableInfo.getCacheType() == null || ECacheType.NONE.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ return; } - AbsSideCache sideCache; + AbstractSideCache sideCache; if(ECacheType.LRU.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ sideCache = new LRUSideCache(sideTableInfo); sideInfo.setSideCache(sideCache); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java similarity index 91% rename from core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java rename to core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java index df41e1663..9c6e1c575 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; @@ -41,7 +41,7 @@ * @author xuchao */ -public abstract class SideInfo implements Serializable{ +public abstract class BaseSideInfo implements Serializable{ protected RowTypeInfo rowTypeInfo; @@ -66,12 +66,12 @@ public abstract class SideInfo implements Serializable{ //key:Returns the value of the position, value: the ref field name​in the side table protected Map sideFieldNameIndex = Maps.newHashMap(); - protected SideTableInfo sideTableInfo; + protected AbstractSideTableInfo sideTableInfo; - protected AbsSideCache sideCache; + protected AbstractSideCache sideCache; - public SideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, - SideTableInfo sideTableInfo){ + public BaseSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, + AbstractSideTableInfo sideTableInfo){ this.rowTypeInfo = rowTypeInfo; this.outFieldInfoList = outFieldInfoList; this.joinType = joinInfo.getJoinType(); @@ -158,7 +158,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName){ } } - public abstract void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo); + public abstract void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo); public RowTypeInfo getRowTypeInfo() { return rowTypeInfo; @@ -232,19 +232,19 @@ public void setSideFieldIndex(Map sideFieldIndex) { this.sideFieldIndex = sideFieldIndex; } - public SideTableInfo getSideTableInfo() { + public AbstractSideTableInfo getSideTableInfo() { return sideTableInfo; } - public void setSideTableInfo(SideTableInfo sideTableInfo) { + public void setSideTableInfo(AbstractSideTableInfo sideTableInfo) { this.sideTableInfo = sideTableInfo; } - public AbsSideCache getSideCache() { + public AbstractSideCache getSideCache() { return sideCache; } - public void setSideCache(AbsSideCache sideCache) { + public void setSideCache(AbstractSideCache sideCache) { this.sideCache = sideCache; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java index 64f9e646b..02b9c97f8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java @@ -34,7 +34,6 @@ import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static org.apache.calcite.sql.SqlKind.*; @@ -46,7 +45,7 @@ * @author maqi */ public class SidePredicatesParser { - public void fillPredicatesForSideTable(String exeSql, Map sideTableMap) throws SqlParseException { + public void fillPredicatesForSideTable(String exeSql, Map sideTableMap) throws SqlParseException { SqlParser sqlParser = SqlParser.create(exeSql, CalciteConfig.MYSQL_LEX_CONFIG); SqlNode sqlNode = sqlParser.parseStmt(); parseSql(sqlNode, sideTableMap, Maps.newHashMap()); @@ -58,7 +57,7 @@ public void fillPredicatesForSideTable(String exeSql, Map * @param sideTableMap * @param tabMapping 谓词属性中别名对应的真实维表名称 */ - private void parseSql(SqlNode sqlNode, Map sideTableMap, Map tabMapping) { + private void parseSql(SqlNode sqlNode, Map sideTableMap, Map tabMapping) { SqlKind sqlKind = sqlNode.getKind(); switch (sqlKind) { case INSERT: @@ -106,7 +105,7 @@ private void parseSql(SqlNode sqlNode, Map sideTableMap, } } - private void fillToSideTableInfo(Map sideTableMap, Map tabMapping, List predicateInfoList) { + private void fillToSideTableInfo(Map sideTableMap, Map tabMapping, List predicateInfoList) { predicateInfoList.stream().filter(info -> sideTableMap.containsKey(tabMapping.getOrDefault(info.getOwnerTable(), info.getOwnerTable()))) .map(info -> sideTableMap.get(tabMapping.getOrDefault(info.getOwnerTable(), info.getOwnerTable())).getPredicateInfoes().add(info)) .count(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 1689949ca..885fa422c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -96,7 +96,7 @@ public class SideSqlExec { private Map localTableCache = Maps.newHashMap(); private StreamTableEnvironment tableEnv ; - public void exec(String sql, Map sideTableMap, StreamTableEnvironment tableEnv, + public void exec(String sql, Map sideTableMap, StreamTableEnvironment tableEnv, Map tableCache, StreamQueryConfig queryConfig) throws Exception { if(localSqlPluginPath == null){ throw new RuntimeException("need to set localSqlPluginPath"); @@ -657,7 +657,7 @@ private SqlNode replaceSelectFieldName(SqlNode selectNode, FieldReplaceInfo repl * * @return */ - private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, SideTableInfo sideTableInfo) { + private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, AbstractSideTableInfo sideTableInfo) { List conditionFields = getConditionFields(conditionNode, sideTableAlias, sideTableInfo); if(CollectionUtils.isEqualCollection(conditionFields, convertPrimaryAlias(sideTableInfo))){ return true; @@ -665,7 +665,7 @@ private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, return false; } - private List convertPrimaryAlias(SideTableInfo sideTableInfo) { + private List convertPrimaryAlias(AbstractSideTableInfo sideTableInfo) { List res = Lists.newArrayList(); sideTableInfo.getPrimaryKeys().forEach(field -> { res.add(sideTableInfo.getPhysicalFields().getOrDefault(field, field)); @@ -673,7 +673,7 @@ private List convertPrimaryAlias(SideTableInfo sideTableInfo) { return res; } - public List getConditionFields(SqlNode conditionNode, String specifyTableName, SideTableInfo sideTableInfo){ + public List getConditionFields(SqlNode conditionNode, String specifyTableName, AbstractSideTableInfo sideTableInfo){ List sqlNodeList = Lists.newArrayList(); ParseUtils.parseAnd(conditionNode, sqlNodeList); List conditionFields = Lists.newArrayList(); @@ -704,7 +704,7 @@ public List getConditionFields(SqlNode conditionNode, String specifyTabl } public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, - Map sideTableMap, StreamTableEnvironment tableEnv, + Map sideTableMap, StreamTableEnvironment tableEnv, Map tableCache) throws Exception { @@ -778,7 +778,7 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, SqlNode pollSq } private void joinFun(Object pollObj, Map localTableCache, - Map sideTableMap, StreamTableEnvironment tableEnv, + Map sideTableMap, StreamTableEnvironment tableEnv, List replaceInfoList) throws Exception{ JoinInfo joinInfo = (JoinInfo) pollObj; @@ -799,7 +799,7 @@ private void joinFun(Object pollObj, Map localTableCache, JoinScope.ScopeChild rightScopeChild = new JoinScope.ScopeChild(); rightScopeChild.setAlias(joinInfo.getRightTableAlias()); rightScopeChild.setTableName(joinInfo.getRightTableName()); - SideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); + AbstractSideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); if(sideTableInfo == null){ sideTableInfo = sideTableMap.get(joinInfo.getRightTableAlias()); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java b/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java index 8417c4519..583e4597b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java @@ -21,8 +21,8 @@ import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.enums.ECacheType; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableParser; import com.dtstack.flink.sql.util.PluginUtil; /** @@ -37,7 +37,7 @@ public class StreamSideFactory { private static final String CURR_TYPE = "side"; - public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir, String cacheType) throws Exception { + public static AbstractTableParser getSqlParser(String pluginType, String sqlRootDir, String cacheType) throws Exception { String sideOperator = ECacheType.ALL.name().equalsIgnoreCase(cacheType) ? "all" : "async"; String pluginJarPath = PluginUtil.getSideJarFileDirPath(pluginType, sideOperator, "side", sqlRootDir); @@ -45,10 +45,10 @@ public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir, return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class sideParser = cl.loadClass(className); - if (!AbsSideTableParser.class.isAssignableFrom(sideParser)) { + if (!AbstractSideTableParser.class.isAssignableFrom(sideParser)) { throw new RuntimeException("class " + sideParser.getName() + " not subClass of AbsSideTableParser"); } - return sideParser.asSubclass(AbsTableParser.class).newInstance(); + return sideParser.asSubclass(AbstractTableParser.class).newInstance(); }); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java b/core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java similarity index 84% rename from core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java rename to core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java index 757f91600..58832c583 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.cache; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; /** * Reason: @@ -30,11 +30,11 @@ * @author xuchao */ -public abstract class AbsSideCache { +public abstract class AbstractSideCache { - protected SideTableInfo sideTableInfo; + protected AbstractSideTableInfo sideTableInfo; - public AbsSideCache(SideTableInfo sideTableInfo){ + public AbstractSideCache(AbstractSideTableInfo sideTableInfo){ this.sideTableInfo = sideTableInfo; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java b/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java index 700e13bb2..2664efd34 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.cache; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -33,11 +33,11 @@ * @author xuchao */ -public class LRUSideCache extends AbsSideCache{ +public class LRUSideCache extends AbstractSideCache { protected transient Cache cache; - public LRUSideCache(SideTableInfo sideTableInfo) { + public LRUSideCache(AbstractSideTableInfo sideTableInfo) { super(sideTableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java index 290804200..3c2010907 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java @@ -20,10 +20,10 @@ package com.dtstack.flink.sql.side.operator; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.AsyncDataStream; @@ -49,20 +49,20 @@ public class SideAsyncOperator { private static final String ORDERED = "ordered"; - private static AsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, - JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { + private static BaseAsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { String pathOfType = String.format(PATH_FORMAT, sideType); String pluginJarPath = PluginUtil.getJarFileDirPath(pathOfType, sqlRootDir); String className = PluginUtil.getSqlSideClassName(sideType, "side", OPERATOR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> - cl.loadClass(className).asSubclass(AsyncReqRow.class) - .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class) + cl.loadClass(className).asSubclass(BaseAsyncReqRow.class) + .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, AbstractSideTableInfo.class) .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, - List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { - AsyncReqRow asyncDbReq = loadAsyncReq(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { + BaseAsyncReqRow asyncDbReq = loadAsyncReq(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); //TODO How much should be set for the degree of parallelism? Timeout? capacity settings? if (ORDERED.equals(sideTableInfo.getCacheMode())){ diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java index 5aa810b0f..6b6f9fe1b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java @@ -20,10 +20,10 @@ package com.dtstack.flink.sql.side.operator; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; @@ -44,22 +44,22 @@ public class SideWithAllCacheOperator { private static final String OPERATOR_TYPE = "All"; - private static AllReqRow loadFlatMap(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, - JoinInfo joinInfo, List outFieldInfoList, - SideTableInfo sideTableInfo) throws Exception { + private static BaseAllReqRow loadFlatMap(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, List outFieldInfoList, + AbstractSideTableInfo sideTableInfo) throws Exception { String pathOfType = String.format(PATH_FORMAT, sideType); String pluginJarPath = PluginUtil.getJarFileDirPath(pathOfType, sqlRootDir); String className = PluginUtil.getSqlSideClassName(sideType, "side", OPERATOR_TYPE); - return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> cl.loadClass(className).asSubclass(AllReqRow.class) - .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class) + return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> cl.loadClass(className).asSubclass(BaseAllReqRow.class) + .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, AbstractSideTableInfo.class) .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, - List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { - AllReqRow allReqRow = loadFlatMap(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { + BaseAllReqRow allReqRow = loadFlatMap(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); return inputStream.flatMap(allReqRow); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java b/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java index 3cfd48f1b..e29421369 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; /** * Reason: @@ -30,5 +30,5 @@ */ public interface IStreamSinkGener { - T genStreamSink(TargetTableInfo targetTableInfo); + T genStreamSink(AbstractTargetTableInfo targetTableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java b/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java index 53460081d..89061db3a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.sink; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.table.sinks.TableSink; @@ -40,21 +40,21 @@ public class StreamSinkFactory { private static final String DIR_NAME_FORMAT = "%ssink"; - public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { + public static AbstractTableParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { String pluginJarPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), sqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); String className = PluginUtil.getSqlParserClassName(typeNoVersion, CURR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class targetParser = cl.loadClass(className); - if(!AbsTableParser.class.isAssignableFrom(targetParser)){ + if(!AbstractTableParser.class.isAssignableFrom(targetParser)){ throw new RuntimeException("class " + targetParser.getName() + " not subClass of AbsTableParser"); } - return targetParser.asSubclass(AbsTableParser.class).newInstance(); + return targetParser.asSubclass(AbstractTableParser.class).newInstance(); }); } - public static TableSink getTableSink(TargetTableInfo targetTableInfo, String localSqlRootDir) throws Exception { + public static TableSink getTableSink(AbstractTargetTableInfo targetTableInfo, String localSqlRootDir) throws Exception { String pluginType = targetTableInfo.getType(); String pluginJarDirPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), localSqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); diff --git a/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java b/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java index b8b8f6edc..fde5a235d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java +++ b/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.source; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.java.StreamTableEnvironment; @@ -38,6 +38,6 @@ public interface IStreamSourceGener { * @param tableEnv * @return */ - T genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv); + T genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv); } diff --git a/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java b/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java index 1057fb0ed..e0cec1415 100644 --- a/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.table.AbsSourceParser; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceParser; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -43,17 +43,17 @@ public class StreamSourceFactory { private static final String DIR_NAME_FORMAT = "%ssource"; - public static AbsSourceParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { + public static AbstractSourceParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { String pluginJarPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), sqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); String className = PluginUtil.getSqlParserClassName(typeNoVersion, CURR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class sourceParser = cl.loadClass(className); - if(!AbsSourceParser.class.isAssignableFrom(sourceParser)){ + if(!AbstractSourceParser.class.isAssignableFrom(sourceParser)){ throw new RuntimeException("class " + sourceParser.getName() + " not subClass of AbsSourceParser"); } - return sourceParser.asSubclass(AbsSourceParser.class).newInstance(); + return sourceParser.asSubclass(AbstractSourceParser.class).newInstance(); }); } @@ -62,7 +62,7 @@ public static AbsSourceParser getSqlParser(String pluginType, String sqlRootDir) * @param sourceTableInfo * @return */ - public static Table getStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, + public static Table getStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String sqlRootDir) throws Exception { String sourceTypeStr = sourceTableInfo.getType(); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java similarity index 71% rename from core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 49c5629bc..8da46b079 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.table; import com.dtstack.flink.sql.enums.ECacheType; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; import java.util.regex.Matcher; @@ -34,24 +34,24 @@ * @author xuchao */ -public abstract class AbsSideTableParser extends AbsTableParser { +public abstract class AbstractSideTableParser extends AbstractTableParser { private final static String SIDE_SIGN_KEY = "sideSignKey"; private final static Pattern SIDE_TABLE_SIGN = Pattern.compile("(?i)^PERIOD\\s+FOR\\s+SYSTEM_TIME$"); - public AbsSideTableParser() { + public AbstractSideTableParser() { addParserHandler(SIDE_SIGN_KEY, SIDE_TABLE_SIGN, this::dealSideSign); } - private void dealSideSign(Matcher matcher, TableInfo tableInfo){ + private void dealSideSign(Matcher matcher, AbstractTableInfo tableInfo){ //FIXME SIDE_TABLE_SIGN current just used as a sign for side table; and do nothing } //Analytical create table attributes ==> Get information cache - protected void parseCacheProp(SideTableInfo sideTableInfo, Map props){ - if(props.containsKey(SideTableInfo.CACHE_KEY.toLowerCase())){ - String cacheType = MathUtil.getString(props.get(SideTableInfo.CACHE_KEY.toLowerCase())); + protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map props){ + if(props.containsKey(AbstractSideTableInfo.CACHE_KEY.toLowerCase())){ + String cacheType = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_KEY.toLowerCase())); if(cacheType == null){ return; } @@ -61,31 +61,31 @@ protected void parseCacheProp(SideTableInfo sideTableInfo, Map p } sideTableInfo.setCacheType(cacheType); - if(props.containsKey(SideTableInfo.CACHE_SIZE_KEY.toLowerCase())){ - Integer cacheSize = MathUtil.getIntegerVal(props.get(SideTableInfo.CACHE_SIZE_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.CACHE_SIZE_KEY.toLowerCase())){ + Integer cacheSize = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.CACHE_SIZE_KEY.toLowerCase())); if(cacheSize < 0){ throw new RuntimeException("cache size need > 0."); } sideTableInfo.setCacheSize(cacheSize); } - if(props.containsKey(SideTableInfo.CACHE_TTLMS_KEY.toLowerCase())){ - Long cacheTTLMS = MathUtil.getLongVal(props.get(SideTableInfo.CACHE_TTLMS_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.CACHE_TTLMS_KEY.toLowerCase())){ + Long cacheTTLMS = MathUtil.getLongVal(props.get(AbstractSideTableInfo.CACHE_TTLMS_KEY.toLowerCase())); if(cacheTTLMS < 1000){ throw new RuntimeException("cache time out need > 1000 ms."); } sideTableInfo.setCacheTimeout(cacheTTLMS); } - if(props.containsKey(SideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())){ - Boolean partitionedJoinKey = MathUtil.getBoolean(props.get(SideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())){ + Boolean partitionedJoinKey = MathUtil.getBoolean(props.get(AbstractSideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())); if(partitionedJoinKey){ sideTableInfo.setPartitionedJoin(true); } } - if(props.containsKey(SideTableInfo.CACHE_MODE_KEY.toLowerCase())){ - String cachemode = MathUtil.getString(props.get(SideTableInfo.CACHE_MODE_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.CACHE_MODE_KEY.toLowerCase())){ + String cachemode = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_MODE_KEY.toLowerCase())); if(!"ordered".equalsIgnoreCase(cachemode) && !"unordered".equalsIgnoreCase(cachemode)){ throw new RuntimeException("cachemode must ordered or unordered!"); @@ -93,16 +93,16 @@ protected void parseCacheProp(SideTableInfo sideTableInfo, Map p sideTableInfo.setCacheMode(cachemode.toLowerCase()); } - if(props.containsKey(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())){ - Integer asyncCap = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.ASYNC_CAP_KEY.toLowerCase())){ + Integer asyncCap = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_CAP_KEY.toLowerCase())); if(asyncCap < 0){ throw new RuntimeException("asyncCapacity size need > 0."); } sideTableInfo.setAsyncCapacity(asyncCap); } - if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())){ - Integer asyncTimeout = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())){ + Integer asyncTimeout = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())); if (asyncTimeout<0){ throw new RuntimeException("asyncTimeout size need > 0."); } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java similarity index 81% rename from core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java index 745357162..308f5859f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java @@ -33,7 +33,7 @@ * @author xuchao */ -public abstract class AbsSourceParser extends AbsTableParser { +public abstract class AbstractSourceParser extends AbstractTableParser { private static final String VIRTUAL_KEY = "virtualFieldKey"; private static final String WATERMARK_KEY = "waterMarkKey"; @@ -43,21 +43,21 @@ public abstract class AbsSourceParser extends AbsTableParser { private static Pattern waterMarkKeyPattern = Pattern.compile("(?i)^\\s*WATERMARK\\s+FOR\\s+(\\S+)\\s+AS\\s+withOffset\\(\\s*(\\S+)\\s*,\\s*(\\d+)\\s*\\)$"); private static Pattern notNullKeyPattern = Pattern.compile("(?i)^(\\w+)\\s+(\\w+)\\s+NOT\\s+NULL?$"); - public AbsSourceParser() { + public AbstractSourceParser() { addParserHandler(VIRTUAL_KEY, virtualFieldKeyPattern, this::dealVirtualField); addParserHandler(WATERMARK_KEY, waterMarkKeyPattern, this::dealWaterMark); addParserHandler(NOTNULL_KEY, notNullKeyPattern, this::dealNotNull); } - protected void dealVirtualField(Matcher matcher, TableInfo tableInfo){ - SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; + protected void dealVirtualField(Matcher matcher, AbstractTableInfo tableInfo){ + AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; String fieldName = matcher.group(2); String expression = matcher.group(1); sourceTableInfo.addVirtualField(fieldName, expression); } - protected void dealWaterMark(Matcher matcher, TableInfo tableInfo){ - SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; + protected void dealWaterMark(Matcher matcher, AbstractTableInfo tableInfo){ + AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; String eventTimeField = matcher.group(1); //FIXME Temporarily resolve the second parameter row_time_field Integer offset = MathUtil.getIntegerVal(matcher.group(3)); @@ -65,11 +65,11 @@ protected void dealWaterMark(Matcher matcher, TableInfo tableInfo){ sourceTableInfo.setMaxOutOrderness(offset); } - protected void dealNotNull(Matcher matcher, TableInfo tableInfo) { + protected void dealNotNull(Matcher matcher, AbstractTableInfo tableInfo) { String fieldName = matcher.group(1); String fieldType = matcher.group(2); Class fieldClass= dbTypeConvertToJavaType(fieldType); - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); fieldExtraInfo.setNotNull(true); tableInfo.addPhysicalMappings(fieldName, fieldName); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java similarity index 97% rename from core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java index 9a41fa0a1..0fdc0e911 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java @@ -36,7 +36,7 @@ * @author xuchao */ -public abstract class SourceTableInfo extends TableInfo { +public abstract class AbstractSourceTableInfo extends AbstractTableInfo { public static final String SOURCE_SUFFIX = "Source"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java similarity index 98% rename from core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java index 59e4fdd39..e450adb2f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java @@ -34,7 +34,7 @@ * @author xuchao */ -public abstract class TableInfo implements Serializable { +public abstract class AbstractTableInfo implements Serializable { public static final String PARALLELISM_KEY = "parallelism"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java similarity index 85% rename from core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java index ae98d90ae..c6687433c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.enums.ETableType; import com.dtstack.flink.sql.parser.CreateTableParser; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.StreamSideFactory; import com.dtstack.flink.sql.sink.StreamSinkFactory; import com.dtstack.flink.sql.source.StreamSourceFactory; @@ -41,7 +41,7 @@ * @author xuchao */ -public class TableInfoParser { +public class AbstractTableInfoParser { private final static String TYPE_KEY = "type"; @@ -49,16 +49,16 @@ public class TableInfoParser { private final static Pattern SIDE_PATTERN = Pattern.compile(SIDE_TABLE_SIGN); - private Map sourceTableInfoMap = Maps.newConcurrentMap(); + private Map sourceTableInfoMap = Maps.newConcurrentMap(); - private Map targetTableInfoMap = Maps.newConcurrentMap(); + private Map targetTableInfoMap = Maps.newConcurrentMap(); - private Map sideTableInfoMap = Maps.newConcurrentMap(); + private Map sideTableInfoMap = Maps.newConcurrentMap(); //Parsing loaded plugin - public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserResult parserResult, - String localPluginRoot) throws Exception { - AbsTableParser absTableParser = null; + public AbstractTableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserResult parserResult, + String localPluginRoot) throws Exception { + AbstractTableParser absTableParser = null; Map props = parserResult.getPropMap(); String type = MathUtil.getString(props.get(TYPE_KEY)); @@ -78,7 +78,7 @@ public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserRe }else{ absTableParser = sideTableInfoMap.get(type); if(absTableParser == null){ - String cacheType = MathUtil.getString(props.get(SideTableInfo.CACHE_KEY)); + String cacheType = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_KEY)); absTableParser = StreamSideFactory.getSqlParser(type, localPluginRoot, cacheType); sideTableInfoMap.put(type, absTableParser); } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java similarity index 90% rename from core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 31e70caa2..52d9d9af5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -39,7 +39,7 @@ * @author xuchao */ -public abstract class AbsTableParser { +public abstract class AbstractTableParser { private static final String PRIMARY_KEY = "primaryKey"; private static final String NEST_JSON_FIELD_KEY = "nestFieldKey"; @@ -52,7 +52,7 @@ public abstract class AbsTableParser { private Map handlerMap = Maps.newHashMap(); - public AbsTableParser() { + public AbstractTableParser() { addParserHandler(PRIMARY_KEY, primaryKeyPattern, this::dealPrimaryKey); addParserHandler(NEST_JSON_FIELD_KEY, nestJsonFieldKeyPattern, this::dealNestField); } @@ -61,9 +61,9 @@ protected boolean fieldNameNeedsUpperCase() { return true; } - public abstract TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception; + public abstract AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception; - public boolean dealKeyPattern(String fieldRow, TableInfo tableInfo){ + public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ for(Map.Entry keyPattern : patternMap.entrySet()){ Pattern pattern = keyPattern.getValue(); String key = keyPattern.getKey(); @@ -82,7 +82,7 @@ public boolean dealKeyPattern(String fieldRow, TableInfo tableInfo){ return false; } - public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ + public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); for(String fieldRow : fieldRows){ @@ -119,7 +119,7 @@ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ tableInfo.finish(); } - public void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ + public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo){ String primaryFields = matcher.group(1).trim(); String[] splitArry = primaryFields.split(","); List primaryKes = Lists.newArrayList(splitArry); @@ -131,7 +131,7 @@ public void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ * @param matcher * @param tableInfo */ - protected void dealNestField(Matcher matcher, TableInfo tableInfo) { + protected void dealNestField(Matcher matcher, AbstractTableInfo tableInfo) { String physicalField = matcher.group(1); Preconditions.checkArgument(!physicalFieldFunPattern.matcher(physicalField).find(), "No need to add data types when using functions, The correct way is : strLen(name) as nameSize, "); @@ -140,7 +140,7 @@ protected void dealNestField(Matcher matcher, TableInfo tableInfo) { String mappingField = matcher.group(4); Class fieldClass = dbTypeConvertToJavaType(fieldType); boolean notNull = matcher.group(5) != null; - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); fieldExtraInfo.setNotNull(notNull); tableInfo.addPhysicalMappings(mappingField, physicalField); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java similarity index 94% rename from core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java index 0dc9ca77b..dbd99f993 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java @@ -29,7 +29,7 @@ * @author xuchao */ -public abstract class TargetTableInfo extends TableInfo { +public abstract class AbstractTargetTableInfo extends AbstractTableInfo { public static final String TARGET_SUFFIX = "Sink"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java b/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java index db804ea34..fab634cdc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java @@ -30,5 +30,5 @@ */ public interface ITableFieldDealHandler { - void dealPrimaryKey(Matcher matcher, TableInfo tableInfo); + void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index da5a42b57..c942fb064 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -39,7 +39,6 @@ */ public class DateUtil { - static final String timeZone = "GMT+8"; static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss"; static final String dateFormat = "yyyy-MM-dd"; static final String timeFormat = "HH:mm:ss"; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java index a63e5a663..8eb743cf0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java @@ -109,7 +109,7 @@ public static String getSqlSideClassName(String pluginTypeName, String type, Str return CLASS_PRE_STR + "." + type.toLowerCase() + "." + pluginTypeName + "." + pluginClassName; } - public static Map ObjectToMap(Object obj) throws Exception{ + public static Map objectToMap(Object obj) throws Exception{ return objectMapper.readValue(objectMapper.writeValueAsBytes(obj), Map.class); } @@ -173,8 +173,8 @@ public static void addPluginJar(String pluginDir, DtClassLoader classLoader) thr } for(File file : files){ - URL pluginJarURL = file.toURI().toURL(); - classLoader.addURL(pluginJarURL); + URL pluginJarUrl = file.toURI().toURL(); + classLoader.addURL(pluginJarUrl); } } @@ -191,8 +191,8 @@ public static URL[] getPluginJarUrls(String pluginDir) throws MalformedURLExcept } for(File file : files){ - URL pluginJarURL = file.toURI().toURL(); - urlList.add(pluginJarURL); + URL pluginJarUrl = file.toURI().toURL(); + urlList.add(pluginJarUrl); } return urlList.toArray(new URL[urlList.size()]); } diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java similarity index 94% rename from core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java rename to core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java index ed8f64288..d75d26a61 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java @@ -38,7 +38,7 @@ * @author xuchao */ -public abstract class AbsCustomerWaterMarker extends BoundedOutOfOrdernessTimestampExtractor implements RichFunction { +public abstract class AbstractCustomerWaterMarker extends BoundedOutOfOrdernessTimestampExtractor implements RichFunction { private static final long serialVersionUID = 1L; @@ -55,7 +55,7 @@ public abstract class AbsCustomerWaterMarker extends BoundedOutOfOrdernessTim protected TimeZone timezone; - public AbsCustomerWaterMarker(Time maxOutOfOrderness) { + public AbstractCustomerWaterMarker(Time maxOutOfOrderness) { super(maxOutOfOrderness); } diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java index 99415ba61..979b4c3d6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java @@ -35,7 +35,7 @@ * @author xuchao */ -public class CustomerWaterMarkerForLong extends AbsCustomerWaterMarker { +public class CustomerWaterMarkerForLong extends AbstractCustomerWaterMarker { private static final Logger logger = LoggerFactory.getLogger(CustomerWaterMarkerForLong.class); diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java index b7961e410..05ee3e46e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java @@ -20,7 +20,6 @@ package com.dtstack.flink.sql.watermarker; -import com.dtstack.flink.sql.util.MathUtil; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -36,7 +35,7 @@ * @author xuchao */ -public class CustomerWaterMarkerForTimeStamp extends AbsCustomerWaterMarker { +public class CustomerWaterMarkerForTimeStamp extends AbstractCustomerWaterMarker { private static final Logger logger = LoggerFactory.getLogger(CustomerWaterMarkerForTimeStamp.class); diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java index f3208db03..b0ca0335f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.watermarker; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.base.Strings; @@ -40,7 +40,7 @@ public class WaterMarkerAssigner { - public boolean checkNeedAssignWaterMarker(SourceTableInfo tableInfo){ + public boolean checkNeedAssignWaterMarker(AbstractSourceTableInfo tableInfo){ if(Strings.isNullOrEmpty(tableInfo.getEventTimeField())){ return false; } @@ -48,7 +48,7 @@ public boolean checkNeedAssignWaterMarker(SourceTableInfo tableInfo){ return true; } - public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo typeInfo, SourceTableInfo sourceTableInfo){ + public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo typeInfo, AbstractSourceTableInfo sourceTableInfo){ String eventTimeFieldName = sourceTableInfo.getEventTimeField(); @@ -75,7 +75,7 @@ public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo type TypeInformation fieldType = fieldTypes[pos]; - AbsCustomerWaterMarker waterMarker = null; + AbstractCustomerWaterMarker waterMarker = null; if(fieldType.getTypeClass().isAssignableFrom(Timestamp.class)){ waterMarker = new CustomerWaterMarkerForTimeStamp(Time.milliseconds(maxOutOrderness), pos,timeZone); }else if(fieldType.getTypeClass().isAssignableFrom(Long.class)){ diff --git a/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java b/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java index 546fcbbac..feec8e547 100644 --- a/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java +++ b/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java @@ -54,7 +54,7 @@ public void testfillPredicatesForSideTable() throws SqlParseException { " MyTable.a='1' and s.d='1' and s.d <> '3' and s.c LIKE '%xx%' and s.c in ('1','2') and s.c between '10' and '23' and s.d is not null\n"; - SideTableInfo sideTableInfo = new SideTableInfo(){ + AbstractSideTableInfo sideTableInfo = new AbstractSideTableInfo(){ @Override public boolean check() { return false; @@ -63,7 +63,7 @@ public boolean check() { sideTableInfo.setName("sideTable"); - Map sideTableMap = new HashMap<>(); + Map sideTableMap = new HashMap<>(); sideTableMap.put("sideTable", sideTableInfo); SidePredicatesParser sidePredicatesParser = new SidePredicatesParser(); diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java index e64e55987..b354a6269 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.slf4j.Logger; @@ -43,15 +43,15 @@ public class Db2AllReqRow extends AbstractRdbAllReqRow { private static final String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; - public Db2AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Db2AllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Class.forName(DB2_DRIVER); - Connection conn = DriverManager.getConnection(dbURL, userName, password); + Connection conn = DriverManager.getConnection(dbUrl, userName, password); return conn; } catch (Exception e) { LOG.error("", e); diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java index 282ee440c..0120c58a0 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class Db2AllSideInfo extends RdbAllSideInfo { - public Db2AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index 59571aa56..240bab9d1 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -48,7 +48,7 @@ public class Db2AsyncReqRow extends RdbAsyncReqRow { private final static String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; - public Db2AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Db2AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java index e53dd7f27..9cbdc997e 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class Db2AsyncSideInfo extends RdbAsyncSideInfo { - public Db2AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java b/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java index db8a07dc6..96be4ec15 100644 --- a/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java +++ b/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.db2.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -36,8 +36,8 @@ public class Db2SideParser extends RdbSideParser { private static final String CURR_TYPE = "db2"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURR_TYPE); return tableInfo; diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java index ba11aabf3..3e823ab20 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java @@ -1,7 +1,7 @@ package com.dtstack.flink.sql.sink.db.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -10,8 +10,8 @@ public class DbSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "db2"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURR_TYPE); return tableInfo; } diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 7ae2fb1ae..8605613d8 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -162,7 +162,7 @@ public void setBulkFlushMaxActions(int bulkFlushMaxActions) { } @Override - public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { + public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { ElasticsearchTableInfo elasticsearchTableInfo = (ElasticsearchTableInfo) targetTableInfo; esTableInfo = elasticsearchTableInfo; clusterName = elasticsearchTableInfo.getClusterName(); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 93682de46..7988e597c 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -32,7 +32,7 @@ * @author sishu.yss * @Company: www.dtstack.com */ -public class ElasticsearchSinkParser extends AbsTableParser { +public class ElasticsearchSinkParser extends AbstractTableParser { private static final String KEY_ES_ADDRESS = "address"; @@ -56,7 +56,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ElasticsearchTableInfo elasticsearchTableInfo = new ElasticsearchTableInfo(); elasticsearchTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearchTableInfo); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index ac1d712ce..b6b6e0102 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; /** @@ -29,7 +29,7 @@ * @author sishu.yss * @Company: www.dtstack.com */ -public class ElasticsearchTableInfo extends TargetTableInfo { +public class ElasticsearchTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "elasticsearch"; diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 11380eea6..6f7720472 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -45,7 +45,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicReference; -public class HbaseAllReqRow extends AllReqRow { +public class HbaseAllReqRow extends BaseAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(HbaseAllReqRow.class); @@ -55,7 +55,7 @@ public class HbaseAllReqRow extends AllReqRow { private AtomicReference>> cacheRef = new AtomicReference<>(); - public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new HbaseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); tableName = ((HbaseSideTableInfo)sideTableInfo).getTableName(); @@ -134,7 +134,7 @@ public void flatMap(CRow input, Collector out) throws Exception { Map cacheList = null; - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; if (hbaseSideTableInfo.isPreRowKey()) { for (Map.Entry> entry : cacheRef.get().entrySet()) { @@ -153,7 +153,7 @@ public void flatMap(CRow input, Collector out) throws Exception { } private void loadData(Map> tmpCache) throws SQLException { - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; Configuration conf = new Configuration(); conf.set("hbase.zookeeper.quorum", hbaseSideTableInfo.getHost()); diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java index ea51f46e4..663c2927b 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,16 +31,16 @@ import java.util.List; -public class HbaseAllSideInfo extends SideInfo { +public class HbaseAllSideInfo extends BaseSideInfo { private RowKeyBuilder rowKeyBuilder; - public HbaseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { rowKeyBuilder = new RowKeyBuilder(); if(sideTableInfo.getPrimaryKeys().size() < 1){ throw new RuntimeException("Primary key dimension table must be filled"); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 251b88034..24ee01786 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -21,12 +21,12 @@ package com.dtstack.flink.sql.side.hbase; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; -import com.dtstack.flink.sql.side.hbase.rowkeydealer.AbsRowKeyModeDealer; +import com.dtstack.flink.sql.side.hbase.rowkeydealer.AbstractRowKeyModeDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.PreRowKeyModeDealerDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.RowKeyEqualModeDealer; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; @@ -58,7 +58,7 @@ * @author xuchao */ -public class HbaseAsyncReqRow extends AsyncReqRow { +public class HbaseAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 2098635104857937717L; @@ -73,13 +73,13 @@ public class HbaseAsyncReqRow extends AsyncReqRow { private transient HBaseClient hBaseClient; - private transient AbsRowKeyModeDealer rowKeyMode; + private transient AbstractRowKeyModeDealer rowKeyMode; private String tableName; private String[] colNames; - public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new HbaseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); tableName = ((HbaseSideTableInfo)sideTableInfo).getTableName(); @@ -89,7 +89,7 @@ public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List colRefType; - public HbaseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { rowKeyBuilder = new RowKeyBuilder(); if(sideTableInfo.getPrimaryKeys().size() < 1){ throw new RuntimeException("Primary key dimension table must be filled"); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java similarity index 87% rename from hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java rename to hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java index 1506bc440..90ee289bd 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.side.hbase.rowkeydealer; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import org.apache.calcite.sql.JoinType; import com.google.common.collect.Maps; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -41,7 +41,7 @@ * @author xuchao */ -public abstract class AbsRowKeyModeDealer { +public abstract class AbstractRowKeyModeDealer { protected Map colRefType; @@ -60,9 +60,9 @@ public abstract class AbsRowKeyModeDealer { protected Map sideFieldIndex = Maps.newHashMap(); - public AbsRowKeyModeDealer(Map colRefType, String[] colNames, HBaseClient hBaseClient, - boolean openCache, JoinType joinType, List outFieldInfoList, - Map inFieldIndex, Map sideFieldIndex){ + public AbstractRowKeyModeDealer(Map colRefType, String[] colNames, HBaseClient hBaseClient, + boolean openCache, JoinType joinType, List outFieldInfoList, + Map inFieldIndex, Map sideFieldIndex){ this.colRefType = colRefType; this.colNames = colNames; this.hBaseClient = hBaseClient; @@ -111,5 +111,5 @@ protected Row fillData(Row input, Object sideInput){ } public abstract void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbsSideCache sideCache); + AbstractSideCache sideCache); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java index a41ffe916..3fad216b2 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.google.common.collect.Maps; @@ -44,7 +44,6 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -55,7 +54,7 @@ * @author xuchao */ -public class PreRowKeyModeDealerDealer extends AbsRowKeyModeDealer { +public class PreRowKeyModeDealerDealer extends AbstractRowKeyModeDealer { private static final Logger LOG = LoggerFactory.getLogger(PreRowKeyModeDealerDealer.class); @@ -67,7 +66,7 @@ public PreRowKeyModeDealerDealer(Map colRefType, String[] colNam @Override public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbsSideCache sideCache) { + AbstractSideCache sideCache) { Scanner prefixScanner = hBaseClient.newScanner(tableName); ScanFilter scanFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(Bytes.UTF8(rowKeyStr))); prefixScanner.setFilter(scanFilter); @@ -80,7 +79,7 @@ public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultF } - private String dealOneRow(ArrayList> args, String rowKeyStr, CRow input, ResultFuture resultFuture, AbsSideCache sideCache) { + private String dealOneRow(ArrayList> args, String rowKeyStr, CRow input, ResultFuture resultFuture, AbstractSideCache sideCache) { if(args == null || args.size() == 0){ dealMissKey(input, resultFuture); if (openCache) { diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index b20c316db..01f43b246 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.google.common.collect.Maps; @@ -49,7 +49,7 @@ * @author xuchao */ -public class RowKeyEqualModeDealer extends AbsRowKeyModeDealer { +public class RowKeyEqualModeDealer extends AbstractRowKeyModeDealer { private static final Logger LOG = LoggerFactory.getLogger(RowKeyEqualModeDealer.class); @@ -62,7 +62,7 @@ public RowKeyEqualModeDealer(Map colRefType, String[] colNames, @Override public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbsSideCache sideCache){ + AbstractSideCache sideCache){ //TODO 是否有查询多个col family 和多个col的方法 GetRequest getRequest = new GetRequest(tableName, rowKeyStr); hBaseClient.get(getRequest).addCallbacks(arg -> { diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java similarity index 93% rename from hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java rename to hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java index c3c9b5cfe..c75fa43c1 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java @@ -31,11 +31,11 @@ * @author xuchao */ -public abstract class ReplaceOperator { +public abstract class AbstractReplaceOperator { private EReplaceOpType opType; - public ReplaceOperator(EReplaceOpType opType){ + public AbstractReplaceOperator(EReplaceOpType opType){ this.opType = opType; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java index af5e24dd6..ab826c98f 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java @@ -30,7 +30,7 @@ * @author xuchao */ -public class Md5ReplaceOperator extends ReplaceOperator{ +public class Md5ReplaceOperator extends AbstractReplaceOperator { public Md5ReplaceOperator(EReplaceOpType opType) { super(opType); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java index 834edf9ae..ac9a3b87c 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java @@ -28,5 +28,12 @@ */ public enum EReplaceOpType { - NO_FUNC, MD5_FUNC; + /** + * 没有func + */ + NO_FUNC, + /** + * md5 func + */ + MD5_FUNC; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java index 3768d7c5e..1ebb06216 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java @@ -28,6 +28,12 @@ * @author xuchao */ public enum EReplaceType { + /** + * 参数 + */ PARAM, + /** + * 常量 + */ CONSTANT; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 7b627da4d..cf44c181f 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -20,15 +20,15 @@ package com.dtstack.flink.sql.side.hbase.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * hbase field information must include the definition of an alias -> sql which does not allow ":" @@ -37,7 +37,7 @@ * @author xuchao */ -public class HbaseSideParser extends AbsSideTableParser { +public class HbaseSideParser extends AbstractSideTableParser { private final static String FIELD_KEY = "fieldKey"; @@ -58,7 +58,7 @@ public HbaseSideParser() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseSideTableInfo hbaseTableInfo = new HbaseSideTableInfo(); hbaseTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, hbaseTableInfo); @@ -77,7 +77,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map { +public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(HbaseOutputFormat.class); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index 447b02921..9308725a2 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.hbase.table.HbaseTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -30,7 +30,6 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.apache.flink.table.sinks.AppendStreamTableSink; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; @@ -59,7 +58,7 @@ public HbaseSink() { } @Override - public HbaseSink genStreamSink(TargetTableInfo targetTableInfo) { + public HbaseSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { HbaseTableInfo hbaseTableInfo = (HbaseTableInfo) targetTableInfo; this.zookeeperQuorum = hbaseTableInfo.getHost(); this.port = hbaseTableInfo.getPort(); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index f1ae72b6b..d09f54128 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.enums.EUpdateMode; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.MathUtil; @@ -31,14 +31,14 @@ import java.util.List; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Date: 2018/09/14 * Company: www.dtstack.com * @author sishu.yss */ -public class HbaseSinkParser extends AbsTableParser { +public class HbaseSinkParser extends AbstractTableParser { public static final String HBASE_ZOOKEEPER_QUORUM = "zookeeperQuorum"; @@ -59,7 +59,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseTableInfo hbaseTableInfo = new HbaseTableInfo(); hbaseTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, hbaseTableInfo); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java index 610eb5c30..62f7c9936 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.sink.hbase.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import java.util.Map; @@ -30,7 +30,7 @@ * Company: www.dtstack.com * @author sishu.yss */ -public class HbaseTableInfo extends TargetTableInfo { +public class HbaseTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "hbase"; diff --git a/hbase/pom.xml b/hbase/pom.xml index 835457e29..f64ffab1b 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -29,6 +29,13 @@ 1.0-SNAPSHOT provided + + jdk.tools + jdk.tools + 1.6 + system + ${JAVA_HOME}/lib/tools.jar + org.apache.hbase diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java index 4118421ee..10938308a 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.JDBCUtils; @@ -53,13 +53,13 @@ public class ImpalaAllReqRow extends AbstractRdbAllReqRow { private ImpalaSideTableInfo impalaSideTableInfo; - public ImpalaAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ImpalaAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); this.impalaSideTableInfo = (ImpalaSideTableInfo) sideTableInfo; } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Connection connection ; String url = getUrl(); diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java index d40c5f48c..a5e643967 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java @@ -20,10 +20,9 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; -import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -32,7 +31,7 @@ public class ImpalaAllSideInfo extends RdbAllSideInfo { - public ImpalaAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -61,7 +60,7 @@ private String buildPartitionCondition(ImpalaSideTableInfo impalaSideTableInfo) private String getPartitionVaule(String fieldType, List values) { String partitionVaule = values.stream().map(val -> { - return (fieldType.toLowerCase().equals("string") || fieldType.toLowerCase().equals("varchar")) ? "'" + val + "'" : val.toString(); + return ("string".equals(fieldType.toLowerCase()) || "varchar".equals(fieldType.toLowerCase())) ? "'" + val + "'" : val.toString(); }).collect(Collectors.joining(" , ")).toString(); return partitionVaule; diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 1b983ca2c..77d3f3007 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import io.vertx.core.Vertx; @@ -50,7 +50,7 @@ public class ImpalaAsyncReqRow extends RdbAsyncReqRow { private final static String IMPALA_DRIVER = "com.cloudera.impala.jdbc41.Driver"; - public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ImpalaAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java index 53559b9a7..14fbf0ed3 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -38,7 +38,7 @@ public class ImpalaAsyncSideInfo extends RdbAsyncSideInfo { - public ImpalaAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -69,7 +69,7 @@ private String buildPartitionCondition(ImpalaSideTableInfo impalaSideTableInfo) private String getPartitionVaule(String fieldType, List values) { String partitionVaule = values.stream().map(val -> { - return (fieldType.toLowerCase().equals("string") || fieldType.toLowerCase().equals("varchar")) ? "'" + val + "'" : val.toString(); + return ("string".equals(fieldType.toLowerCase()) || "varchar".equals(fieldType.toLowerCase())) ? "'" + val + "'" : val.toString(); }).collect(Collectors.joining(" , ")).toString(); return partitionVaule; diff --git a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java index 344037124..10a13ec72 100644 --- a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java +++ b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java @@ -19,14 +19,13 @@ package com.dtstack.flink.sql.side.impala.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.math.BigDecimal; -import java.sql.Date; import java.sql.Timestamp; import java.util.Arrays; import java.util.HashMap; @@ -48,7 +47,7 @@ public class ImpalaSideParser extends RdbSideParser { private static final String CURR_TYPE = "impala"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ImpalaSideTableInfo impalaSideTableInfo = new ImpalaSideTableInfo(); impalaSideTableInfo.setType(CURR_TYPE); impalaSideTableInfo.setName(tableName); @@ -147,6 +146,8 @@ public Class dbTypeConvertToJavaType(String fieldType) { return String.class; case "timestamp": return Timestamp.class; + default: + break; } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index 91b6532bf..cd1e1e945 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -121,7 +121,7 @@ public String getImpalaJdbcUrl() { } @Override - public AbstractRdbSink genStreamSink(TargetTableInfo targetTableInfo) { + public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { super.genStreamSink(targetTableInfo); this.impalaTableInfo = (ImpalaTableInfo) targetTableInfo; return this; diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java index 4921f5e51..5b6fdeafe 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java @@ -19,12 +19,11 @@ package com.dtstack.flink.sql.sink.impala.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; import java.sql.Timestamp; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -42,7 +41,7 @@ public class ImpalaSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "impala"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ImpalaTableInfo impalaTableInfo = new ImpalaTableInfo(); impalaTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, impalaTableInfo); @@ -117,6 +116,8 @@ public Class dbTypeConvertToJavaType(String fieldType) { return String.class; case "timestamp": return Timestamp.class; + default: + break; } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index 4fb567f82..c6cce594c 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.sink.kafka.table; import com.dtstack.flink.sql.format.FormatType; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -32,9 +32,9 @@ * @author DocLi * @modifyer maqi */ -public class KafkaSinkParser extends AbsTableParser { +public class KafkaSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KafkaSinkTableInfo kafkaSinkTableInfo = new KafkaSinkTableInfo(); kafkaSinkTableInfo.setName(tableName); kafkaSinkTableInfo.setType(MathUtil.getString(props.get(KafkaSinkTableInfo.TYPE_KEY.toLowerCase()))); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java index 6ac728ee0..ec554d5f1 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java @@ -18,8 +18,7 @@ package com.dtstack.flink.sql.sink.kafka.table; -import com.dtstack.flink.sql.format.FormatType; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import java.util.HashMap; @@ -33,7 +32,7 @@ * @author DocLi * @modifyer maqi */ -public class KafkaSinkTableInfo extends TargetTableInfo { +public class KafkaSinkTableInfo extends AbstractTargetTableInfo { public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 10049cbbe..8b0b9753c 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.source.kafka.table; -import com.dtstack.flink.sql.table.AbsSourceParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSourceParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -32,9 +32,9 @@ * * @author sishu.yss */ -public class KafkaSourceParser extends AbsSourceParser { +public class KafkaSourceParser extends AbstractSourceParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { KafkaSourceTableInfo kafkaSourceTableInfo = new KafkaSourceTableInfo(); kafkaSourceTableInfo.setName(tableName); @@ -43,7 +43,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map, IStreamSinkGener< protected Optional> partitioner; @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafkaSinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; this.topic = kafkaSinkTableInfo.getTopic(); diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index a445a6529..7e2d77c27 100644 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; @@ -56,7 +56,7 @@ public class KafkaSource implements IStreamSourceGener { */ @SuppressWarnings("rawtypes") @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); @@ -91,7 +91,7 @@ public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnv } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} try { Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); + Map offsetMap = PluginUtil.objectToMap(properties); Map specificStartupOffsets = new HashMap<>(); for (Map.Entry entry : offsetMap.entrySet()) { specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 6d39536a5..d8c66cbc8 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -68,7 +68,7 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka09SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; this.topic = kafka09SinkTableInfo.getTopic(); diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 21ed7c3e3..ffb466a4c 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; @@ -59,7 +59,7 @@ public class KafkaSource implements IStreamSourceGener
{ */ @SuppressWarnings("rawtypes") @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); @@ -94,7 +94,7 @@ public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnv }else if(DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){// {"0":12312,"1":12321,"2":12312} try { Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); + Map offsetMap = PluginUtil.objectToMap(properties); Map specificStartupOffsets = new HashMap<>(); for(Map.Entry entry:offsetMap.entrySet()){ specificStartupOffsets.put(new KafkaTopicPartition(topicName,Integer.valueOf(entry.getKey())),Long.valueOf(entry.getValue().toString())); diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index dee05ab08..67df2159c 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -69,7 +69,7 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< protected Optional> partitioner; @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; this.topic = kafka10SinkTableInfo.getTopic(); diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 55cffbf1e..ff4aed89d 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; @@ -60,7 +60,7 @@ public class KafkaSource implements IStreamSourceGener
{ */ @SuppressWarnings("rawtypes") @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); @@ -94,7 +94,7 @@ public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnv } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} try { Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); + Map offsetMap = PluginUtil.objectToMap(properties); Map specificStartupOffsets = new HashMap<>(); for (Map.Entry entry : offsetMap.entrySet()) { specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index c2ef7d35a..a3533b7aa 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -70,7 +70,7 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka11SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; this.topic = kafka11SinkTableInfo.getTopic(); diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 1a4eed08a..11be1898a 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; @@ -60,7 +60,7 @@ public class KafkaSource implements IStreamSourceGener
{ */ @SuppressWarnings("rawtypes") @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); @@ -96,7 +96,7 @@ public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnv } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} try { Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); + Map offsetMap = PluginUtil.objectToMap(properties); Map specificStartupOffsets = new HashMap<>(); for (Map.Entry entry : offsetMap.entrySet()) { specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index cf2c70f1f..45b312ffa 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -1,10 +1,10 @@ package com.dtstack.flink.sql.side.kudu; -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; import org.apache.calcite.sql.JoinType; @@ -39,7 +39,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; -public class KuduAllReqRow extends AllReqRow { +public class KuduAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 6051774809356082219L; @@ -60,7 +60,7 @@ public class KuduAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public KuduAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new KuduAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java index ddfa73201..1241a6f37 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java @@ -2,8 +2,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; @@ -12,16 +12,16 @@ import java.util.List; -public class KuduAllSideInfo extends SideInfo { +public class KuduAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = 9005389633060174746L; - public KuduAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { // no use } diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index 12689a07b..1cdc75e1b 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -29,7 +29,7 @@ import java.util.List; import java.util.Map; -public class KuduAsyncReqRow extends AsyncReqRow { +public class KuduAsyncReqRow extends BaseAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(KuduAsyncReqRow.class); /** @@ -52,7 +52,7 @@ public class KuduAsyncReqRow extends AsyncReqRow { private AsyncKuduScanner.AsyncKuduScannerBuilder scannerBuilder; - public KuduAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new KuduAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java index de75f23fb..5ace515f7 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java @@ -2,8 +2,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; @@ -12,15 +12,15 @@ import java.util.List; -public class KuduAsyncSideInfo extends SideInfo { +public class KuduAsyncSideInfo extends BaseSideInfo { - public KuduAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } @Override diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java index 83864fa22..62d215d87 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java @@ -1,7 +1,7 @@ package com.dtstack.flink.sql.side.kudu.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; @@ -9,9 +9,9 @@ import java.sql.Timestamp; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; -public class KuduSideParser extends AbsSideTableParser { +public class KuduSideParser extends AbstractSideTableParser { public static final String KUDU_MASTERS = "kuduMasters"; @@ -52,7 +52,7 @@ public class KuduSideParser extends AbsSideTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KuduSideTableInfo kuduSideTableInfo = new KuduSideTableInfo(); kuduSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, kuduSideTableInfo); diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java index c527ec268..eb8df5ad9 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java @@ -1,9 +1,9 @@ package com.dtstack.flink.sql.side.kudu.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; -public class KuduSideTableInfo extends SideTableInfo { +public class KuduSideTableInfo extends AbstractSideTableInfo { private static final String CURR_TYPE = "kudu"; diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index 780319d82..c211c7a24 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.kudu; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; @@ -42,7 +42,7 @@ * @author gituser * @modify xiuzhu */ -public class KuduOutputFormat extends DtRichOutputFormat { +public class KuduOutputFormat extends AbstractDtRichOutputFormat { private static final long serialVersionUID = 1L; diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java index 0c90ffe94..4c7c3bea8 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java @@ -3,7 +3,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kudu.table.KuduTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -39,7 +39,7 @@ public class KuduSink implements RetractStreamTableSink, Serializable, IStr private int parallelism = -1; @Override - public KuduSink genStreamSink(TargetTableInfo targetTableInfo) { + public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KuduTableInfo kuduTableInfo = (KuduTableInfo) targetTableInfo; this.kuduMasters = kuduTableInfo.getKuduMasters(); this.tableName = kuduTableInfo.getTableName(); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java index e38a85390..20302d44f 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java @@ -1,8 +1,8 @@ package com.dtstack.flink.sql.sink.kudu.table; import com.dtstack.flink.sql.sink.kudu.KuduOutputFormat; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; @@ -10,9 +10,9 @@ import java.sql.Timestamp; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; -public class KuduSinkParser extends AbsTableParser { +public class KuduSinkParser extends AbstractTableParser { public static final String KUDU_MASTERS = "kuduMasters"; @@ -27,7 +27,7 @@ public class KuduSinkParser extends AbsTableParser { public static final String SOCKET_READ_TIMEOUT_MS = "defaultSocketReadTimeoutMs"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KuduTableInfo kuduTableInfo = new KuduTableInfo(); kuduTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, kuduTableInfo); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java index c086c7298..80e47761b 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java @@ -1,10 +1,10 @@ package com.dtstack.flink.sql.sink.kudu.table; import com.dtstack.flink.sql.sink.kudu.KuduOutputFormat; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; -public class KuduTableInfo extends TargetTableInfo { +public class KuduTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "kudu"; diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index 3229101c6..79f5bd9ca 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -18,10 +18,10 @@ package com.dtstack.flink.sql.side.mongo; -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.mongodb.BasicDBObject; @@ -57,7 +57,7 @@ * * @author xuqianjin */ -public class MongoAllReqRow extends AllReqRow { +public class MongoAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = -675332795591842778L; @@ -73,7 +73,7 @@ public class MongoAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public MongoAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MongoAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java index b2d5c2686..4a33f1a69 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; @@ -36,16 +36,16 @@ * * @author xuqianjin */ -public class MongoAllSideInfo extends SideInfo{ +public class MongoAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = -1696860430075523841L; - public MongoAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } @Override diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 5ef06c80e..491f62922 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -20,27 +20,22 @@ package com.dtstack.flink.sql.side.mongo; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.mongodb.BasicDBObject; import com.mongodb.Block; import com.mongodb.ConnectionString; -import com.mongodb.MongoCredential; -import com.mongodb.ServerAddress; import com.mongodb.async.SingleResultCallback; import com.mongodb.async.client.MongoClient; import com.mongodb.MongoClientSettings; import com.mongodb.async.client.MongoClients; import com.mongodb.async.client.MongoCollection; import com.mongodb.async.client.MongoDatabase; -import com.mongodb.connection.ClusterSettings; -import com.mongodb.connection.ConnectionPoolSettings; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import org.apache.flink.configuration.Configuration; @@ -53,7 +48,6 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -66,7 +60,7 @@ * * @author xuqianjin */ -public class MongoAsyncReqRow extends AsyncReqRow { +public class MongoAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = -1183158242862673706L; private static final Logger LOG = LoggerFactory.getLogger(MongoAsyncReqRow.class); @@ -77,7 +71,7 @@ public class MongoAsyncReqRow extends AsyncReqRow { private MongoSideTableInfo mongoSideTableInfo; - public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MongoAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java index de81ed1f1..19534d4e1 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java @@ -20,16 +20,13 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; -import com.dtstack.flink.sql.util.ParseUtils; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; import java.util.List; @@ -39,16 +36,16 @@ * * @author xuqianjin */ -public class MongoAsyncSideInfo extends SideInfo { +public class MongoAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = -3694857194254465989L; - public MongoAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java index 5edc12419..4fe1ebee1 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java @@ -19,15 +19,13 @@ package com.dtstack.flink.sql.side.mongo.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -35,7 +33,7 @@ * * @author xuqianjin */ -public class MongoSideParser extends AbsSideTableParser { +public class MongoSideParser extends AbstractSideTableParser { public static final String ADDRESS_KEY = "address"; @@ -48,7 +46,7 @@ public class MongoSideParser extends AbsSideTableParser { public static final String DATABASE_KEY = "database"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { MongoSideTableInfo mongoSideTableInfo = new MongoSideTableInfo(); mongoSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, mongoSideTableInfo); diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java index a5c834469..c83de7bbc 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.mongo.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; /** @@ -28,7 +28,7 @@ * * @author xuqianjin */ -public class MongoSideTableInfo extends SideTableInfo { +public class MongoSideTableInfo extends AbstractSideTableInfo { private static final long serialVersionUID = -1L; diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index daaded15e..03ef8f3b3 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.mongo; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.mongodb.MongoClient; import com.mongodb.MongoClientURI; import com.mongodb.client.MongoCollection; @@ -41,7 +41,7 @@ * * @author xuqianjin */ -public class MongoOutputFormat extends DtRichOutputFormat { +public class MongoOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(MongoOutputFormat.class); private String address; diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java index 4e28d8fd2..3f34055ec 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.mongo.table.MongoTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -54,7 +54,7 @@ public MongoSink() { } @Override - public MongoSink genStreamSink(TargetTableInfo targetTableInfo) { + public MongoSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { MongoTableInfo mongoTableInfo = (MongoTableInfo) targetTableInfo; this.address = mongoTableInfo.getAddress(); this.tableName = mongoTableInfo.getTableName(); diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java index d4810fb6d..f093b70aa 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java @@ -19,13 +19,13 @@ package com.dtstack.flink.sql.sink.mongo.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -34,7 +34,7 @@ * @author xuqianjin */ -public class MongoSinkParser extends AbsTableParser { +public class MongoSinkParser extends AbstractTableParser { private static final String CURR_TYPE = "mongo"; @@ -49,7 +49,7 @@ public class MongoSinkParser extends AbsTableParser { public static final String DATABASE_KEY = "database"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { MongoTableInfo mongoTableInfo = new MongoTableInfo(); mongoTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, mongoTableInfo); diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java index 02a96d6bb..c9d94dfe6 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.mongo.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; /** @@ -30,7 +30,7 @@ */ -public class MongoTableInfo extends TargetTableInfo { +public class MongoTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "mongo"; diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java index 8cf6ab597..eb294e64e 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -49,7 +49,7 @@ public class MysqlAllReqRow extends AbstractRdbAllReqRow { private static final String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; - public MysqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MysqlAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java index 81193c0f9..89a62a5ef 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class MysqlAllSideInfo extends RdbAllSideInfo { - public MysqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index 2d2f5a72a..290f02e2b 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -49,7 +49,7 @@ public class MysqlAsyncReqRow extends RdbAsyncReqRow { private final static String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; - public MysqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MysqlAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java index f72671ce0..8fe10d06d 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -36,7 +36,7 @@ public class MysqlAsyncSideInfo extends RdbAsyncSideInfo { - public MysqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java index 40f68e7e4..b0a38e344 100644 --- a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java +++ b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.mysql.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -37,8 +37,8 @@ public class MysqlSideParser extends RdbSideParser { private static final String CURR_TYPE = "mysql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java index 2247eb8cc..49105a7a8 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.mysql.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -36,8 +36,8 @@ public class MysqlSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "mysql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java index 5477514a5..3be6687ec 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -42,7 +42,7 @@ public class OracleAllReqRow extends AbstractRdbAllReqRow { private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; - public OracleAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new OracleAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java index 361366929..342533681 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java @@ -19,18 +19,17 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; public class OracleAllSideInfo extends RdbAllSideInfo { - public OracleAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index f6827f41b..c0b37e7ac 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -42,7 +42,7 @@ public class OracleAsyncReqRow extends RdbAsyncReqRow { private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; - public OracleAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new OracleAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java index b811cf783..8c620b013 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java @@ -20,22 +20,18 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.ParseUtils; -import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import java.util.Arrays; import java.util.List; public class OracleAsyncSideInfo extends RdbAsyncSideInfo { - public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java b/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java index f9124bd10..897b77d30 100644 --- a/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java +++ b/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side.oracle.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -27,8 +27,8 @@ public class OracleSideParser extends RdbSideParser { private static final String CURR_TYPE = "oracle"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); oracleTableInfo.setType(CURR_TYPE); return oracleTableInfo; } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java index aff096bd3..1732c8cc2 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.oracle.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -33,8 +33,8 @@ public class OracleSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "oracle"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); oracleTableInfo.setType(CURR_TYPE); return oracleTableInfo; } diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java index a5eec511b..5fbf0824b 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.collect.Maps; @@ -45,7 +45,7 @@ public class PolardbAllReqRow extends AbstractRdbAllReqRow { private static final String POLARDB_DRIVER = "com.mysql.cj.jdbc.Driver"; - public PolardbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PolardbAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java index fa2ad63e7..8f30c8dd3 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,7 +31,7 @@ * @author yinxi */ public class PolardbAllSideInfo extends RdbAllSideInfo { - public PolardbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 0f8f82738..286f17286 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -44,7 +44,7 @@ public class PolardbAsyncReqRow extends RdbAsyncReqRow { private final static String POLARDB_DRIVER = "com.mysql.cj.jdbc.Driver"; - public PolardbAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PolardbAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java index f6afb8da8..b52d52390 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -32,7 +32,7 @@ */ public class PolardbAsyncSideInfo extends RdbAsyncSideInfo { - public PolardbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java index 1afb661f0..71aad7cbb 100644 --- a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java +++ b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.polardb.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -32,8 +32,8 @@ public class PolardbSideParser extends RdbSideParser { private static final String CURR_TYPE = "polardb"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java index a57089941..b4f02665e 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.polardb.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -31,8 +31,8 @@ public class PolardbSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "polardb"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo polardbTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo polardbTableInfo = super.getTableInfo(tableName, fieldsInfo, props); polardbTableInfo.setType(CURR_TYPE); return polardbTableInfo; } diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java index 1c9aecfe1..0ec8f96ce 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -49,7 +49,7 @@ public class PostgresqlAllReqRow extends AbstractRdbAllReqRow { private static final String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - public PostgresqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PostgresqlAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java index d383ee46d..a4a52eabc 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class PostgresqlAllSideInfo extends RdbAllSideInfo { - public PostgresqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index a26c8b229..2a9e0137c 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -48,7 +48,7 @@ public class PostgresqlAsyncReqRow extends RdbAsyncReqRow { private final static String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - public PostgresqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PostgresqlAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java index 1d89f4894..f19a488bc 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -36,7 +36,7 @@ public class PostgresqlAsyncSideInfo extends RdbAsyncSideInfo { - public PostgresqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java b/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java index faee2c704..be0c5ea8b 100644 --- a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java +++ b/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.postgresql.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -37,8 +37,8 @@ public class PostgresqlSideParser extends RdbSideParser { private static final String CURR_TYPE = "postgresql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); pgTableInfo.setType(CURR_TYPE); return pgTableInfo; } diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java index 5943b5e52..e94cfc8a0 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.postgresql.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -35,8 +35,8 @@ public class PostgresqlSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "postgresql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); pgTableInfo.setType(CURR_TYPE); return pgTableInfo; } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index aeb88f7a3..e2452f73b 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.side.rdb.all; -import com.dtstack.flink.sql.side.AllReqRow; -import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.BaseAllReqRow; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import org.apache.calcite.sql.JoinType; @@ -51,7 +51,7 @@ * @author maqi */ -public abstract class AbstractRdbAllReqRow extends AllReqRow { +public abstract class AbstractRdbAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 2098635140857937718L; @@ -61,7 +61,7 @@ public abstract class AbstractRdbAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public AbstractRdbAllReqRow(SideInfo sideInfo) { + public AbstractRdbAllReqRow(BaseSideInfo sideInfo) { super(sideInfo); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 619d08529..1751b12aa 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -21,8 +21,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; @@ -43,17 +43,17 @@ * @author maqi */ -public class RdbAllSideInfo extends SideInfo { +public class RdbAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = -5858335638589472159L; - public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideTableInfo; sqlCondition = getSelectFromStatement(getTableName(rdbSideTableInfo), Arrays.asList(sideSelectFields.split(",")), sideTableInfo.getPredicateInfoes()); System.out.println("--------dimension sql query-------\n" + sqlCondition); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index bf7f8f774..f78c30c30 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -20,9 +20,9 @@ package com.dtstack.flink.sql.side.rdb.async; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; -import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import io.vertx.core.json.JsonArray; @@ -47,7 +47,7 @@ * @author maqi */ -public class RdbAsyncReqRow extends AsyncReqRow { +public class RdbAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 2098635244857937720L; @@ -69,7 +69,7 @@ public class RdbAsyncReqRow extends AsyncReqRow { private transient SQLClient rdbSqlClient; - public RdbAsyncReqRow(SideInfo sideInfo) { + public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java index 0fe3eb0b9..d79d27460 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java @@ -21,8 +21,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlBasicCall; @@ -34,7 +34,6 @@ import java.util.Arrays; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; @@ -46,16 +45,16 @@ * @author maqi */ -public class RdbAsyncSideInfo extends SideInfo { +public class RdbAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = 1942629132469918611L; - public RdbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RdbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideTableInfo; String sideTableName = joinInfo.getSideTableName(); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java index b9811b0ee..79942ae11 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.side.rdb.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -33,10 +33,10 @@ * @author maqi */ -public class RdbSideParser extends AbsSideTableParser { +public class RdbSideParser extends AbstractSideTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RdbSideTableInfo rdbTableInfo = new RdbSideTableInfo(); rdbTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, rdbTableInfo); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java index 0dfbef325..506033007 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java @@ -17,7 +17,7 @@ */ package com.dtstack.flink.sql.side.rdb.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; /** @@ -27,7 +27,7 @@ * * @author maqi */ -public class RdbSideTableInfo extends SideTableInfo { +public class RdbSideTableInfo extends AbstractSideTableInfo { private static final long serialVersionUID = -1L; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java index b7990f9f4..bdc7e3dbf 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -85,7 +85,7 @@ public AbstractRdbSink(JDBCDialect jdbcDialect) { } @Override - public AbstractRdbSink genStreamSink(TargetTableInfo targetTableInfo) { + public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { RdbTableInfo rdbTableInfo = (RdbTableInfo) targetTableInfo; this.batchNum = rdbTableInfo.getBatchSize() == null ? batchNum : rdbTableInfo.getBatchSize(); this.batchWaitInterval = rdbTableInfo.getBatchWaitInterval() == null ? diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java index 3378ff3f2..2a3ce5e90 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.format; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; @@ -36,7 +36,7 @@ * @see Row * @see DriverManager */ -public abstract class AbstractJDBCOutputFormat extends DtRichOutputFormat { +public abstract class AbstractJDBCOutputFormat extends AbstractDtRichOutputFormat { private static final long serialVersionUID = 1L; public static final int DEFAULT_FLUSH_MAX_SIZE = 100; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java index 9bbff3b4d..aca7acb6c 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java @@ -17,9 +17,8 @@ */ package com.dtstack.flink.sql.sink.rdb.table; -import com.dtstack.flink.sql.constrant.ConfigConstrant; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -31,9 +30,9 @@ * * @author maqi */ -public class RdbSinkParser extends AbsTableParser { +public class RdbSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RdbTableInfo rdbTableInfo = new RdbTableInfo(); rdbTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, rdbTableInfo); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java index d5ad4eab6..cf185308c 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java @@ -18,12 +18,10 @@ package com.dtstack.flink.sql.sink.rdb.table; import com.dtstack.flink.sql.enums.EUpdateMode; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; -import java.util.Arrays; - /** * Reason: * Date: 2018/11/27 @@ -31,7 +29,7 @@ * * @author maqi */ -public class RdbTableInfo extends TargetTableInfo { +public class RdbTableInfo extends AbstractTargetTableInfo { public static final String URL_KEY = "url"; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java index 373a59f6a..a2a915bb1 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.writer; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; @@ -59,7 +59,7 @@ public static AbstractUpsertWriter create( String[] partitionFields, boolean objectReuse, boolean allReplace, - DtRichOutputFormat metricOutputFormat) { + AbstractDtRichOutputFormat metricOutputFormat) { checkNotNull(keyFields); @@ -97,9 +97,9 @@ public static AbstractUpsertWriter create( private transient Map> keyToRows; private transient PreparedStatement deleteStatement; // only use metric - private transient DtRichOutputFormat metricOutputFormat; + private transient AbstractDtRichOutputFormat metricOutputFormat; - private AbstractUpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSql, boolean objectReuse, DtRichOutputFormat metricOutputFormat) { + private AbstractUpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSql, boolean objectReuse, AbstractDtRichOutputFormat metricOutputFormat) { this.fieldTypes = fieldTypes; this.pkFields = pkFields; this.pkTypes = pkTypes; @@ -217,7 +217,7 @@ private UpsertWriterUsingUpsertStatement( boolean objectReuse, String deleteSql, String upsertSql, - DtRichOutputFormat metricOutputFormat) { + AbstractDtRichOutputFormat metricOutputFormat) { super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); this.upsertSql = upsertSql; } @@ -275,7 +275,7 @@ private UpsertWriterUsingInsertUpdateStatement( String existSql, String insertSql, String updateSql, - DtRichOutputFormat metricOutputFormat) { + AbstractDtRichOutputFormat metricOutputFormat) { super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); this.existSql = existSql; this.insertSql = insertSql; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 2aa7b01f9..341ecf983 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.writer; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -48,9 +48,9 @@ public class AppendOnlyWriter implements JDBCWriter { private transient PreparedStatement statement; private transient List rows; // only use metric - private transient DtRichOutputFormat metricOutputFormat; + private transient AbstractDtRichOutputFormat metricOutputFormat; - public AppendOnlyWriter(String insertSql, int[] fieldTypes, DtRichOutputFormat metricOutputFormat) { + public AppendOnlyWriter(String insertSql, int[] fieldTypes, AbstractDtRichOutputFormat metricOutputFormat) { this.insertSql = insertSql; this.fieldTypes = fieldTypes; this.metricOutputFormat = metricOutputFormat; diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index a215c9d75..8f884ca03 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -42,7 +42,7 @@ /** * @author yanxi */ -public class RedisAllReqRow extends AllReqRow{ +public class RedisAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 7578879189085344807L; @@ -60,7 +60,7 @@ public class RedisAllReqRow extends AllReqRow{ private RedisSideReqRow redisSideReqRow; - public RedisAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new RedisAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); this.redisSideReqRow = new RedisSideReqRow(super.sideInfo); } diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java index fc24bc4ef..ec0fa0d15 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,16 +31,16 @@ /** * @author yanxi */ -public class RedisAllSideInfo extends SideInfo { +public class RedisAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = 1998703966487857613L; - public RedisAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index ee951a928..119ff5634 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -46,7 +46,7 @@ /** * @author yanxi */ -public class RedisAsyncReqRow extends AsyncReqRow { +public class RedisAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = -2079908694523987738L; @@ -64,7 +64,7 @@ public class RedisAsyncReqRow extends AsyncReqRow { private RedisSideReqRow redisSideReqRow; - public RedisAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new RedisAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); redisSideReqRow = new RedisSideReqRow(super.sideInfo); } diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java index 2736fc8a7..c23035ec7 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,15 +31,15 @@ /** * @author yanxi */ -public class RedisAsyncSideInfo extends SideInfo { +public class RedisAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = -4851348392924455039L; - public RedisAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java index 2d0fb77d4..514984f0a 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.side.redis.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -27,10 +27,10 @@ /** * @author yanxi */ -public class RedisSideParser extends AbsSideTableParser { +public class RedisSideParser extends AbstractSideTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RedisSideTableInfo redisSideTableInfo = new RedisSideTableInfo(); redisSideTableInfo.setName(tableName); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java index 5007868e9..17557f916 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.redis.table; import com.dtstack.flink.sql.side.ISideReqRow; -import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; @@ -40,9 +40,9 @@ public class RedisSideReqRow implements ISideReqRow, Serializable { private static final long serialVersionUID = 3751171828444748982L; - private SideInfo sideInfo; + private BaseSideInfo sideInfo; - public RedisSideReqRow(SideInfo sideInfo){ + public RedisSideReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -72,7 +72,7 @@ public Row fillData(Row input, Object sideInput) { return row; } - public void setRowField(Row row, Integer index, SideInfo sideInfo, String value) { + public void setRowField(Row row, Integer index, BaseSideInfo sideInfo, String value) { Integer keyIndex = sideInfo.getSideFieldIndex().get(index); String classType = sideInfo.getSideTableInfo().getFieldClassList().get(keyIndex).getName(); switch (classType){ diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java index ae6cdd017..5224134c8 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java @@ -18,12 +18,12 @@ package com.dtstack.flink.sql.side.redis.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; /** * @author yanxi */ -public class RedisSideTableInfo extends SideTableInfo { +public class RedisSideTableInfo extends AbstractSideTableInfo { private static final long serialVersionUID = -1L; diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index bbf49dff7..a51c62417 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.redis; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -43,7 +43,7 @@ /** * @author yanxi */ -public class RedisOutputFormat extends DtRichOutputFormat { +public class RedisOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(RedisOutputFormat.class); private String url; diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java index 007c7fd3b..cc49a3ba8 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.redis.table.RedisTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -69,7 +69,7 @@ public RedisSink(){ } @Override - public RedisSink genStreamSink(TargetTableInfo targetTableInfo) { + public RedisSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { RedisTableInfo redisTableInfo = (RedisTableInfo) targetTableInfo; this.url = redisTableInfo.getUrl(); this.database = redisTableInfo.getDatabase(); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 201abab2c..825d6901e 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.sink.redis.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; @@ -30,9 +30,9 @@ /** * @author yanxi */ -public class RedisSinkParser extends AbsTableParser { +public class RedisSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RedisTableInfo redisTableInfo = new RedisTableInfo(); redisTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, redisTableInfo); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java index 5c9940252..2425f8c38 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java @@ -18,12 +18,12 @@ package com.dtstack.flink.sql.sink.redis.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; /** * @author yanxi */ -public class RedisTableInfo extends TargetTableInfo { +public class RedisTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "redis"; diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java index 2f508b3e5..b032fdce6 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.format.DeserializationMetricWrapper; import com.dtstack.flink.sql.format.dtnest.DtNestRowDeserializationSchema; import com.dtstack.flink.sql.source.serversocket.table.ServersocketSourceTableInfo; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.flink.types.Row; @@ -67,7 +67,7 @@ public class CustomerSocketTextStreamFunction implements SourceFunction { ServersocketSourceTableInfo tableInfo; public CustomerSocketTextStreamFunction(ServersocketSourceTableInfo tableInfo, TypeInformation typeInfo, - Map rowAndFieldMapping, List fieldExtraInfos) { + Map rowAndFieldMapping, List fieldExtraInfos) { this.tableInfo = tableInfo; this.deserializationSchema = new DtNestRowDeserializationSchema(typeInfo, rowAndFieldMapping, fieldExtraInfos); this.deserializationMetricWrapper = new DeserializationMetricWrapper(typeInfo, deserializationSchema); diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java index 3ab01fb42..b4dda5f3b 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.serversocket.table.ServersocketSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -37,7 +37,7 @@ */ public class ServersocketSource implements IStreamSourceGener
{ @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { ServersocketSourceTableInfo serversocketSourceTableInfo = (ServersocketSourceTableInfo) sourceTableInfo; String tableName = serversocketSourceTableInfo.getName(); diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java index b52e38e59..f2d861b17 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java @@ -17,8 +17,8 @@ */ package com.dtstack.flink.sql.source.serversocket.table; -import com.dtstack.flink.sql.table.AbsSourceParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSourceParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -30,9 +30,9 @@ * * @author maqi */ -public class ServersocketSourceParser extends AbsSourceParser { +public class ServersocketSourceParser extends AbstractSourceParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ServersocketSourceTableInfo serversocketSourceTableInfo = new ServersocketSourceTableInfo(); serversocketSourceTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, serversocketSourceTableInfo); diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java index 02f6e9f30..3123b477f 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java @@ -17,7 +17,7 @@ */ package com.dtstack.flink.sql.source.serversocket.table; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.google.common.base.Preconditions; /** @@ -27,7 +27,7 @@ * * @author maqi */ -public class ServersocketSourceTableInfo extends SourceTableInfo { +public class ServersocketSourceTableInfo extends AbstractSourceTableInfo { private static final String CURR_TYPE = "serversocket"; diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java index af1d94171..dbae271e4 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -46,7 +46,7 @@ public class SqlserverAllReqRow extends AbstractRdbAllReqRow { private static final String SQLSERVER_DRIVER = "net.sourceforge.jtds.jdbc.Driver"; - public SqlserverAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new SqlserverAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java index 8f0fc63a4..a40af5697 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -31,7 +31,7 @@ */ public class SqlserverAllSideInfo extends RdbAllSideInfo { - public SqlserverAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index 070bfe334..f78061fca 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -44,7 +44,7 @@ public class SqlserverAsyncReqRow extends RdbAsyncReqRow { private final static String SQLSERVER_DRIVER = "net.sourceforge.jtds.jdbc.Driver"; - public SqlserverAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new SqlserverAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java index d3c833c96..e8a487721 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -34,7 +34,7 @@ */ public class SqlserverAsyncSideInfo extends RdbAsyncSideInfo { - public SqlserverAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java index be8ebb152..e06c13898 100644 --- a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java +++ b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side.sqlserver.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; /** @@ -30,8 +30,8 @@ public class SqlserverSideParser extends RdbSideParser { private static final String CURR_TYPE = "sqlserver"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo sqlServerTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo sqlServerTableInfo = super.getTableInfo(tableName, fieldsInfo, props); sqlServerTableInfo.setType(CURR_TYPE); return sqlServerTableInfo; } diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java index a695d6c3b..5300884bc 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.sqlserver.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -33,8 +33,8 @@ public class SqlserverSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "sqlserver"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo sqlserverTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo sqlserverTableInfo = super.getTableInfo(tableName, fieldsInfo, props); sqlserverTableInfo.setType(CURR_TYPE); return sqlserverTableInfo; } From 2bfd651137a5242878d489c16ce5b5d3aa6d580b Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 5 Mar 2020 12:33:51 +0800 Subject: [PATCH 115/523] =?UTF-8?q?=E5=85=88=E5=8E=BB=E9=99=A4join?= =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E7=9A=84=E6=9D=A1=E4=BB=B6=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/side/SideSqlExec.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 0da1b9c81..502dbc6f4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -782,9 +782,9 @@ private void joinFun(Object pollObj, Map localTableCache, throw new RuntimeException("can't not find side table:" + joinInfo.getRightTableName()); } - if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ - throw new RuntimeException("ON condition must contain all equal fields!!!"); - } +// if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ +// throw new RuntimeException("ON condition must contain all equal fields!!!"); +// } rightScopeChild.setRowTypeInfo(sideTableInfo.getRowTypeInfo()); From 1a973cf785a926fcd45c51fa474acc2ae880f8cb Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 5 Mar 2020 14:48:27 +0800 Subject: [PATCH 116/523] =?UTF-8?q?=E6=97=A5=E5=BF=97=E7=BA=A7=E5=88=AB?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=94=BE=E5=88=B0confProp=E9=87=8C=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 5 +---- .../com/dtstack/flink/sql/constrant/ConfigConstrant.java | 2 ++ .../java/com/dtstack/flink/sql/launcher/LauncherMain.java | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8b9e76f68..db25164b6 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,7 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack * taskmanager.slots：per_job模式下指定每个taskmanager对应的slot数量(默认1) * savePointPath：任务恢复点的路径（默认无） * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） + * logLevel: 日志级别动态配置（默认info） * [prometheus 相关参数](docs/prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 @@ -182,10 +183,6 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack * 必选：否 * 默认值：false -* **logLevel** - * 描述：设定日志级别 - * 必选：否 - * 默认值：info ## 2 结构 ### 2.1 源表插件 diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java index dcda4c093..00d1ea0a5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java @@ -63,6 +63,8 @@ public class ConfigConstrant { public static final String CHECKPOINTS_DIRECTORY_KEY = "state.checkpoints.dir"; public static final String STATE_BACKEND_INCREMENTAL_KEY = "state.backend.incremental"; + public static final String LOG_LEVEL_KEY = "logLevel"; + // restart plocy public static final int failureRate = 3; diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 54eea3657..95afc86cd 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -84,7 +84,7 @@ public static void main(String[] args) throws Exception { confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - setLogLevel(launcherOptions.getLogLevel()); + setLogLevel(confProperties.getProperty(ConfigConstrant.LOG_LEVEL_KEY)); if(mode.equals(ClusterMode.local.name())) { String[] localArgs = argList.toArray(new String[argList.size()]); From 9ae4741974591282538afc6c52dc480f2eb5f09f Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 5 Mar 2020 14:54:33 +0800 Subject: [PATCH 117/523] es id null check --- .../sql/sink/elasticsearch/CustomerSinkFunc.java | 13 ++++++++----- .../sql/sink/elasticsearch/CustomerSinkFunc.java | 13 ++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index 1f3efb8d7..0d97f5995 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -96,11 +96,14 @@ public void setOutRecords(Counter outRecords) { } private IndexRequest createIndexRequest(Row element) { - // index start at 1, - String idFieldStr = idFieldIndexList.stream() - .filter(index -> index > 0 && index <= element.getArity()) - .map(index -> element.getField(index - 1).toString()) - .collect(Collectors.joining(ID_VALUE_SPLIT)); + String idFieldStr = ""; + if (null != idFieldIndexList) { + // index start at 1, + idFieldStr = idFieldIndexList.stream() + .filter(index -> index > 0 && index <= element.getArity()) + .map(index -> element.getField(index - 1).toString()) + .collect(Collectors.joining(ID_VALUE_SPLIT)); + } Map dataMap = EsUtil.rowToJsonMap(element,fieldNames,fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index b1c8a5e97..ec1732d0e 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -95,11 +95,14 @@ public void setOutDirtyRecords(Counter outDirtyRecords) { } private IndexRequest createIndexRequest(Row element) { - // index start at 1, - String idFieldStr = idFieldIndexList.stream() - .filter(index -> index > 0 && index <= element.getArity()) - .map(index -> element.getField(index - 1).toString()) - .collect(Collectors.joining(ID_VALUE_SPLIT)); + String idFieldStr = ""; + if (null != idFieldIndexList) { + // index start at 1, + idFieldStr = idFieldIndexList.stream() + .filter(index -> index > 0 && index <= element.getArity()) + .map(index -> element.getField(index - 1).toString()) + .collect(Collectors.joining(ID_VALUE_SPLIT)); + } Map dataMap = Es6Util.rowToJsonMap(element, fieldNames, fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); From 8030b4aa250fa895256824ed2c0bb8c378db88fa Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 5 Mar 2020 17:02:31 +0800 Subject: [PATCH 118/523] es id null check --- .../sql/sink/elasticsearch/CustomerSinkFunc.java | 13 ++++++++----- .../sql/sink/elasticsearch/CustomerSinkFunc.java | 13 ++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index 1f3efb8d7..0d97f5995 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -96,11 +96,14 @@ public void setOutRecords(Counter outRecords) { } private IndexRequest createIndexRequest(Row element) { - // index start at 1, - String idFieldStr = idFieldIndexList.stream() - .filter(index -> index > 0 && index <= element.getArity()) - .map(index -> element.getField(index - 1).toString()) - .collect(Collectors.joining(ID_VALUE_SPLIT)); + String idFieldStr = ""; + if (null != idFieldIndexList) { + // index start at 1, + idFieldStr = idFieldIndexList.stream() + .filter(index -> index > 0 && index <= element.getArity()) + .map(index -> element.getField(index - 1).toString()) + .collect(Collectors.joining(ID_VALUE_SPLIT)); + } Map dataMap = EsUtil.rowToJsonMap(element,fieldNames,fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index b1c8a5e97..ec1732d0e 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -95,11 +95,14 @@ public void setOutDirtyRecords(Counter outDirtyRecords) { } private IndexRequest createIndexRequest(Row element) { - // index start at 1, - String idFieldStr = idFieldIndexList.stream() - .filter(index -> index > 0 && index <= element.getArity()) - .map(index -> element.getField(index - 1).toString()) - .collect(Collectors.joining(ID_VALUE_SPLIT)); + String idFieldStr = ""; + if (null != idFieldIndexList) { + // index start at 1, + idFieldStr = idFieldIndexList.stream() + .filter(index -> index > 0 && index <= element.getArity()) + .map(index -> element.getField(index - 1).toString()) + .collect(Collectors.joining(ID_VALUE_SPLIT)); + } Map dataMap = Es6Util.rowToJsonMap(element, fieldNames, fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); From 228c935e6a62f7f875805f0dad3f3881698c4021 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 5 Mar 2020 17:13:43 +0800 Subject: [PATCH 119/523] =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=9F=BA=E6=9C=AC?= =?UTF-8?q?=E4=BD=8D=E7=BD=AE=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/pom.xml | 7 ++++++- core/src/main/java/com/dtstack/flink/sql/Main.java | 1 + .../dtstack/flink/sql/constrant/ConfigConstrant.java | 3 ++- .../dtstack/flink/sql/exec/ExecuteProcessHelper.java | 11 ++++++++++- launcher/pom.xml | 5 ----- .../com/dtstack/flink/sql/launcher/LauncherMain.java | 8 -------- pom.xml | 1 - 7 files changed, 19 insertions(+), 17 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 677f9d5ee..eb470a3c0 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -20,6 +20,7 @@ 1.16.0 2.7.9 19.0 + 1.0.0-SNAPSHOT @@ -121,7 +122,11 @@ junit 4.12 - + + com.aiweiergou + tools-logger + ${logger.tool.version} + diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index 5d7528869..fe925f9a6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -38,6 +38,7 @@ public class Main { public static void main(String[] args) throws Exception { ParamsInfo paramsInfo = ExecuteProcessHelper.parseParams(args); + ExecuteProcessHelper.setLogLevel(paramsInfo); StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExecution(paramsInfo); env.execute(paramsInfo.getName()); LOG.info("program {} execution success", paramsInfo.getName()); diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java index 8e5f34d80..d1885a570 100644 --- a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java @@ -65,7 +65,7 @@ public class ConfigConstrant { public static final String RESTOREENABLE = "restore.enable"; - public static final String LOG_LEVEL_KEY = "logLevel"; + public static final String LOG_LEVEL_KEY = "logLevel"; // restart plocy @@ -75,4 +75,5 @@ public class ConfigConstrant { public static final String DELAYINTERVAL= "delay.interval"; //sec + } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 5cd1248e2..648745409 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -18,8 +18,10 @@ package com.dtstack.flink.sql.exec; +import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.config.CalciteConfig; +import com.dtstack.flink.sql.constrant.ConfigConstrant; import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.enums.EPluginLoadMode; @@ -353,5 +355,12 @@ public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperti return env; } - + public static void setLogLevel(ParamsInfo paramsInfo){ + String logLevel = paramsInfo.getConfProp().getProperty(ConfigConstrant.LOG_LEVEL_KEY); + if(org.apache.commons.lang3.StringUtils.isBlank(logLevel)){ + return; + } + ChangeLogLevelProcess logLevelProcess = new ChangeLogLevelProcess(); + logLevelProcess.process(logLevel); + } } \ No newline at end of file diff --git a/launcher/pom.xml b/launcher/pom.xml index 8c1ce53d5..1771df1ec 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -34,11 +34,6 @@ fastjson 1.2.7 - - com.aiweiergou - tools-logger - ${logger.tool.version} - diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 95afc86cd..f8338d2d4 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -149,12 +149,4 @@ private static String[] parseJson(String[] args) { String[] array = list.toArray(new String[list.size()]); return array; } - - private static void setLogLevel(String logLevel){ - if(org.apache.commons.lang3.StringUtils.isBlank(logLevel)){ - return; - } - ChangeLogLevelProcess logLevelProcess = new ChangeLogLevelProcess(); - logLevelProcess.process(logLevel); - } } diff --git a/pom.xml b/pom.xml index b0ba9f47d..5493645e9 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,6 @@ UTF-8 1.8.1 - 1.0.0-SNAPSHOT From 02e896e35a5cf597c85b4ff06314519a898dcf92 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 5 Mar 2020 17:21:57 +0800 Subject: [PATCH 120/523] =?UTF-8?q?=E7=A7=BB=E9=99=A4launcher=E7=9A=84?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E8=AE=BE=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/launcher/LauncherMain.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index f8338d2d4..0b036c87c 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -84,8 +84,6 @@ public static void main(String[] args) throws Exception { confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - setLogLevel(confProperties.getProperty(ConfigConstrant.LOG_LEVEL_KEY)); - if(mode.equals(ClusterMode.local.name())) { String[] localArgs = argList.toArray(new String[argList.size()]); Main.main(localArgs); From 2a0663e8302fe3111c310183f56e3c5e1d27717d Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 6 Mar 2020 13:14:47 +0800 Subject: [PATCH 121/523] codeview --- .../com/dtstack/flink/sql/classloader/ClassLoaderManager.java | 4 ++-- .../java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java | 4 ++-- .../main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java | 4 ++-- .../main/java/com/dtstack/flink/sql/side/ParserJoinField.java | 2 ++ 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java index c03f60617..2e62e11ab 100644 --- a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java @@ -98,9 +98,9 @@ public static List getClassPath() { - public static URLClassLoader loadExtraJar(List jarURLList, URLClassLoader classLoader) + public static URLClassLoader loadExtraJar(List jarUrlList, URLClassLoader classLoader) throws IllegalAccessException, InvocationTargetException { - for(URL url : jarURLList){ + for(URL url : jarUrlList){ if(url.toString().endsWith(".jar")){ urlClassLoaderAddUrl(classLoader, url); } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index ed54a9c48..40729d257 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -117,7 +117,7 @@ public static ParamsInfo parseParams(String[] args) throws Exception { String confProp = URLDecoder.decode(options.getConfProp(), Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - List jarURList = getExternalJarUrls(options.getAddjar()); + List jarUrlList = getExternalJarUrls(options.getAddjar()); return ParamsInfo.builder() .setSql(sql) @@ -127,7 +127,7 @@ public static ParamsInfo parseParams(String[] args) throws Exception { .setPluginLoadMode(pluginLoadMode) .setDeployMode(deployMode) .setConfProp(confProperties) - .setJarUrlList(jarURList) + .setJarUrlList(jarUrlList) .setLogLevel(logLevel) .build(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java index 8a6851add..357c65571 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java @@ -26,8 +26,8 @@ import org.apache.flink.table.runtime.types.CRow; import java.sql.SQLException; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; /** @@ -60,7 +60,7 @@ public void open(Configuration parameters) throws Exception { //start reload cache thread AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); - es = Executors.newSingleThreadScheduledExecutor(new DTThreadFactory("cache-all-reload")); + es = new ScheduledThreadPoolExecutor(1,new DTThreadFactory("cache-all-reload")); es.scheduleAtFixedRate(() -> reloadCache(), sideTableInfo.getCacheTimeout(), sideTableInfo.getCacheTimeout(), TimeUnit.MILLISECONDS); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java b/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java index 738e0f84e..d9d79f34e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java @@ -80,6 +80,7 @@ public static List getRowTypeInfo(SqlNode sqlNode, JoinScope scope, b switch(identifierSize) { case 1: fieldInfoList.addAll(getAllField(scope)); + break; default: SqlIdentifier tableIdentify = identifier.skipLast(1); JoinScope.ScopeChild scopeChild = scope.getScope(tableIdentify.getSimple()); @@ -99,6 +100,7 @@ public static List getRowTypeInfo(SqlNode sqlNode, JoinScope scope, b fieldInfo.setTypeInformation(type); fieldInfoList.add(fieldInfo); } + break; } } } From 78b3a1a4eab7812892e15ed2f773b5002c2ede0f Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 6 Mar 2020 17:05:29 +0800 Subject: [PATCH 122/523] =?UTF-8?q?=E6=97=B6=E9=97=B4=E6=A0=BC=E5=BC=8F?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/util/DateUtil.java | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index c942fb064..462eed30b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -23,6 +23,12 @@ import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.Calendar; import java.util.Date; import java.util.Locale; @@ -39,12 +45,9 @@ */ public class DateUtil { - static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss"; - static final String dateFormat = "yyyy-MM-dd"; - static final String timeFormat = "HH:mm:ss"; - static final SimpleDateFormat datetimeFormatter = new SimpleDateFormat(datetimeFormat); - static final SimpleDateFormat dateFormatter = new SimpleDateFormat(dateFormat); - static final SimpleDateFormat timeFormatter = new SimpleDateFormat(timeFormat); + static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss"); public static java.sql.Date columnToDate(Object column) { if(column instanceof String) { @@ -69,23 +72,32 @@ public static Date stringToDate(String strDate) { return null; } try { - return datetimeFormatter.parse(strDate); - } catch (ParseException ignored) { + ; + return localDateTimetoDate(LocalDateTime.parse(strDate, DATE_TIME_FORMATTER)); + } catch (DateTimeParseException ignored) { } try { - return dateFormatter.parse(strDate); - } catch (ParseException ignored) { + return localDateTimetoDate(LocalDate.parse(strDate, DATE_FORMATTER).atStartOfDay()); + } catch (DateTimeParseException ignored) { } try { - return timeFormatter.parse(strDate); - } catch (ParseException ignored) { + return localDateTimetoDate(LocalDateTime.of(LocalDate.now(), LocalTime.parse(strDate, TIME_FORMATTER))); + } catch (DateTimeParseException ignored) { } throw new RuntimeException("can't parse date"); } + public static Date localDateTimetoDate(LocalDateTime localDateTime){ + return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant()); + } + + public static LocalDateTime dateToLocalDateTime(Date date){ + return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } + /** * * @@ -762,11 +774,12 @@ public static java.sql.Timestamp columnToTimestamp(Object column) { } public static String dateToString(Date date) { - return dateFormatter.format(date); + LocalDateTime localDateTime = dateToLocalDateTime(date); + return localDateTime.format(DATE_FORMATTER); } public static String timestampToString(Date date) { - return datetimeFormatter.format(date); + LocalDateTime localDateTime = dateToLocalDateTime(date); + return localDateTime.format(DATE_TIME_FORMATTER); } - } From 02f2c37407225332bfcadb05f279e7042c3be5f9 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 6 Mar 2020 17:12:50 +0800 Subject: [PATCH 123/523] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E9=81=97=E6=BC=8F?= =?UTF-8?q?=E7=9A=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/console/table/TablePrintUtil.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java index c0d35404d..6ddc60386 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java @@ -22,6 +22,7 @@ public class TablePrintUtil { public static final int ALIGN_LEFT = 1;//左对齐 public static final int ALIGN_RIGHT = 2;//右对齐 public static final int ALIGN_CENTER = 3;//居中对齐 + private static final Pattern PATTERN = Pattern.compile("[\u4e00-\u9fa5]"); private int align = ALIGN_CENTER;//默认居中对齐 private boolean equilong = false;//默认不等宽 @@ -138,8 +139,8 @@ private static ListgetColList(Object obj) { * @return */ private int getStringCharLength(String str) { - Pattern p = Pattern.compile("[\u4e00-\u9fa5]");//利用正则找到中文 - Matcher m = p.matcher(str); + //利用正则找到中文 + Matcher m = PATTERN.matcher(str); int count = 0; while (m.find()) { count++; @@ -219,6 +220,8 @@ public String getTableString() { sb.append(cell); for (int i = 0; i < right + padding; i++) {sb.append(s);} break; + default: + break; } sb.append(v); } From 6c514b2ad43a9fe2a92231e973c048bac0a10694 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 9 Mar 2020 10:19:37 +0800 Subject: [PATCH 124/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E6=B1=A0=E7=9A=84=E5=8F=82=E6=95=B0=E3=80=90?= =?UTF-8?q?http://redmine.prod.dtstack.cn/issues/23230=E3=80=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java | 6 +++++- .../dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java | 5 ++++- .../flink/sql/side/postgresql/PostgresqlAsyncReqRow.java | 5 ++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index 305d65118..2b5223412 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -51,7 +51,11 @@ public void open(Configuration parameters) throws Exception { .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) - .put("provider_class", DT_PROVIDER_CLASS); + .put("provider_class", DT_PROVIDER_CLASS) + .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) + .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) + .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); + System.setProperty("vertx.disableFileCPResolving", "true"); VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 63d53b80d..725e7c3ee 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -77,7 +77,10 @@ public JsonObject getClientConfig() { .put("provider_class", DT_PROVIDER_CLASS) .put("idle_connection_test_period", 300) .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN) - .put("max_idle_time", 600); + .put("max_idle_time", 600) + .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) + .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) + .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); return impalaClientConfig; } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index 02b333819..86fed91fa 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -62,7 +62,10 @@ public void open(Configuration parameters) throws Exception { .put("driver_class", POSTGRESQL_DRIVER) .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) - .put("password", rdbSideTableInfo.getPassword()); + .put("password", rdbSideTableInfo.getPassword()) + .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) + .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) + .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); From 785811c41652c2499762d2591dc509729b52c333 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 9 Mar 2020 14:01:45 +0800 Subject: [PATCH 125/523] parse async side timestamp type --- .../com/dtstack/flink/sql/util/MathUtil.java | 83 +++++- .../main/java/io/vertx/core/json/Json.java | 257 ++++++++++++++++++ 2 files changed, 328 insertions(+), 12 deletions(-) create mode 100644 rdb/rdb-side/src/main/java/io/vertx/core/json/Json.java diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index 8708f86d7..76de225a4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -16,7 +16,6 @@ * limitations under the License. */ - package com.dtstack.flink.sql.util; @@ -27,6 +26,16 @@ import java.text.ParseException; import java.text.SimpleDateFormat; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.util.TimeZone; +import java.util.regex.Pattern; + +import static java.time.format.DateTimeFormatter.ISO_INSTANT; + /** * Convert val to specified numeric type * Date: 2017/4/21 @@ -36,6 +45,13 @@ public class MathUtil { + private static final Pattern DATETIME = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); + private static final Pattern DATE = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}$"); + private static final SimpleDateFormat TIMESTAMP_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd"); + + private static final int MILLIS_PER_SECOND = 1000; + public static Long getLongVal(Object obj) { if (obj == null) { return null; @@ -126,12 +142,12 @@ public static Double getDoubleVal(Object obj) { return Double.valueOf((String) obj); } else if (obj instanceof Float) { return ((Float) obj).doubleValue(); - } else if (obj instanceof Double){ + } else if (obj instanceof Double) { return (Double) obj; - }else if (obj instanceof BigDecimal) { + } else if (obj instanceof BigDecimal) { return ((BigDecimal) obj).doubleValue(); - }else if (obj instanceof Integer){ - return ((Integer)obj).doubleValue(); + } else if (obj instanceof Integer) { + return ((Integer) obj).doubleValue(); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Double."); @@ -229,12 +245,7 @@ public static Date getDate(Object obj) { return null; } if (obj instanceof String) { - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - try { - return new Date(format.parse((String) obj).getTime()); - } catch (ParseException e) { - throw new RuntimeException("String convert to Date fail."); - } + return getDateFromStr((String) obj); } else if (obj instanceof Timestamp) { return new Date(((Timestamp) obj).getTime()); } else if (obj instanceof Date) { @@ -243,6 +254,30 @@ public static Date getDate(Object obj) { throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); } + + public static Date getDateFromStr(String dateStr) { + // 2020-01-01 format + if (DATE.matcher(dateStr).matches()) { + // convert from local date to instant + Instant instant = LocalDate.parse(dateStr).atTime(LocalTime.of(0, 0, 0, 0)).toInstant(ZoneOffset.UTC); + // calculate the timezone offset in millis + int offset = TimeZone.getDefault().getOffset(instant.toEpochMilli()); + // need to remove the offset since time has no TZ component + return new Date(instant.toEpochMilli() - offset); + } else if (DATETIME.matcher(dateStr).matches()) { + // 2020-01-01T12:12:12Z format + Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); + return new Date(instant.toEpochMilli()); + } else { + try { + // 2020-01-01 12:12:12.0 format + return new Date(TIMESTAMP_FORMAT.parse(dateStr).getTime()); + } catch (ParseException e) { + throw new RuntimeException("String convert to Date fail."); + } + } + } + public static Timestamp getTimestamp(Object obj) { if (obj == null) { return null; @@ -252,8 +287,32 @@ public static Timestamp getTimestamp(Object obj) { } else if (obj instanceof Date) { return new Timestamp(((Date) obj).getTime()); } else if (obj instanceof String) { - return new Timestamp(getDate(obj).getTime()); + return getTimestampFromStr(obj.toString()); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); } + + public static Timestamp getTimestampFromStr(String timeStr) { + if (DATETIME.matcher(timeStr).matches()) { + Instant instant = Instant.from(ISO_INSTANT.parse(timeStr)); + return new Timestamp(instant.getEpochSecond() * MILLIS_PER_SECOND); + } else { + Date date = null; + try { + date = new Date(TIMESTAMP_FORMAT.parse(timeStr).getTime()); + } catch (ParseException e) { + throw new RuntimeException("getTimestampFromStr error data is " + timeStr); + } + return new Timestamp(date.getTime()); + } + } + + public static String getStringFromTimestamp(Timestamp timestamp) { + return TIMESTAMP_FORMAT.format(timestamp); + } + + public static String getStringFromDate(Date date) { + return DATE_FORMAT.format(date); + } + } diff --git a/rdb/rdb-side/src/main/java/io/vertx/core/json/Json.java b/rdb/rdb-side/src/main/java/io/vertx/core/json/Json.java new file mode 100644 index 000000000..2d6af178d --- /dev/null +++ b/rdb/rdb-side/src/main/java/io/vertx/core/json/Json.java @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2011-2017 Contributors to the Eclipse Foundation + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 + * which is available at https://www.apache.org/licenses/LICENSE-2.0. + * + * SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 + */ + +package io.vertx.core.json; + +import com.dtstack.flink.sql.util.MathUtil; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.*; +import com.fasterxml.jackson.databind.module.SimpleModule; +import io.netty.buffer.ByteBufInputStream; +import io.vertx.core.buffer.Buffer; + +import java.io.DataInput; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.Base64; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import static java.time.format.DateTimeFormatter.ISO_INSTANT; + +/** + * @author Tim Fox + */ +public class Json { + + public static ObjectMapper mapper = new ObjectMapper(); + public static ObjectMapper prettyMapper = new ObjectMapper(); + + static { + // Non-standard JSON but we allow C style comments in our JSON + mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); + + prettyMapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); + prettyMapper.configure(SerializationFeature.INDENT_OUTPUT, true); + + SimpleModule module = new SimpleModule(); + // custom types + module.addSerializer(JsonObject.class, new JsonObjectSerializer()); + module.addSerializer(JsonArray.class, new JsonArraySerializer()); + // he have 2 extensions: RFC-7493 + module.addSerializer(Instant.class, new InstantSerializer()); + module.addSerializer(byte[].class, new ByteArraySerializer()); + + mapper.registerModule(module); + prettyMapper.registerModule(module); + } + + /** + * Encode a POJO to JSON using the underlying Jackson mapper. + * + * @param obj a POJO + * @return a String containing the JSON representation of the given POJO. + * @throws EncodeException if a property cannot be encoded. + */ + public static String encode(Object obj) throws EncodeException { + try { + return mapper.writeValueAsString(obj); + } catch (Exception e) { + throw new EncodeException("Failed to encode as JSON: " + e.getMessage()); + } + } + + /** + * Encode a POJO to JSON using the underlying Jackson mapper. + * + * @param obj a POJO + * @return a Buffer containing the JSON representation of the given POJO. + * @throws EncodeException if a property cannot be encoded. + */ + public static Buffer encodeToBuffer(Object obj) throws EncodeException { + try { + return Buffer.buffer(mapper.writeValueAsBytes(obj)); + } catch (Exception e) { + throw new EncodeException("Failed to encode as JSON: " + e.getMessage()); + } + } + + /** + * Encode a POJO to JSON with pretty indentation, using the underlying Jackson mapper. + * + * @param obj a POJO + * @return a String containing the JSON representation of the given POJO. + * @throws EncodeException if a property cannot be encoded. + */ + public static String encodePrettily(Object obj) throws EncodeException { + try { + return prettyMapper.writeValueAsString(obj); + } catch (Exception e) { + throw new EncodeException("Failed to encode as JSON: " + e.getMessage()); + } + } + + /** + * Decode a given JSON string to a POJO of the given class type. + * @param str the JSON string. + * @param clazz the class to map to. + * @param the generic type. + * @return an instance of T + * @throws DecodeException when there is a parsing or invalid mapping. + */ + public static T decodeValue(String str, Class clazz) throws DecodeException { + try { + return mapper.readValue(str, clazz); + } catch (Exception e) { + throw new DecodeException("Failed to decode: " + e.getMessage()); + } + } + + /** + * Decode a given JSON string to a POJO of the given type. + * @param str the JSON string. + * @param type the type to map to. + * @param the generic type. + * @return an instance of T + * @throws DecodeException when there is a parsing or invalid mapping. + */ + public static T decodeValue(String str, TypeReference type) throws DecodeException { + try { + return mapper.readValue(str, type); + } catch (Exception e) { + throw new DecodeException("Failed to decode: " + e.getMessage(), e); + } + } + + /** + * Decode a given JSON buffer to a POJO of the given class type. + * @param buf the JSON buffer. + * @param type the type to map to. + * @param the generic type. + * @return an instance of T + * @throws DecodeException when there is a parsing or invalid mapping. + */ + public static T decodeValue(Buffer buf, TypeReference type) throws DecodeException { + try { + return mapper.readValue(new ByteBufInputStream(buf.getByteBuf()), type); + } catch (Exception e) { + throw new DecodeException("Failed to decode:" + e.getMessage(), e); + } + } + + /** + * Decode a given JSON buffer to a POJO of the given class type. + * @param buf the JSON buffer. + * @param clazz the class to map to. + * @param the generic type. + * @return an instance of T + * @throws DecodeException when there is a parsing or invalid mapping. + */ + public static T decodeValue(Buffer buf, Class clazz) throws DecodeException { + try { + return mapper.readValue((InputStream) new ByteBufInputStream(buf.getByteBuf()), clazz); + } catch (Exception e) { + throw new DecodeException("Failed to decode:" + e.getMessage(), e); + } + } + + @SuppressWarnings("unchecked") + static Object checkAndCopy(Object val, boolean copy) { + if (val == null) { + // OK + } else if (val instanceof Number && !(val instanceof BigDecimal)) { + // OK + } else if (val instanceof Boolean) { + // OK + } else if (val instanceof String) { + // OK + } else if (val instanceof Character) { + // OK + } else if (val instanceof CharSequence) { + val = val.toString(); + } else if (val instanceof JsonObject) { + if (copy) { + val = ((JsonObject) val).copy(); + } + } else if (val instanceof JsonArray) { + if (copy) { + val = ((JsonArray) val).copy(); + } + } else if (val instanceof Map) { + if (copy) { + val = (new JsonObject((Map)val)).copy(); + } else { + val = new JsonObject((Map)val); + } + } else if (val instanceof List) { + if (copy) { + val = (new JsonArray((List)val)).copy(); + } else { + val = new JsonArray((List)val); + } + } else if (val instanceof byte[]) { + val = Base64.getEncoder().encodeToString((byte[]) val); + } else if (val instanceof Instant) { + val = ISO_INSTANT.format((Instant) val); + } else if (val instanceof Timestamp) { + val = MathUtil.getStringFromTimestamp((Timestamp) val); + } else if (val instanceof Date) { + val = MathUtil.getStringFromDate((java.sql.Date) val); + } else { + val = val.toString(); + } + return val; + } + + static Stream asStream(Iterator sourceIterator) { + Iterable iterable = () -> sourceIterator; + return StreamSupport.stream(iterable.spliterator(), false); + } + + private static class JsonObjectSerializer extends JsonSerializer { + @Override + public void serialize(JsonObject value, JsonGenerator jgen, SerializerProvider provider) throws IOException { + jgen.writeObject(value.getMap()); + } + } + + private static class JsonArraySerializer extends JsonSerializer { + @Override + public void serialize(JsonArray value, JsonGenerator jgen, SerializerProvider provider) throws IOException { + jgen.writeObject(value.getList()); + } + } + + private static class InstantSerializer extends JsonSerializer { + @Override + public void serialize(Instant value, JsonGenerator jgen, SerializerProvider provider) throws IOException { + jgen.writeString(ISO_INSTANT.format(value)); + } + } + + private static class ByteArraySerializer extends JsonSerializer { + private final Base64.Encoder BASE64 = Base64.getEncoder(); + + @Override + public void serialize(byte[] value, JsonGenerator jgen, SerializerProvider provider) throws IOException { + jgen.writeString(BASE64.encodeToString(value)); + } + } +} From 507e4cb050d2ecdcef25ed3466f6c0c6b4483d8a Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 9 Mar 2020 14:28:23 +0800 Subject: [PATCH 126/523] modify time parse position --- .../com/dtstack/flink/sql/util/DateUtil.java | 62 +++++ .../com/dtstack/flink/sql/util/MathUtil.java | 57 +--- .../perjob/PerJobClusterClientBuilder.java | 2 +- .../sql/side/rdb/async/RdbAsyncReqRow.java | 14 +- .../main/java/io/vertx/core/json/Json.java | 257 ------------------ 5 files changed, 78 insertions(+), 314 deletions(-) delete mode 100644 rdb/rdb-side/src/main/java/io/vertx/core/json/Json.java diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index 5bfa2f203..a0f5291e7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -23,10 +23,18 @@ import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneOffset; import java.util.Calendar; import java.util.Date; import java.util.Locale; import java.util.SimpleTimeZone; +import java.util.TimeZone; +import java.util.regex.Pattern; + +import static java.time.format.DateTimeFormatter.ISO_INSTANT; /** @@ -47,6 +55,12 @@ public class DateUtil { static final SimpleDateFormat dateFormatter = new SimpleDateFormat(dateFormat); static final SimpleDateFormat timeFormatter = new SimpleDateFormat(timeFormat); + private static final Pattern DATETIME = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); + private static final Pattern DATE = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}$"); + + private static final int MILLIS_PER_SECOND = 1000; + + public static java.sql.Date columnToDate(Object column) { if(column instanceof String) { return new java.sql.Date(stringToDate((String)column).getTime()); @@ -770,4 +784,52 @@ public static String timestampToString(Date date) { return datetimeFormatter.format(date); } + + public static Timestamp getTimestampFromStr(String timeStr) { + if (DATETIME.matcher(timeStr).matches()) { + Instant instant = Instant.from(ISO_INSTANT.parse(timeStr)); + return new Timestamp(instant.getEpochSecond() * MILLIS_PER_SECOND); + } else { + java.sql.Date date = null; + try { + date = new java.sql.Date(datetimeFormatter.parse(timeStr).getTime()); + } catch (ParseException e) { + throw new RuntimeException("getTimestampFromStr error data is " + timeStr); + } + return new Timestamp(date.getTime()); + } + } + + public static java.sql.Date getDateFromStr(String dateStr) { + // 2020-01-01 format + if (DATE.matcher(dateStr).matches()) { + // convert from local date to instant + Instant instant = LocalDate.parse(dateStr).atTime(LocalTime.of(0, 0, 0, 0)).toInstant(ZoneOffset.UTC); + // calculate the timezone offset in millis + int offset = TimeZone.getDefault().getOffset(instant.toEpochMilli()); + // need to remove the offset since time has no TZ component + return new java.sql.Date(instant.toEpochMilli() - offset); + } else if (DATETIME.matcher(dateStr).matches()) { + // 2020-01-01T12:12:12Z format + Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); + return new java.sql.Date(instant.toEpochMilli()); + } else { + try { + // 2020-01-01 12:12:12.0 format + return new java.sql.Date(datetimeFormatter.parse(dateStr).getTime()); + } catch (ParseException e) { + throw new RuntimeException("String convert to Date fail."); + } + } + } + + + public static String getStringFromTimestamp(Timestamp timestamp) { + return datetimeFormatter.format(timestamp); + } + + public static String getStringFromDate(java.sql.Date date) { + return dateFormatter.format(date); + } + } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index 76de225a4..23513a912 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -44,14 +44,6 @@ */ public class MathUtil { - - private static final Pattern DATETIME = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); - private static final Pattern DATE = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}$"); - private static final SimpleDateFormat TIMESTAMP_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd"); - - private static final int MILLIS_PER_SECOND = 1000; - public static Long getLongVal(Object obj) { if (obj == null) { return null; @@ -245,7 +237,7 @@ public static Date getDate(Object obj) { return null; } if (obj instanceof String) { - return getDateFromStr((String) obj); + return DateUtil.getDateFromStr((String) obj); } else if (obj instanceof Timestamp) { return new Date(((Timestamp) obj).getTime()); } else if (obj instanceof Date) { @@ -255,28 +247,6 @@ public static Date getDate(Object obj) { } - public static Date getDateFromStr(String dateStr) { - // 2020-01-01 format - if (DATE.matcher(dateStr).matches()) { - // convert from local date to instant - Instant instant = LocalDate.parse(dateStr).atTime(LocalTime.of(0, 0, 0, 0)).toInstant(ZoneOffset.UTC); - // calculate the timezone offset in millis - int offset = TimeZone.getDefault().getOffset(instant.toEpochMilli()); - // need to remove the offset since time has no TZ component - return new Date(instant.toEpochMilli() - offset); - } else if (DATETIME.matcher(dateStr).matches()) { - // 2020-01-01T12:12:12Z format - Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); - return new Date(instant.toEpochMilli()); - } else { - try { - // 2020-01-01 12:12:12.0 format - return new Date(TIMESTAMP_FORMAT.parse(dateStr).getTime()); - } catch (ParseException e) { - throw new RuntimeException("String convert to Date fail."); - } - } - } public static Timestamp getTimestamp(Object obj) { if (obj == null) { @@ -287,32 +257,9 @@ public static Timestamp getTimestamp(Object obj) { } else if (obj instanceof Date) { return new Timestamp(((Date) obj).getTime()); } else if (obj instanceof String) { - return getTimestampFromStr(obj.toString()); + return DateUtil.getTimestampFromStr(obj.toString()); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); } - public static Timestamp getTimestampFromStr(String timeStr) { - if (DATETIME.matcher(timeStr).matches()) { - Instant instant = Instant.from(ISO_INSTANT.parse(timeStr)); - return new Timestamp(instant.getEpochSecond() * MILLIS_PER_SECOND); - } else { - Date date = null; - try { - date = new Date(TIMESTAMP_FORMAT.parse(timeStr).getTime()); - } catch (ParseException e) { - throw new RuntimeException("getTimestampFromStr error data is " + timeStr); - } - return new Timestamp(date.getTime()); - } - } - - public static String getStringFromTimestamp(Timestamp timestamp) { - return TIMESTAMP_FORMAT.format(timestamp); - } - - public static String getStringFromDate(Date date) { - return DATE_FORMAT.format(date); - } - } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 5dc0971bc..4497f28ec 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -69,7 +69,7 @@ public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(Properties co Configuration newConf = new Configuration(); confProp.forEach((key, val) -> newConf.setString(key.toString(), val.toString())); - AbstractYarnClusterDescriptor clusterDescriptor = getClusterDescriptor(newConf, yarnConf, "."); + AbstractYarnClusterDescriptor clusterDescriptor = getClusterDescriptor(newConf, yarnConf, launcherOptions.getFlinkconf()); if (StringUtils.isNotBlank(flinkJarPath)) { if (!new File(flinkJarPath).exists()) { diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 41dfd9ebe..b1582416c 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -23,6 +23,8 @@ import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; +import com.dtstack.flink.sql.util.DateUtil; +import com.dtstack.flink.sql.util.MathUtil; import io.vertx.core.Vertx; import io.vertx.core.VertxOptions; import io.vertx.core.json.JsonArray; @@ -38,6 +40,7 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; +import java.util.Date; import java.util.List; import java.util.Map; @@ -84,7 +87,7 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except dealMissKey(inputRow, resultFuture); return; } - inputParams.add(equalObj); + inputParams.add(convertDateType(equalObj)); } String key = buildCacheKey(inputParams); @@ -148,6 +151,15 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except }); } + private Object convertDateType(Object val) { + if (val instanceof Timestamp) { + val = DateUtil.getStringFromTimestamp((Timestamp) val); + } else if (val instanceof Date) { + val = DateUtil.getStringFromDate((java.sql.Date) val); + } + return val; + } + protected List getRows(Row inputRow, List cacheContent, List results) { List rowList = Lists.newArrayList(); for (JsonArray line : results) { diff --git a/rdb/rdb-side/src/main/java/io/vertx/core/json/Json.java b/rdb/rdb-side/src/main/java/io/vertx/core/json/Json.java deleted file mode 100644 index 2d6af178d..000000000 --- a/rdb/rdb-side/src/main/java/io/vertx/core/json/Json.java +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright (c) 2011-2017 Contributors to the Eclipse Foundation - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 - * which is available at https://www.apache.org/licenses/LICENSE-2.0. - * - * SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 - */ - -package io.vertx.core.json; - -import com.dtstack.flink.sql.util.MathUtil; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.*; -import com.fasterxml.jackson.databind.module.SimpleModule; -import io.netty.buffer.ByteBufInputStream; -import io.vertx.core.buffer.Buffer; - -import java.io.DataInput; -import java.io.IOException; -import java.io.InputStream; -import java.math.BigDecimal; -import java.sql.Timestamp; -import java.time.Instant; -import java.util.Base64; -import java.util.Date; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -import static java.time.format.DateTimeFormatter.ISO_INSTANT; - -/** - * @author Tim Fox - */ -public class Json { - - public static ObjectMapper mapper = new ObjectMapper(); - public static ObjectMapper prettyMapper = new ObjectMapper(); - - static { - // Non-standard JSON but we allow C style comments in our JSON - mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); - - prettyMapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); - prettyMapper.configure(SerializationFeature.INDENT_OUTPUT, true); - - SimpleModule module = new SimpleModule(); - // custom types - module.addSerializer(JsonObject.class, new JsonObjectSerializer()); - module.addSerializer(JsonArray.class, new JsonArraySerializer()); - // he have 2 extensions: RFC-7493 - module.addSerializer(Instant.class, new InstantSerializer()); - module.addSerializer(byte[].class, new ByteArraySerializer()); - - mapper.registerModule(module); - prettyMapper.registerModule(module); - } - - /** - * Encode a POJO to JSON using the underlying Jackson mapper. - * - * @param obj a POJO - * @return a String containing the JSON representation of the given POJO. - * @throws EncodeException if a property cannot be encoded. - */ - public static String encode(Object obj) throws EncodeException { - try { - return mapper.writeValueAsString(obj); - } catch (Exception e) { - throw new EncodeException("Failed to encode as JSON: " + e.getMessage()); - } - } - - /** - * Encode a POJO to JSON using the underlying Jackson mapper. - * - * @param obj a POJO - * @return a Buffer containing the JSON representation of the given POJO. - * @throws EncodeException if a property cannot be encoded. - */ - public static Buffer encodeToBuffer(Object obj) throws EncodeException { - try { - return Buffer.buffer(mapper.writeValueAsBytes(obj)); - } catch (Exception e) { - throw new EncodeException("Failed to encode as JSON: " + e.getMessage()); - } - } - - /** - * Encode a POJO to JSON with pretty indentation, using the underlying Jackson mapper. - * - * @param obj a POJO - * @return a String containing the JSON representation of the given POJO. - * @throws EncodeException if a property cannot be encoded. - */ - public static String encodePrettily(Object obj) throws EncodeException { - try { - return prettyMapper.writeValueAsString(obj); - } catch (Exception e) { - throw new EncodeException("Failed to encode as JSON: " + e.getMessage()); - } - } - - /** - * Decode a given JSON string to a POJO of the given class type. - * @param str the JSON string. - * @param clazz the class to map to. - * @param the generic type. - * @return an instance of T - * @throws DecodeException when there is a parsing or invalid mapping. - */ - public static T decodeValue(String str, Class clazz) throws DecodeException { - try { - return mapper.readValue(str, clazz); - } catch (Exception e) { - throw new DecodeException("Failed to decode: " + e.getMessage()); - } - } - - /** - * Decode a given JSON string to a POJO of the given type. - * @param str the JSON string. - * @param type the type to map to. - * @param the generic type. - * @return an instance of T - * @throws DecodeException when there is a parsing or invalid mapping. - */ - public static T decodeValue(String str, TypeReference type) throws DecodeException { - try { - return mapper.readValue(str, type); - } catch (Exception e) { - throw new DecodeException("Failed to decode: " + e.getMessage(), e); - } - } - - /** - * Decode a given JSON buffer to a POJO of the given class type. - * @param buf the JSON buffer. - * @param type the type to map to. - * @param the generic type. - * @return an instance of T - * @throws DecodeException when there is a parsing or invalid mapping. - */ - public static T decodeValue(Buffer buf, TypeReference type) throws DecodeException { - try { - return mapper.readValue(new ByteBufInputStream(buf.getByteBuf()), type); - } catch (Exception e) { - throw new DecodeException("Failed to decode:" + e.getMessage(), e); - } - } - - /** - * Decode a given JSON buffer to a POJO of the given class type. - * @param buf the JSON buffer. - * @param clazz the class to map to. - * @param the generic type. - * @return an instance of T - * @throws DecodeException when there is a parsing or invalid mapping. - */ - public static T decodeValue(Buffer buf, Class clazz) throws DecodeException { - try { - return mapper.readValue((InputStream) new ByteBufInputStream(buf.getByteBuf()), clazz); - } catch (Exception e) { - throw new DecodeException("Failed to decode:" + e.getMessage(), e); - } - } - - @SuppressWarnings("unchecked") - static Object checkAndCopy(Object val, boolean copy) { - if (val == null) { - // OK - } else if (val instanceof Number && !(val instanceof BigDecimal)) { - // OK - } else if (val instanceof Boolean) { - // OK - } else if (val instanceof String) { - // OK - } else if (val instanceof Character) { - // OK - } else if (val instanceof CharSequence) { - val = val.toString(); - } else if (val instanceof JsonObject) { - if (copy) { - val = ((JsonObject) val).copy(); - } - } else if (val instanceof JsonArray) { - if (copy) { - val = ((JsonArray) val).copy(); - } - } else if (val instanceof Map) { - if (copy) { - val = (new JsonObject((Map)val)).copy(); - } else { - val = new JsonObject((Map)val); - } - } else if (val instanceof List) { - if (copy) { - val = (new JsonArray((List)val)).copy(); - } else { - val = new JsonArray((List)val); - } - } else if (val instanceof byte[]) { - val = Base64.getEncoder().encodeToString((byte[]) val); - } else if (val instanceof Instant) { - val = ISO_INSTANT.format((Instant) val); - } else if (val instanceof Timestamp) { - val = MathUtil.getStringFromTimestamp((Timestamp) val); - } else if (val instanceof Date) { - val = MathUtil.getStringFromDate((java.sql.Date) val); - } else { - val = val.toString(); - } - return val; - } - - static Stream asStream(Iterator sourceIterator) { - Iterable iterable = () -> sourceIterator; - return StreamSupport.stream(iterable.spliterator(), false); - } - - private static class JsonObjectSerializer extends JsonSerializer { - @Override - public void serialize(JsonObject value, JsonGenerator jgen, SerializerProvider provider) throws IOException { - jgen.writeObject(value.getMap()); - } - } - - private static class JsonArraySerializer extends JsonSerializer { - @Override - public void serialize(JsonArray value, JsonGenerator jgen, SerializerProvider provider) throws IOException { - jgen.writeObject(value.getList()); - } - } - - private static class InstantSerializer extends JsonSerializer { - @Override - public void serialize(Instant value, JsonGenerator jgen, SerializerProvider provider) throws IOException { - jgen.writeString(ISO_INSTANT.format(value)); - } - } - - private static class ByteArraySerializer extends JsonSerializer { - private final Base64.Encoder BASE64 = Base64.getEncoder(); - - @Override - public void serialize(byte[] value, JsonGenerator jgen, SerializerProvider provider) throws IOException { - jgen.writeString(BASE64.encodeToString(value)); - } - } -} From dba0d559672224c0826bbe9d7c66d97b7847c9cc Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 9 Mar 2020 15:04:41 +0800 Subject: [PATCH 127/523] RdbAsyncReqRow convertDataType --- .../sql/side/rdb/async/RdbAsyncReqRow.java | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index b1582416c..237f1277d 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -39,7 +39,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.math.BigDecimal; import java.sql.Timestamp; +import java.time.Instant; import java.util.Date; import java.util.List; import java.util.Map; @@ -87,7 +89,7 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except dealMissKey(inputRow, resultFuture); return; } - inputParams.add(convertDateType(equalObj)); + inputParams.add(convertDataType(equalObj)); } String key = buildCacheKey(inputParams); @@ -151,15 +153,43 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except }); } - private Object convertDateType(Object val) { - if (val instanceof Timestamp) { + + private Object convertDataType(Object val) { + if (val == null) { + // OK + } else if (val instanceof Number && !(val instanceof BigDecimal)) { + // OK + } else if (val instanceof Boolean) { + // OK + } else if (val instanceof String) { + // OK + } else if (val instanceof Character) { + // OK + } else if (val instanceof CharSequence) { + + } else if (val instanceof JsonObject) { + + } else if (val instanceof JsonArray) { + + } else if (val instanceof Map) { + + } else if (val instanceof List) { + + } else if (val instanceof byte[]) { + + } else if (val instanceof Instant) { + + } else if (val instanceof Timestamp) { val = DateUtil.getStringFromTimestamp((Timestamp) val); - } else if (val instanceof Date) { + } else if (val instanceof java.util.Date) { val = DateUtil.getStringFromDate((java.sql.Date) val); + } else { + val = val.toString(); } return val; } + protected List getRows(Row inputRow, List cacheContent, List results) { List rowList = Lists.newArrayList(); for (JsonArray line : results) { From 006298da4ec97ba5de2720ba3a21148eb81f9808 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 10 Mar 2020 13:57:37 +0800 Subject: [PATCH 128/523] extract select field to FieldInfo --- .../com/dtstack/flink/sql/side/FieldInfo.java | 21 ++++ .../dtstack/flink/sql/util/TableUtils.java | 101 +++++++++++++++--- 2 files changed, 109 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/FieldInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/FieldInfo.java index f8ba77383..85bad8c2c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/FieldInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/FieldInfo.java @@ -23,6 +23,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import java.io.Serializable; +import java.util.Objects; /** * Reason: @@ -64,4 +65,24 @@ public TypeInformation getTypeInformation() { public void setTypeInformation(TypeInformation typeInformation) { this.typeInformation = typeInformation; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + FieldInfo fieldInfo = (FieldInfo) o; + return Objects.equals(table, fieldInfo.table) && + Objects.equals(fieldName, fieldInfo.fieldName); + } + + @Override + public int hashCode() { + return Objects.hash(table, fieldName); + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 3c8740213..dad6f9399 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -20,12 +20,10 @@ package com.dtstack.flink.sql.util; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.FieldReplaceInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlDataTypeSpec; @@ -40,7 +38,6 @@ import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.table.api.Table; import java.util.List; @@ -72,31 +69,48 @@ public static List parserSelectField(SqlSelect sqlSelect, Map fieldInfoList = Lists.newArrayList(); String fromNode = sqlSelect.getFrom().toString(); - for(SqlNode fieldNode : sqlNodeList.getList()){ - SqlIdentifier identifier = (SqlIdentifier)fieldNode; - if(!identifier.isStar()) { + for (SqlNode fieldNode : sqlNodeList.getList()) { + extractSelectFieldToFieldInfo(fieldNode,fromNode,fieldInfoList,localTableCache); + } + + return fieldInfoList; + } + + /** + * 解析select Node 提取FieldInfo + * @param fieldNode + * @param fromNode + * @param fieldInfoList + * @param localTableCache + */ + public static void extractSelectFieldToFieldInfo(SqlNode fieldNode, String fromNode, List fieldInfoList, Map localTableCache) { + if (fieldNode.getKind() == IDENTIFIER) { + SqlIdentifier identifier = (SqlIdentifier) fieldNode; + if (!identifier.isStar()) { String tableName = identifier.names.size() == 1 ? fromNode : identifier.getComponent(0).getSimple(); String fieldName = identifier.names.size() == 1 ? identifier.getComponent(0).getSimple() : identifier.getComponent(1).getSimple(); FieldInfo fieldInfo = new FieldInfo(); fieldInfo.setTable(tableName); fieldInfo.setFieldName(fieldName); - fieldInfoList.add(fieldInfo); + + if (!fieldInfoList.contains(fieldInfo)) { + fieldInfoList.add(fieldInfo); + } } else { //处理 int identifierSize = identifier.names.size(); - - switch(identifierSize) { + switch (identifierSize) { case 1: throw new RuntimeException("not support to parse * without scope of table"); default: SqlIdentifier tableIdentify = identifier.skipLast(1); Table registerTable = localTableCache.get(tableIdentify.getSimple()); - if(registerTable == null){ + if (registerTable == null) { throw new RuntimeException("can't find table alias " + tableIdentify.getSimple()); } String[] fieldNames = registerTable.getSchema().getFieldNames(); - for(String fieldName : fieldNames){ + for (String fieldName : fieldNames) { FieldInfo fieldInfo = new FieldInfo(); fieldInfo.setTable(tableIdentify.getSimple()); fieldInfo.setFieldName(fieldName); @@ -104,9 +118,70 @@ public static List parserSelectField(SqlSelect sqlSelect, Map Date: Tue, 10 Mar 2020 15:45:22 +0800 Subject: [PATCH 129/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E5=86=B2=E7=AA=81=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/exec/ExecuteProcessHelper.java | 2 +- .../dtstack/flink/sql/side/SideSqlExec.java | 4 ++-- .../sql/table/AbstractSideTableParser.java | 4 ++-- .../sink/elasticsearch/ElasticsearchSink.java | 2 +- .../Elasticsearch6AllReqRow.java | 21 ++++++++----------- .../Elasticsearch6AllSideInfo.java | 11 +++++----- .../Elasticsearch6AsyncReqRow.java | 4 ++-- .../Elasticsearch6AsyncSideInfo.java | 13 ++++++------ .../table/Elasticsearch6SideParser.java | 8 +++---- .../table/Elasticsearch6SideTableInfo.java | 4 ++-- .../sql/side/elasticsearch6/util/Es6Util.java | 10 ++++----- .../sink/elasticsearch/ElasticsearchSink.java | 4 ++-- .../table/ElasticsearchSinkParser.java | 8 +++---- .../table/ElasticsearchTableInfo.java | 4 ++-- .../sql/sink/hbase/HbaseOutputFormat.java | 7 +------ .../flink/sql/sink/kafka/KafkaSink.java | 1 + .../flink/sql/sink/kafka/KafkaSink.java | 1 + .../flink/sql/sink/kafka/KafkaSink.java | 1 + .../flink/sql/sink/kafka/KafkaSink.java | 1 + .../flink/sql/side/mongo/MongoAllReqRow.java | 5 +++-- .../side/rdb/all/AbstractRdbAllReqRow.java | 3 +-- .../sink/rdb/writer/AbstractUpsertWriter.java | 10 ++++----- .../sql/sink/rdb/writer/AppendOnlyWriter.java | 2 +- .../sql/side/redis/RedisAsyncReqRow.java | 4 ++-- 24 files changed, 65 insertions(+), 69 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 85568f19e..c02c3b142 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -127,7 +127,7 @@ public static ParamsInfo parseParams(String[] args) throws Exception { .setPluginLoadMode(pluginLoadMode) .setDeployMode(deployMode) .setConfProp(confProperties) - .setJarUrlList(jarURList) + .setJarUrlList(jarUrlList) .build(); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index a92685f3c..e828bec03 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -99,7 +99,7 @@ public class SideSqlExec { private Map localTableCache = Maps.newHashMap(); - public void exec(String sql, Map sideTableMap, StreamTableEnvironment tableEnv, + public void exec(String sql, Map sideTableMap, StreamTableEnvironment tableEnv, Map tableCache, StreamQueryConfig queryConfig, CreateTmpTableParser.SqlParserResult createView) throws Exception { if(localSqlPluginPath == null){ throw new RuntimeException("need to set localSqlPluginPath"); @@ -732,7 +732,7 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, Map tableCache, List replaceInfoList) throws SqlParseException { - AliasInfo aliasInfo = parseASNode(pollSqlNode); + AliasInfo aliasInfo = parseAsNode(pollSqlNode); if (localTableCache.containsKey(aliasInfo.getName())) { return; } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index a168eebbc..39b72c22f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -108,8 +108,8 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0){ sideTableInfo.setAsyncTimeoutNumLimit(asyncTimeoutNum); } diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 4229abb99..5ca81c5ed 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.sink.elasticsearch; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -33,7 +34,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index 648f09fba..b8cf2c46a 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -18,24 +18,22 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; - -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; import com.dtstack.flink.sql.side.elasticsearch6.util.SwitchUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -49,7 +47,6 @@ import java.io.IOException; import java.io.Serializable; -import java.sql.SQLException; import java.sql.Timestamp; import java.util.Calendar; import java.util.List; @@ -60,7 +57,7 @@ * @author yinxi * @date 2020/1/13 - 1:00 */ -public class Elasticsearch6AllReqRow extends AllReqRow implements Serializable { +public class Elasticsearch6AllReqRow extends BaseAllReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AllReqRow.class); @@ -70,7 +67,7 @@ public class Elasticsearch6AllReqRow extends AllReqRow implements Serializable { private SearchRequest searchRequest; private BoolQueryBuilder boolQueryBuilder; - public Elasticsearch6AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Elasticsearch6AllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java index bf21b3ca8..d353a583d 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java @@ -18,12 +18,13 @@ package com.dtstack.flink.sql.side.elasticsearch6; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; + import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; @@ -35,15 +36,15 @@ * @author yinxi * @date 2020/1/13 - 1:01 */ -public class Elasticsearch6AllSideInfo extends SideInfo { +public class Elasticsearch6AllSideInfo extends BaseSideInfo { - public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index f67177fe4..e8164edb2 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -57,14 +57,14 @@ * @author yinxi * @date 2020/2/13 - 13:10 */ -public class Elasticsearch6AsyncReqRow extends AsyncReqRow implements Serializable { +public class Elasticsearch6AsyncReqRow extends BaseAsyncReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AsyncReqRow.class); private transient RestHighLevelClient rhlClient; private SearchRequest searchRequest; private List sqlJoinCompareOperate = Lists.newArrayList(); - public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Elasticsearch6AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); SqlNode conditionNode = joinInfo.getCondition(); ParseUtils.parseJoinCompareOperate(conditionNode, sqlJoinCompareOperate); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java index 7b3a2f9e7..4118988ab 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java @@ -18,18 +18,17 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import org.apache.flink.api.java.typeutils.RowTypeInfo; - +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -37,15 +36,15 @@ * @author yinxi * @date 2020/2/13 - 13:09 */ -public class Elasticsearch6AsyncSideInfo extends SideInfo { +public class Elasticsearch6AsyncSideInfo extends BaseSideInfo { - public Elasticsearch6AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java index 13dfe2995..1b39bbf0f 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.side.elasticsearch6.table; import com.dtstack.flink.sql.side.elasticsearch6.util.ClassUtil; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -30,7 +30,7 @@ * @author yinxi * @date 2020/1/13 - 1:07 */ -public class Elasticsearch6SideParser extends AbsSideTableParser { +public class Elasticsearch6SideParser extends AbstractSideTableParser { private static final String KEY_ES6_ADDRESS = "address"; @@ -55,7 +55,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { Elasticsearch6SideTableInfo elasticsearch6SideTableInfo = new Elasticsearch6SideTableInfo(); elasticsearch6SideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearch6SideTableInfo); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java index 0afe2d59e..a01c90a53 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side.elasticsearch6.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -26,7 +26,7 @@ * @author yinxi * @date 2020/1/13 - 15:00 */ -public class Elasticsearch6SideTableInfo extends SideTableInfo { +public class Elasticsearch6SideTableInfo extends AbstractSideTableInfo { private static final String CURR_TYPE = "elasticsearch6"; diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java index cc591789e..3d9e4d62a 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.side.elasticsearch6.util; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHost; @@ -98,7 +98,7 @@ public static RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh } // add index and type to search request - public static SearchRequest setSearchRequest(SideInfo sideInfo) { + public static SearchRequest setSearchRequest(BaseSideInfo sideInfo) { SearchRequest searchRequest = new SearchRequest(); Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); // determine existence of index @@ -129,7 +129,7 @@ public static SearchRequest setSearchRequest(SideInfo sideInfo) { } // build where cause - public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { + public static BoolQueryBuilder setPredicateclause(BaseSideInfo sideInfo) { BoolQueryBuilder boolQueryBuilder = null; List predicateInfoes = sideInfo.getSideTableInfo().getPredicateInfoes(); @@ -144,7 +144,7 @@ public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { } // build filter condition - public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, SideInfo sideInfo) { + public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, BaseSideInfo sideInfo) { switch (info.getOperatorKind()) { case "IN": return boolQueryBuilder.must(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName(), sideInfo), removeSpaceAndApostrophe(info.getCondition()))); @@ -202,7 +202,7 @@ public static String[] removeSpaceAndApostrophe(String str) { } // prevent word segmentation - public static String textConvertToKeyword(String fieldName, SideInfo sideInfo) { + public static String textConvertToKeyword(String fieldName, BaseSideInfo sideInfo) { String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); String fieldType = sideFieldTypes[fieldIndex]; diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 2646c50e9..b7d9de6fc 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.elasticsearch; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,7 +32,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.collect.Maps; import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; @@ -128,7 +128,7 @@ public void emitDataStream(DataStream> dataStream) { } @Override - public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { + public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { esTableInfo = (ElasticsearchTableInfo) targetTableInfo; clusterName = esTableInfo.getClusterName(); index = esTableInfo.getIndex(); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 8902b953c..22c2b72bc 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -29,7 +29,7 @@ * @author yinxi * @date 2020/1/9 - 15:06 */ -public class ElasticsearchSinkParser extends AbsTableParser { +public class ElasticsearchSinkParser extends AbstractTableParser { private static final String KEY_ES6_ADDRESS = "address"; @@ -55,7 +55,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ElasticsearchTableInfo elasticsearchTableInfo = new ElasticsearchTableInfo(); elasticsearchTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearchTableInfo); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index 3cc3dd9ff..d0eef18f0 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -29,7 +29,7 @@ * @author yinxi * @date 2020/1/9 - 15:06 */ -public class ElasticsearchTableInfo extends TargetTableInfo { +public class ElasticsearchTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "elasticsearch6"; diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 7709e68d9..382e8f975 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -20,19 +20,14 @@ package com.dtstack.flink.sql.sink.hbase; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.enums.EUpdateMode; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; - -import com.dtstack.flink.sql.enums.EUpdateMode; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; -import com.google.common.collect.Lists; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index dbd231dfe..7105bc037 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 7dad2896f..0a991a8ea 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 5144f4486..34ea8fc5f 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 3c5d638cc..d7807a935 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index 93b5a379f..3373f3667 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -35,6 +35,7 @@ import com.mongodb.client.MongoDatabase; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -215,7 +216,7 @@ private void loadData(Map>> tmpCache) throws SQ String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); BasicDBObject basicDBObject = new BasicDBObject(); for (String selectField : sideFieldNames) { - basicDbObject.append(selectField, 1); + basicDBObject.append(selectField, 1); } BasicDBObject filterObject = new BasicDBObject(); try { @@ -232,7 +233,7 @@ private void loadData(Map>> tmpCache) throws SQ } - FindIterable findIterable = dbCollection.find(filterObject).projection(basicDbObject).limit(FETCH_SIZE); + FindIterable findIterable = dbCollection.find(filterObject).projection(basicDBObject).limit(FETCH_SIZE); MongoCursor mongoCursor = findIterable.iterator(); while (mongoCursor.hasNext()) { Document doc = mongoCursor.next(); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 339a4a3e8..17daecfb0 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -24,8 +24,7 @@ import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import com.dtstack.flink.sql.side.AllReqRow; -import com.dtstack.flink.sql.side.SideInfo; + import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java index 481cd27d6..9ca0fd754 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -116,7 +116,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { - this.deleteStatement = connection.prepareStatement(deleteSQL); + this.deleteStatement = connection.prepareStatement(deleteSql); } @Override @@ -235,7 +235,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { super.prepareStatement(connection); - upsertStatement = connection.prepareStatement(upsertSQL); + upsertStatement = connection.prepareStatement(upsertSql); } @Override @@ -300,9 +300,9 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { super.prepareStatement(connection); - existStatement = connection.prepareStatement(existSQL); - insertStatement = connection.prepareStatement(insertSQL); - updateStatement = connection.prepareStatement(updateSQL); + existStatement = connection.prepareStatement(existSql); + insertStatement = connection.prepareStatement(insertSql); + updateStatement = connection.prepareStatement(updateSql); } @Override diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 5c9fcf8ac..3559d4376 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -64,7 +64,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { - this.statement = connection.prepareStatement(insertSQL); + this.statement = connection.prepareStatement(insertSql); } /** diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index ba6dc51ab..d2a2aed56 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -18,6 +18,8 @@ package com.dtstack.flink.sql.side.redis; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -25,11 +27,9 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; From b78a00511222379d99f2b7cb19f3ac63dbfd2779 Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Tue, 10 Mar 2020 16:09:31 +0800 Subject: [PATCH 130/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E5=86=B2=E7=AA=81=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sink/cassandra/CassandraOutputFormat.java | 2 +- .../flink/sql/side/kudu/KuduAllReqRow.java | 6 ++++++ .../flink/sql/side/kudu/KuduAsyncReqRow.java | 19 +++++++++---------- .../flink/sql/side/redis/RedisAllReqRow.java | 13 ++++++------- .../sql/sink/redis/RedisOutputFormat.java | 5 +---- 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java index 9cb180f8e..db9b87d8d 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java @@ -54,7 +54,7 @@ import com.datastax.driver.core.SocketOptions; import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 505b21479..31f9aa13d 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -7,11 +7,17 @@ import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.KuduClient; diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index 5d3d10a79..e534a85f9 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -1,20 +1,12 @@ package com.dtstack.flink.sql.side.kudu; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; - import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; @@ -24,6 +16,13 @@ import com.stumbleupon.async.Deferred; import io.vertx.core.json.JsonArray; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.AsyncKuduClient; diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 7bc4fe60e..02f480105 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -18,15 +18,10 @@ package com.dtstack.flink.sql.side.redis; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; - -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; import com.esotericsoftware.minlog.Log; @@ -34,6 +29,10 @@ import org.apache.calcite.sql.JoinType; import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.HostAndPort; diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index 8e807d0a9..ae4fe5a4b 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -19,15 +19,12 @@ package com.dtstack.flink.sql.sink.redis; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; - -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.HostAndPort; From 6bdc1f29fd7b208bdf3b8f35729ac4eb5c74c6f1 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 11 Mar 2020 09:16:43 +0800 Subject: [PATCH 131/523] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=90=8C=E6=97=B6?= =?UTF-8?q?=E4=BD=BF=E7=94=A8all=E5=92=8Casync=E7=BB=B4=E8=A1=A8=E7=9A=84B?= =?UTF-8?q?UG=E3=80=90http://redmine.prod.dtstack.cn/issues/21893=3Fissue?= =?UTF-8?q?=5Fcount=3D4&issue=5Fposition=3D4&prev=5Fissue=5Fid=3D22739?= =?UTF-8?q?=E3=80=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/table/TableInfoParser.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java index ae98d90ae..cd478ce6a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java @@ -76,11 +76,11 @@ public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserRe sourceTableInfoMap.put(type, absTableParser); } }else{ - absTableParser = sideTableInfoMap.get(type); + String cacheType = MathUtil.getString(props.get(SideTableInfo.CACHE_KEY)); + absTableParser = sideTableInfoMap.get(type + cacheType); if(absTableParser == null){ - String cacheType = MathUtil.getString(props.get(SideTableInfo.CACHE_KEY)); absTableParser = StreamSideFactory.getSqlParser(type, localPluginRoot, cacheType); - sideTableInfoMap.put(type, absTableParser); + sideTableInfoMap.put(type + cacheType, absTableParser); } } From 6b9e1e9a6a1127e0610b6ab867b97c33c653859f Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 11 Mar 2020 11:29:17 +0800 Subject: [PATCH 132/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dredis=E7=BB=B4?= =?UTF-8?q?=E8=A1=A8join=E6=97=B6=E4=B8=8D=E5=A4=9A=E7=A7=8D=E6=9D=A1?= =?UTF-8?q?=E4=BB=B6=EF=BC=8C=E5=B9=B6=E4=B8=94join=E5=AD=97=E6=AE=B5value?= =?UTF-8?q?=E5=80=BC=E4=B8=8D=E8=83=BD=E5=87=BA=E7=8E=B0=E2=80=98=EF=BC=9A?= =?UTF-8?q?=E2=80=99=E3=80=90http://redmine.prod.dtstack.cn/issues/23318?= =?UTF-8?q?=3Fissue=5Fcount=3D5&issue=5Fposition=3D1&next=5Fissue=5Fid=3D2?= =?UTF-8?q?3230=E3=80=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/redis/RedisAsyncReqRow.java | 77 +++++++++---------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index bb67c3c47..9d00de4f8 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -23,16 +23,17 @@ import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; -import io.lettuce.core.KeyValue; import io.lettuce.core.RedisClient; import io.lettuce.core.RedisFuture; import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.api.async.RedisHashAsyncCommands; import io.lettuce.core.api.async.RedisKeyAsyncCommands; import io.lettuce.core.api.async.RedisStringAsyncCommands; import io.lettuce.core.cluster.RedisClusterClient; import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -118,7 +119,7 @@ public Row fillData(Row input, Object sideInput) { @Override public void asyncInvoke(Row input, ResultFuture resultFuture) throws Exception { Row inputRow = Row.copy(input); - List keyData = Lists.newLinkedList(); + Map refData = Maps.newHashMap(); for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = inputRow.getField(conValIndex); @@ -126,13 +127,13 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except dealMissKey(inputRow, resultFuture); return; } - String value = equalObj.toString(); - keyData.add(sideInfo.getEqualFieldList().get(i)); - keyData.add(value); + refData.put(sideInfo.getEqualFieldList().get(i), equalObj); } - String key = buildCacheKey(keyData); - + String key = buildCacheKey(refData); + if(StringUtils.isBlank(key)){ + return; + } if(openCache()){ CacheObj val = getFromCache(key); if(val != null){ @@ -154,44 +155,36 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except } } - Map keyValue = Maps.newHashMap(); - List value = async.keys(key + ":*").get(); - String[] values = value.toArray(new String[value.size()]); - if (values.length == 0){ - dealMissKey(inputRow, resultFuture); - } else { - RedisFuture>> future = ((RedisStringAsyncCommands) async).mget(values); - future.thenAccept(new Consumer>>() { - @Override - public void accept(List> keyValues) { - if (keyValues.size() != 0) { - for (int i = 0; i < keyValues.size(); i++) { - String[] splitKeys = keyValues.get(i).getKey().split(":"); - keyValue.put(splitKeys[1], splitKeys[2]); - keyValue.put(splitKeys[3], keyValues.get(i).getValue()); - } - try { - Row row = fillData(inputRow, keyValue); - dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.MultiLine, keyValue)); - resultFuture.complete(Collections.singleton(row)); - } catch (Exception e) { - dealFillDataError(resultFuture, e, inputRow); - } - } else { - dealMissKey(inputRow, resultFuture); - dealCacheData(key,CacheMissVal.getMissKeyObj()); + RedisFuture> future = ((RedisHashAsyncCommands) async).hgetall(key); + future.thenAccept(new Consumer>() { + @Override + public void accept(Map values) { + if (MapUtils.isNotEmpty(values)) { + try { + Row row = fillData(inputRow, values); + dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.MultiLine, values)); + resultFuture.complete(Collections.singleton(row)); + } catch (Exception e) { + dealFillDataError(resultFuture, e, inputRow); } + } else { + dealMissKey(inputRow, resultFuture); + dealCacheData(key,CacheMissVal.getMissKeyObj()); } - }); - } + } + }); } - private String buildCacheKey(List keyData) { - String kv = String.join(":", keyData); - String tableName = redisSideTableInfo.getTableName(); - StringBuilder preKey = new StringBuilder(); - preKey.append(tableName).append(":").append(kv); - return preKey.toString(); + private String buildCacheKey(Map refData) { + StringBuilder keyBuilder = new StringBuilder(redisSideTableInfo.getTableName()); + List primaryKeys = redisSideTableInfo.getPrimaryKeys(); + for(String primaryKey : primaryKeys){ + if(refData.containsKey(primaryKey)){ + keyBuilder.append("_").append(refData.get(primaryKey)); + } + return null; + } + return keyBuilder.toString(); } @Override From af338b7acdf1de377fa2ec5c3208c18299b75022 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 11 Mar 2020 13:51:39 +0800 Subject: [PATCH 133/523] bugfix --- .../com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 9d00de4f8..053621acd 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -179,10 +179,10 @@ private String buildCacheKey(Map refData) { StringBuilder keyBuilder = new StringBuilder(redisSideTableInfo.getTableName()); List primaryKeys = redisSideTableInfo.getPrimaryKeys(); for(String primaryKey : primaryKeys){ - if(refData.containsKey(primaryKey)){ - keyBuilder.append("_").append(refData.get(primaryKey)); + if(!refData.containsKey(primaryKey)){ + return null; } - return null; + keyBuilder.append("_").append(refData.get(primaryKey)); } return keyBuilder.toString(); } From da409ef8f08615c8ecef896c47f65ff179577b49 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 11 Mar 2020 16:08:25 +0800 Subject: [PATCH 134/523] =?UTF-8?q?=E9=87=8D=E6=9E=84redis=E7=BB=B4?= =?UTF-8?q?=E8=A1=A8cache=3Dall=E7=9A=84=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/redisSide.md | 11 +- .../flink/sql/side/redis/RedisAllReqRow.java | 152 ++++++++---------- .../sql/side/redis/RedisAsyncReqRow.java | 11 +- .../flink/sql/side/redis/enums/RedisType.java | 33 ++++ 4 files changed, 113 insertions(+), 94 deletions(-) create mode 100644 redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/enums/RedisType.java diff --git a/docs/redisSide.md b/docs/redisSide.md index 9c7f4b47e..f9b5e41f9 100644 --- a/docs/redisSide.md +++ b/docs/redisSide.md @@ -4,6 +4,7 @@ CREATE TABLE tableName( colName cloType, ... + PRIMARY KEY(colName1,colName2) , PERIOD FOR SYSTEM_TIME )WITH( type ='redis', @@ -27,9 +28,10 @@ | tableName | 注册到flink的表名称(可选填;不填默认和redis对应的表名称相同)| | colName | 列名称，维表列名格式 表名:主键名:主键值:列名]| | colType | 列类型，当前只支持varchar| +| PRIMARY KEY |主键，多个字段做为联合主键时以逗号分隔 | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| -## 3.参数 +## 4.参数 |参数名称|含义|是否必填|默认值| |----|---|---|----| @@ -51,7 +53,7 @@ * cacheTTLMs:缓存的过期时间(ms) * ALL: 缓存全量表数据 -## 4.样例 +## 5.样例 ``` create table sideTable( channel varchar, @@ -70,5 +72,10 @@ create table sideTable( ); ``` +## 6.缓存redis的存储结构规则 +``` +redis使用散列类型 hash 数据结构，key=tableName_primaryKey1_primaryKey2,value={column1=value1, column2=value2} +如果以班级class表为例，id和name作为联合主键，那么redis的结构为 +``` diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 7437d74e5..d9f3a4eac 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -19,10 +19,14 @@ package com.dtstack.flink.sql.side.redis; import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.redis.enums.RedisType; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; import com.esotericsoftware.minlog.Log; import org.apache.calcite.sql.JoinType; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; @@ -103,104 +107,51 @@ public void flatMap(Row row, Collector out) throws Exception { String columnName = sideInfo.getEqualFieldList().get(conValIndex); inputParams.put(columnName, equalObj.toString()); } - String key = buildKey(inputParams); + String key = buildCacheKey(inputParams); + if(StringUtils.isBlank(key)){ + return; + } Map cacheMap = cacheRef.get().get(key); - - if (cacheMap == null){ - if(sideInfo.getJoinType() == JoinType.LEFT){ - Row data = fillData(row, null); - out.collect(data); - }else{ + if(MapUtils.isEmpty(cacheMap)){ + if(sideInfo.getJoinType() != JoinType.LEFT){ return; } - + Row data = fillData(row, null); + out.collect(data); return; - } + } Row newRow = fillData(row, cacheMap); out.collect(newRow); } - private String buildKey(Map inputParams) { - String tableName = tableInfo.getTableName(); - StringBuilder key = new StringBuilder(); - for (int i=0; i refData) { + StringBuilder keyBuilder = new StringBuilder(tableInfo.getTableName()); + List primaryKeys = tableInfo.getPrimaryKeys(); + for(String primaryKey : primaryKeys){ + if(!refData.containsKey(primaryKey)){ + return null; + } + keyBuilder.append("_").append(refData.get(primaryKey)); } - return key.toString(); + return keyBuilder.toString(); } + private void loadData(Map> tmpCache) throws SQLException { JedisCommands jedis = null; - try { - for(int i=0; i keys = ((Jedis) jedis).keys(perKey); - List newPerKeys = new LinkedList<>(); - for (String key : keys){ - String[] splitKey = key.split(":"); - String newKey = splitKey[0] + ":" + splitKey[1] + ":" + splitKey[2]; - newPerKeys.add(newKey); - } - List list = newPerKeys.stream().distinct().collect(Collectors.toList()); - for(String key : list){ - Map kv = Maps.newHashMap(); - String[] primaryKv = key.split(":"); - kv.put(primaryKv[1], primaryKv[2]); - String pattern = key + "*"; - Set realKeys = ((Jedis) jedis).keys(pattern); - for (String realKey : realKeys){ - kv.put(realKey.split(":")[3], jedis.get(realKey)); - } - tmpCache.put(key, kv); - } - } else { - String perKey = tableInfo.getTableName() + "*"; - Set keys = keys((JedisCluster) jedis, perKey); - List newPerKeys = new LinkedList<>(); - for (String key : keys){ - String[] splitKey = key.split(":"); - String newKey = splitKey[0] + ":" + splitKey[1] + ":" + splitKey[2]; - newPerKeys.add(newKey); - } - List list = newPerKeys.stream().distinct().collect(Collectors.toList()); - for(String key : list){ - Map kv = Maps.newHashMap(); - String[] primaryKv = key.split(":"); - kv.put(primaryKv[1], primaryKv[2]); - String pattern = key + "*"; - Set realKeys = keys((JedisCluster) jedis, pattern); - for (String realKey : realKeys){ - kv.put(realKey.split(":")[3], jedis.get(realKey)); - } - tmpCache.put(key, kv); - } + String keyPattern = tableInfo.getTableName() + "*"; + jedis = getJedisWithRetry(CONN_RETRY_NUM); + Set keys = getRedisKeys(RedisType.parse(tableInfo.getRedisType()), jedis, keyPattern); + if(CollectionUtils.isEmpty(keys)){ + return; } - - + JedisCommands redis = jedis; + keys.forEach(k ->{ + tmpCache.put(k, redis.hgetAll(k)); + }); } catch (Exception e){ LOG.error("", e); } finally { @@ -245,33 +196,58 @@ private JedisCommands getJedis(RedisSideTableInfo tableInfo) { } JedisCommands jedis = null; GenericObjectPoolConfig poolConfig = setPoolConfig(tableInfo.getMaxTotal(), tableInfo.getMaxIdle(), tableInfo.getMinIdle()); - switch (tableInfo.getRedisType()){ + switch (RedisType.parse(tableInfo.getRedisType())){ //单机 - case 1: + case STANDALONE: pool = new JedisPool(poolConfig, firstIp, Integer.parseInt(firstPort), timeout, password, Integer.parseInt(database)); jedis = pool.getResource(); break; //哨兵 - case 2: + case SENTINEL: jedisSentinelPool = new JedisSentinelPool(tableInfo.getMasterName(), ipPorts, poolConfig, timeout, password, Integer.parseInt(database)); jedis = jedisSentinelPool.getResource(); break; //集群 - case 3: + case CLUSTER: jedis = new JedisCluster(addresses, timeout, timeout,1, poolConfig); + default: + break; } return jedis; } - private Set keys(JedisCluster jedisCluster, String pattern){ + private JedisCommands getJedisWithRetry(int retryNum) { + while (retryNum-- > 0){ + try { + return getJedis(tableInfo); + } catch (Exception e) { + if(retryNum <= 0){ + throw new RuntimeException("getJedisWithRetry error", e); + } + try { + String jedisInfo = "url:" + tableInfo.getUrl() + ",pwd:" + tableInfo.getPassword() + ",database:" + tableInfo.getDatabase(); + LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + jedisInfo); + Thread.sleep(5 * 1000); + } catch (InterruptedException e1) { + LOG.error("", e1); + } + } + } + return null; + } + + private Set getRedisKeys(RedisType redisType, JedisCommands jedis, String keyPattern){ + if(!redisType.equals(RedisType.CLUSTER)){ + return ((Jedis) jedis).keys(keyPattern); + } Set keys = new TreeSet<>(); - Map clusterNodes = jedisCluster.getClusterNodes(); + Map clusterNodes = ((JedisCluster)jedis).getClusterNodes(); for(String k : clusterNodes.keySet()){ JedisPool jp = clusterNodes.get(k); Jedis connection = jp.getResource(); try { - keys.addAll(connection.keys(pattern)); + keys.addAll(connection.keys(keyPattern)); } catch (Exception e){ LOG.error("Getting keys error: {}", e); } finally { diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 053621acd..768c0cd4e 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; +import com.dtstack.flink.sql.side.redis.enums.RedisType; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; import io.lettuce.core.RedisClient; @@ -86,15 +87,15 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ if (database == null){ database = "0"; } - switch (tableInfo.getRedisType()){ - case 1: + switch (RedisType.parse(tableInfo.getRedisType())){ + case STANDALONE: StringBuilder redisUri = new StringBuilder(); redisUri.append("redis://").append(password).append(url).append("/").append(database); redisClient = RedisClient.create(redisUri.toString()); connection = redisClient.connect(); async = connection.async(); break; - case 2: + case SENTINEL: StringBuilder sentinelUri = new StringBuilder(); sentinelUri.append("redis-sentinel://").append(password) .append(url).append("/").append(database).append("#").append(redisSideTableInfo.getMasterName()); @@ -102,12 +103,14 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ connection = redisClient.connect(); async = connection.async(); break; - case 3: + case CLUSTER: StringBuilder clusterUri = new StringBuilder(); clusterUri.append("redis://").append(password).append(url); clusterClient = RedisClusterClient.create(clusterUri.toString()); clusterConnection = clusterClient.connect(); async = clusterConnection.async(); + default: + break; } } diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/enums/RedisType.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/enums/RedisType.java new file mode 100644 index 000000000..048c32b08 --- /dev/null +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/enums/RedisType.java @@ -0,0 +1,33 @@ +package com.dtstack.flink.sql.side.redis.enums; + +public enum RedisType { + /** + * 单机 + */ + STANDALONE(1), + /** + * 哨兵 + */ + SENTINEL(2), + /** + * 集群 + */ + CLUSTER(3); + int type; + RedisType(int type){ + this.type = type; + } + + public int getType(){ + return type; + } + + public static RedisType parse(int redisType){ + for(RedisType type : RedisType.values()){ + if(type.getType() == redisType){ + return type; + } + } + throw new RuntimeException("unsupport redis type["+ redisType + "]"); + } +} From 6585ad97c1aebab3e9452762e08da585393abf06 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 11 Mar 2020 18:46:24 +0800 Subject: [PATCH 135/523] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=92=8C=E6=9F=A5=E8=AF=A2key=E6=AD=A3=E5=88=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/redis/RedisAllReqRow.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index d9f3a4eac..fc09c0749 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -89,7 +89,7 @@ protected void reloadCache() { } cacheRef.set(newCache); - LOG.info("----- Redis all cacheRef reload end:{}", Calendar.getInstance()); + LOG.info("----- Redis all cacheRef reload end:{}", newCache.size()); } @Override @@ -142,16 +142,18 @@ private String buildCacheKey(Map refData) { private void loadData(Map> tmpCache) throws SQLException { JedisCommands jedis = null; try { - String keyPattern = tableInfo.getTableName() + "*"; + StringBuilder keyPattern = new StringBuilder(tableInfo.getTableName()); + for(String key : tableInfo.getPrimaryKeys()){ + keyPattern.append("_").append("*"); + }; jedis = getJedisWithRetry(CONN_RETRY_NUM); - Set keys = getRedisKeys(RedisType.parse(tableInfo.getRedisType()), jedis, keyPattern); + Set keys = getRedisKeys(RedisType.parse(tableInfo.getRedisType()), jedis, keyPattern.toString()); if(CollectionUtils.isEmpty(keys)){ return; } - JedisCommands redis = jedis; - keys.forEach(k ->{ - tmpCache.put(k, redis.hgetAll(k)); - }); + for(String key : keys){ + tmpCache.put(key, jedis.hgetAll(key)); + } } catch (Exception e){ LOG.error("", e); } finally { @@ -174,7 +176,7 @@ private void loadData(Map> tmpCache) throws SQLExcep private JedisCommands getJedis(RedisSideTableInfo tableInfo) { String url = tableInfo.getUrl(); String password = tableInfo.getPassword(); - String database = tableInfo.getDatabase(); + String database = tableInfo.getDatabase() == null ? "0" : tableInfo.getDatabase(); int timeout = tableInfo.getTimeout(); if (timeout == 0){ timeout = 1000; From d56226622c15ca0c3ece5b8103c6012dd0a0dde9 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 12 Mar 2020 10:41:46 +0800 Subject: [PATCH 136/523] fix conflict --- .../flink/sql/environment/StreamEnvConfigManager.java | 6 ++++++ hbase/pom.xml | 8 +------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index 2522437c5..d7b772c6e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -18,6 +18,12 @@ package com.dtstack.flink.sql.environment; +import com.dtstack.flink.sql.constrant.ConfigConstrant; +import com.dtstack.flink.sql.enums.EStateBackend; +import com.dtstack.flink.sql.util.MathUtil; +import com.dtstack.flink.sql.util.PropertiesUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.time.Time; diff --git a/hbase/pom.xml b/hbase/pom.xml index f64ffab1b..17f767215 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -29,13 +29,7 @@ 1.0-SNAPSHOT provided - - jdk.tools - jdk.tools - 1.6 - system - ${JAVA_HOME}/lib/tools.jar - + org.apache.hbase From c15fcacde7f1fe93267dc038d77c32f3667b5f02 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 12 Mar 2020 12:53:57 +0800 Subject: [PATCH 137/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dredis=20all=20cache?= =?UTF-8?q?=E4=B8=AD=E5=8F=96=E5=80=BCbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/redis/RedisAllReqRow.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index fc09c0749..69a99b32b 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -95,7 +95,8 @@ protected void reloadCache() { @Override public void flatMap(Row row, Collector out) throws Exception { Map inputParams = Maps.newHashMap(); - for(Integer conValIndex : sideInfo.getEqualValIndex()){ + for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { + Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = row.getField(conValIndex); if(equalObj == null){ if(sideInfo.getJoinType() == JoinType.LEFT){ @@ -104,8 +105,7 @@ public void flatMap(Row row, Collector out) throws Exception { } return; } - String columnName = sideInfo.getEqualFieldList().get(conValIndex); - inputParams.put(columnName, equalObj.toString()); + inputParams.put(sideInfo.getEqualFieldList().get(i), equalObj.toString()); } String key = buildCacheKey(inputParams); if(StringUtils.isBlank(key)){ From ba2d191ccbb2444a824205ce413457dcc8aacccd Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 12 Mar 2020 18:42:35 +0800 Subject: [PATCH 138/523] rdb connect pool size --- .../clickhouse/ClickhouseAsyncReqRow.java | 6 ++--- .../flink/sql/side/AbstractSideTableInfo.java | 14 ++++++++++ .../flink/sql/side/BaseAsyncReqRow.java | 2 -- .../sql/table/AbstractSideTableParser.java | 9 +++++++ .../flink/sql/side/db2/Db2AsyncReqRow.java | 4 +-- hbase/pom.xml | 7 ----- .../sql/side/impala/ImpalaAsyncReqRow.java | 25 ++++++++--------- .../sql/side/mysql/MysqlAsyncReqRow.java | 9 ++----- .../sql/side/oracle/OracleAsyncReqRow.java | 9 ++----- .../sql/side/polardb/PolardbAsyncReqRow.java | 4 +-- .../postgresql/PostgresqlAsyncReqRow.java | 4 +-- .../side/rdb/all/AbstractRdbAllReqRow.java | 13 +++++++++ .../sql/side/rdb/async/RdbAsyncReqRow.java | 27 ++++++++++++++++++- .../side/sqlserver/SqlserverAsyncReqRow.java | 4 +-- 14 files changed, 88 insertions(+), 49 deletions(-) diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index e6f008d63..ec9a77c6b 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -48,14 +48,14 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); clickhouseClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", CLICKHOUSE_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS); System.setProperty("vertx.disableFileCPResolving", "true"); VertxOptions vo = new VertxOptions(); - vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setEventLoopPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index cacba74fd..0fe177f96 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -55,6 +55,8 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements public static final String ASYNC_TIMEOUT_NUM_KEY = "asyncTimeoutNum"; + public static final String ASYNC_REQ_POOL_KEY = "asyncPoolSize"; + private String cacheType = "none";//None or LRU or ALL private int cacheSize = 10000; @@ -65,6 +67,11 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements private int asyncTimeout=10000; + /** + * async operator req outside conn pool size, egg rdb conn pool size + */ + private int asyncPoolSize = 0; + private int asyncTimeoutNumLimit = Integer.MAX_VALUE; private boolean partitionedJoin = false; @@ -156,4 +163,11 @@ public void setAsyncTimeoutNumLimit(int asyncTimeoutNumLimit) { this.asyncTimeoutNumLimit = asyncTimeoutNumLimit; } + public int getAsyncPoolSize() { + return asyncPoolSize; + } + + public void setAsyncPoolSize(int asyncPoolSize) { + this.asyncPoolSize = asyncPoolSize; + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index a0deb6b74..e5bce4437 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -30,14 +30,12 @@ import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; -import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collections; -import java.util.concurrent.TimeoutException; /** * All interfaces inherit naming rules: type + "AsyncReqRow" such as == "MysqlAsyncReqRow diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 39b72c22f..4973a891e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -23,6 +23,8 @@ import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.MathUtil; +import org.apache.flink.util.Preconditions; + import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -114,6 +116,13 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0 && asyncPoolSize <= 20, "asyncPoolSize size limit (0,20]"); + sideTableInfo.setAsyncPoolSize(asyncPoolSize); + } + } } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index 240bab9d1..8e7275977 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -59,7 +59,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); db2lientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", DB2_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -71,7 +71,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, db2lientConfig)); diff --git a/hbase/pom.xml b/hbase/pom.xml index f64ffab1b..835457e29 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -29,13 +29,6 @@ 1.0-SNAPSHOT provided - - jdk.tools - jdk.tools - 1.6 - system - ${JAVA_HOME}/lib/tools.jar - org.apache.hbase diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 77d3f3007..7ab603650 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -57,31 +57,28 @@ public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { @@ -61,7 +56,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); mysqlClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", MYSQL_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -73,7 +68,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index c0b37e7ac..938fd870c 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -30,16 +30,11 @@ import io.vertx.ext.jdbc.JDBCClient; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; public class OracleAsyncReqRow extends RdbAsyncReqRow { - - private static final Logger LOG = LoggerFactory.getLogger(OracleAsyncReqRow.class); - private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; public OracleAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { @@ -53,7 +48,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); oracleClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", ORACLE_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -65,7 +60,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 286f17286..13d25b111 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -55,7 +55,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); mysqlClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", POLARDB_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -67,7 +67,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index 2a9e0137c..0efc077e0 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -60,13 +60,13 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); pgClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", POSTGRESQL_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()); VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, pgClientConfig)); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 17daecfb0..8fb35b7eb 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.rdb.all; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.configuration.Configuration; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; @@ -73,6 +74,18 @@ public AbstractRdbAllReqRow(BaseSideInfo sideInfo) { super(sideInfo); } + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + LOG.info("jdbc url: {} ", tableInfo.getUrl()); + LOG.info("query table name: {}", tableInfo.getTableName()); + LOG.info("all cache type: {}", tableInfo.getCacheType()); + LOG.info("all cache size: {}", tableInfo.getCacheSize()); + LOG.info("all cache timeout: {}", tableInfo.getCacheTimeout()); + } + + @Override protected void initCache() throws SQLException { Map>> newCache = Maps.newConcurrentMap(); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index f78c30c30..b4a1c25e2 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -24,11 +24,13 @@ import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.cache.CacheObj; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import io.vertx.core.json.JsonArray; import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import com.google.common.collect.Lists; +import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -57,7 +59,9 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { public final static int DEFAULT_VERTX_WORKER_POOL_SIZE = Runtime.getRuntime().availableProcessors() * 2; - public final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; + public final static int DEFAULT_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; + + public final static int MAX_DB_CONN_POOL_SIZE_LIMIT = 20; public final static int DEFAULT_IDLE_CONNECTION_TEST_PEROID = 60; @@ -71,6 +75,27 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); + init(sideInfo); + } + + protected void init(BaseSideInfo sideInfo) { + RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + int defaultAsyncPoolSize = Math.min(MAX_DB_CONN_POOL_SIZE_LIMIT, DEFAULT_DB_CONN_POOL_SIZE); + int rdbPoolSize = rdbSideTableInfo.getAsyncPoolSize() > 0 ? rdbSideTableInfo.getAsyncPoolSize() : defaultAsyncPoolSize; + rdbSideTableInfo.setAsyncPoolSize(rdbPoolSize); + } + + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + LOG.info("use rdb pool size: {}", rdbSideTableInfo.getAsyncPoolSize()); + LOG.info("jdbc url: {} ", rdbSideTableInfo.getUrl()); + LOG.info("query table name: {}", rdbSideTableInfo.getTableName()); + LOG.info("async cache type: {}", rdbSideTableInfo.getCacheType()); + LOG.info("async cache mode: {}", rdbSideTableInfo.getCacheMode()); + LOG.info("async cache capacity: {}", rdbSideTableInfo.getAsyncCapacity()); + LOG.info("async cache timeout: {}", rdbSideTableInfo.getAsyncTimeout()); } @Override diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index f78061fca..a63d28607 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -55,7 +55,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); sqlserverClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", SQLSERVER_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -68,7 +68,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); From 229afced132eb5a367914b019daf08e4347e61e7 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 13 Mar 2020 11:12:56 +0800 Subject: [PATCH 139/523] connection info --- .../flink/sql/side/AbstractSideTableInfo.java | 15 +++++++++++++++ .../sql/side/rdb/all/AbstractRdbAllReqRow.java | 6 +----- .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 8 +------- .../sql/side/rdb/table/RdbSideTableInfo.java | 11 +++++++++++ 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index 0fe177f96..7a832d0a1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -170,4 +170,19 @@ public int getAsyncPoolSize() { public void setAsyncPoolSize(int asyncPoolSize) { this.asyncPoolSize = asyncPoolSize; } + + @Override + public String toString() { + return "Cache Info{" + + "cacheType='" + cacheType + '\'' + + ", cacheSize=" + cacheSize + + ", cacheTimeout=" + cacheTimeout + + ", asyncCapacity=" + asyncCapacity + + ", asyncTimeout=" + asyncTimeout + + ", asyncPoolSize=" + asyncPoolSize + + ", asyncTimeoutNumLimit=" + asyncTimeoutNumLimit + + ", partitionedJoin=" + partitionedJoin + + ", cacheMode='" + cacheMode + '\'' + + '}'; + } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 8fb35b7eb..2e9986742 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -78,11 +78,7 @@ public AbstractRdbAllReqRow(BaseSideInfo sideInfo) { public void open(Configuration parameters) throws Exception { super.open(parameters); RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - LOG.info("jdbc url: {} ", tableInfo.getUrl()); - LOG.info("query table name: {}", tableInfo.getTableName()); - LOG.info("all cache type: {}", tableInfo.getCacheType()); - LOG.info("all cache size: {}", tableInfo.getCacheSize()); - LOG.info("all cache timeout: {}", tableInfo.getCacheTimeout()); + LOG.info("rdb dim table config info: {} ", tableInfo.toString()); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index b4a1c25e2..356e9d665 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -89,13 +89,7 @@ protected void init(BaseSideInfo sideInfo) { public void open(Configuration parameters) throws Exception { super.open(parameters); RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - LOG.info("use rdb pool size: {}", rdbSideTableInfo.getAsyncPoolSize()); - LOG.info("jdbc url: {} ", rdbSideTableInfo.getUrl()); - LOG.info("query table name: {}", rdbSideTableInfo.getTableName()); - LOG.info("async cache type: {}", rdbSideTableInfo.getCacheType()); - LOG.info("async cache mode: {}", rdbSideTableInfo.getCacheMode()); - LOG.info("async cache capacity: {}", rdbSideTableInfo.getAsyncCapacity()); - LOG.info("async cache timeout: {}", rdbSideTableInfo.getAsyncTimeout()); + LOG.info("rdb dim table config info: {} ", rdbSideTableInfo.toString()); } @Override diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java index 506033007..9a4fa3c4f 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java @@ -99,4 +99,15 @@ public String getPassword() { public void setPassword(String password) { this.password = password; } + + @Override + public String toString() { + String cacheInfo = super.toString(); + String connectionInfo = "Rdb Side Connection Info{" + + "url='" + url + '\'' + + ", tableName='" + tableName + '\'' + + ", schema='" + schema + '\'' + + '}'; + return cacheInfo + " , " + connectionInfo; + } } From a965b11f71aedc06d796181bb24ec337bfed9013 Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Fri, 13 Mar 2020 17:44:14 +0800 Subject: [PATCH 140/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=9D=9E=E8=BF=9E?= =?UTF-8?q?=E7=BB=AD=20join=20=E6=83=85=E5=86=B5=E4=B8=8B=20as=20=E8=A1=A8?= =?UTF-8?q?=E6=9C=AA=E8=A2=AB=20=E5=A4=84=E7=90=86bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/JoinNodeDealer.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index a1c3cecb1..40ebc22c4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -190,11 +190,7 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, Set sideTableSet, tableInfo.setJoinType(joinType); tableInfo.setCondition(joinNode.getCondition()); - if(!needBuildTemp){ - return tableInfo; - } - - if(tableInfo.getLeftNode().getKind() != AS){ + if(tableInfo.getLeftNode().getKind() != AS && needBuildTemp){ extractTemporaryQuery(tableInfo.getLeftNode(), tableInfo.getLeftTableAlias(), (SqlBasicCall) parentWhere, parentSelectList, queueInfo, joinFieldSet, tableRef); }else { From 93f9d79e6b22124128f603a76338aaf66b976142 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 13 Mar 2020 20:15:15 +0800 Subject: [PATCH 141/523] deal char length --- .../flink/sql/table/AbsTableParser.java | 18 ++++++++++++-- .../dtstack/flink/sql/table/TableInfo.java | 20 ++++++++++++++++ .../sql/side/oracle/OracleAsyncSideInfo.java | 24 +++++++++++++++++++ .../sql/side/rdb/async/RdbAsyncSideInfo.java | 22 +++++++++++++---- 4 files changed, 77 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java index 3f4d1217d..dfaf5decc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java @@ -42,9 +42,11 @@ public abstract class AbsTableParser { private static final String PRIMARY_KEY = "primaryKey"; private static final String NEST_JSON_FIELD_KEY = "nestFieldKey"; + private static final String CHAR_TYPE_NO_LENGTH = "CHAR"; private static Pattern primaryKeyPattern = Pattern.compile("(?i)PRIMARY\\s+KEY\\s*\\((.*)\\)"); private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(\\w+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); + private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); private Map patternMap = Maps.newHashMap(); @@ -105,13 +107,25 @@ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ System.arraycopy(filedInfoArr, 0, filedNameArr, 0, filedInfoArr.length - 1); String fieldName = String.join(" ", filedNameArr); String fieldType = filedInfoArr[filedInfoArr.length - 1 ].trim(); - Class fieldClass = dbTypeConvertToJavaType(fieldType); + + + Class fieldClass = null; + TableInfo.FieldExtraInfo fieldExtraInfo = null; + + Matcher matcher = charTypePattern.matcher(fieldType); + if (matcher.find()) { + fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); + fieldExtraInfo = new TableInfo.FieldExtraInfo(); + fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); + } else { + fieldClass = dbTypeConvertToJavaType(fieldType); + } tableInfo.addPhysicalMappings(filedInfoArr[0],filedInfoArr[0]); tableInfo.addField(fieldName); tableInfo.addFieldClass(fieldClass); tableInfo.addFieldType(fieldType); - tableInfo.addFieldExtraInfo(null); + tableInfo.addFieldExtraInfo(fieldExtraInfo); } tableInfo.finish(); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java index 59e4fdd39..2fdc297a2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java @@ -194,6 +194,18 @@ public static class FieldExtraInfo implements Serializable { * default false：allow field is null */ boolean notNull = false; + /** + * field length,eg.char(4) + */ + int length; + + public int getLength() { + return length; + } + + public void setLength(int length) { + this.length = length; + } public boolean getNotNull() { return notNull; @@ -202,5 +214,13 @@ public boolean getNotNull() { public void setNotNull(boolean notNull) { this.notNull = notNull; } + + @Override + public String toString() { + return "FieldExtraInfo{" + + "notNull=" + notNull + + ", length=" + length + + '}'; + } } } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java index b811cf783..82d5640ce 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java @@ -23,9 +23,12 @@ import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.ParseUtils; +import com.mchange.lang.CharUtils; import org.apache.calcite.sql.SqlNode; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; @@ -35,6 +38,13 @@ public class OracleAsyncSideInfo extends RdbAsyncSideInfo { + private final static String SQL_DEFAULT_PLACEHOLDER = " ? "; + private final static String DEAL_CHAR_KEY = "char"; + + private static String rpadFormat = "rpad(?, %d, ' ')"; + + + public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -49,4 +59,18 @@ public String quoteIdentifier(String identifier) { return "\"" + identifier + "\""; } + @Override + public String wrapperPlaceholder(String fieldName) { + int pos = sideTableInfo.getFieldList().indexOf(fieldName); + String type = sideTableInfo.getFieldTypeList().get(pos); + + if (StringUtils.contains(type.toLowerCase(), DEAL_CHAR_KEY)) { + TableInfo.FieldExtraInfo fieldExtraInfo = sideTableInfo.getFieldExtraInfoList().get(pos); + int charLength = fieldExtraInfo.getLength(); + if (fieldExtraInfo.getLength() > 0) { + return String.format(rpadFormat, charLength); + } + } + return SQL_DEFAULT_PLACEHOLDER; + } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java index 0fe3eb0b9..d853d39fb 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java @@ -136,14 +136,26 @@ public String getAdditionalWhereClause() { public String getSelectFromStatement(String tableName, List selectFields, List conditionFields, List sqlJoinCompareOperate, List predicateInfoes) { - String fromClause = selectFields.stream().map(this::quoteIdentifier).collect(Collectors.joining(", ")); - String whereClause = conditionFields.stream().map(f -> quoteIdentifier(f) + sqlJoinCompareOperate.get(conditionFields.indexOf(f)) + " ? ") + String fromClause = selectFields.stream() + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")); + + String whereClause = conditionFields.stream() + .map(f -> quoteIdentifier(f) + sqlJoinCompareOperate.get(conditionFields.indexOf(f)) + wrapperPlaceholder(f)) + .collect(Collectors.joining(" AND ")); + + String predicateClause = predicateInfoes.stream() + .map(this::buildFilterCondition) .collect(Collectors.joining(" AND ")); - String predicateClause = predicateInfoes.stream().map(this::buildFilterCondition).collect(Collectors.joining(" AND ")); - String sql = "SELECT " + fromClause + " FROM " + tableName + (conditionFields.size() > 0 ? " WHERE " + whereClause : "") + String dimQuerySql = "SELECT " + fromClause + " FROM " + tableName + (conditionFields.size() > 0 ? " WHERE " + whereClause : "") + (predicateInfoes.size() > 0 ? " AND " + predicateClause : "") + getAdditionalWhereClause(); - return sql; + + return dimQuerySql; + } + + public String wrapperPlaceholder(String fieldName) { + return " ? "; } public String buildFilterCondition(PredicateInfo info) { From 31e706862ec103594203084b7b4c1acd289b5f35 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 16 Mar 2020 11:42:23 +0800 Subject: [PATCH 142/523] use rpad deal char --- .../sql/side/oracle/OracleAsyncSideInfo.java | 14 +++---- .../flink/sql/sink/oracle/OracleSink.java | 38 ++++++++++++++++--- .../dtstack/flink/sql/sink/rdb/RdbSink.java | 6 +-- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java index 82d5640ce..5ff27c542 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java @@ -38,11 +38,9 @@ public class OracleAsyncSideInfo extends RdbAsyncSideInfo { - private final static String SQL_DEFAULT_PLACEHOLDER = " ? "; - private final static String DEAL_CHAR_KEY = "char"; - - private static String rpadFormat = "rpad(?, %d, ' ')"; - + private final String SQL_DEFAULT_PLACEHOLDER = " ? "; + private final String DEAL_CHAR_KEY = "char"; + private String RPAD_FORMAT = "rpad(?, %d, ' ')"; public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { @@ -66,9 +64,9 @@ public String wrapperPlaceholder(String fieldName) { if (StringUtils.contains(type.toLowerCase(), DEAL_CHAR_KEY)) { TableInfo.FieldExtraInfo fieldExtraInfo = sideTableInfo.getFieldExtraInfoList().get(pos); - int charLength = fieldExtraInfo.getLength(); - if (fieldExtraInfo.getLength() > 0) { - return String.format(rpadFormat, charLength); + int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); + if (charLength > 0) { + return String.format(RPAD_FORMAT, charLength); } } return SQL_DEFAULT_PLACEHOLDER; diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index c30dc9c60..5a82fdf47 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -21,14 +21,17 @@ import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.ExtendOutputFormat; import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import com.google.common.collect.Lists; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Reason: @@ -40,6 +43,10 @@ public class OracleSink extends RdbSink implements IStreamSinkGener { private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; + private final String SQL_DEFAULT_PLACEHOLDER = " ? "; + private final String DEAL_CHAR_KEY = "char"; + private String RPAD_FORMAT = " rpad(?, %d, ' ') "; + @Override public String getDriverName() { return ORACLE_DRIVER; @@ -193,16 +200,35 @@ public String updateKeySql(Map> updateKey) { */ public String makeValues(List column) { StringBuilder sb = new StringBuilder("SELECT "); - for (int i = 0; i < column.size(); ++i) { - if (i != 0) { - sb.append(","); + String collect = column.stream() + .map(col -> wrapperPlaceholder(col) + DtStringUtil.addQuoteForStr(col)) + .collect(Collectors.joining(", ")); + + sb.append(collect).append(" FROM DUAL"); + return sb.toString(); + } + + /** + * char type is wrapped with rpad + * @param fieldName + * @return + */ + public String wrapperPlaceholder(String fieldName) { + int pos = rdbTableInfo.getFieldList().indexOf(fieldName); + String type = rdbTableInfo.getFieldTypeList().get(pos); + + if (StringUtils.contains(type.toLowerCase(), DEAL_CHAR_KEY)) { + TableInfo.FieldExtraInfo fieldExtraInfo = rdbTableInfo.getFieldExtraInfoList().get(pos); + int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); + if (charLength > 0) { + return String.format(RPAD_FORMAT, charLength); } - sb.append("? " + DtStringUtil.addQuoteForStr(column.get(i))); } - sb.append(" FROM DUAL"); - return sb.toString(); + return SQL_DEFAULT_PLACEHOLDER; } + + public boolean containsIgnoreCase(List l, String s) { Iterator it = l.iterator(); while (it.hasNext()) { diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java index babff68f9..e689169ea 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java @@ -83,6 +83,8 @@ public abstract class RdbSink implements RetractStreamTableSink, Serializab private String schema; + protected RdbTableInfo rdbTableInfo; + public RichSinkFunction createJdbcSinkFunc() { if (driverName == null || dbURL == null || userName == null || password == null || sqlTypes == null || tableName == null) { @@ -111,8 +113,7 @@ public RichSinkFunction createJdbcSinkFunc() { @Override public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { - RdbTableInfo rdbTableInfo = (RdbTableInfo) targetTableInfo; - + this.rdbTableInfo = (RdbTableInfo) targetTableInfo; String tmpDbURL = rdbTableInfo.getUrl(); String tmpUserName = rdbTableInfo.getUserName(); String tmpPassword = rdbTableInfo.getPassword(); @@ -263,7 +264,6 @@ public void setDbType(String dbType) { /** * sqlserver and oracle maybe implement * - * @param tableName * @param fieldNames * @param realIndexes * @return From e3607b5bc28826e7d1047fb2e4c68a14f460236f Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 16 Mar 2020 14:38:16 +0800 Subject: [PATCH 143/523] mysql quoteIdentifier --- .../com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java | 5 +++++ .../com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java | 5 +++++ .../dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java | 5 +++++ .../dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java index 81193c0f9..07b06ba38 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java @@ -38,4 +38,9 @@ public class MysqlAllSideInfo extends RdbAllSideInfo { public MysqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java index f72671ce0..5d6ed370d 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java @@ -39,4 +39,9 @@ public class MysqlAsyncSideInfo extends RdbAsyncSideInfo { public MysqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } } diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java index 2802bd402..822cbaa70 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java @@ -12,5 +12,10 @@ public class PolardbAllSideInfo extends RdbAllSideInfo { public PolardbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java index 0b47cf3f7..bad71375f 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java @@ -13,4 +13,9 @@ public class PolardbAsyncSideInfo extends RdbAsyncSideInfo { public PolardbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } } From b3e9ea6b20efb8994e885fd76042827bdb3bb57b Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 16 Mar 2020 20:00:37 +0800 Subject: [PATCH 144/523] =?UTF-8?q?=E6=97=B6=E9=97=B4=E6=A0=BC=E5=BC=8F?= =?UTF-8?q?=E5=8C=96=E5=8E=BB=E9=99=A4=E5=86=97=E4=BD=99=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/util/DateUtil.java | 410 ++++++++---------- .../sql/side/rdb/async/RdbAsyncReqRow.java | 4 +- 2 files changed, 194 insertions(+), 220 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index a0f5291e7..2b2dc4715 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -16,29 +16,24 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.util; import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.ZoneOffset; -import java.util.Calendar; -import java.util.Date; -import java.util.Locale; -import java.util.SimpleTimeZone; -import java.util.TimeZone; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.*; import java.util.regex.Pattern; import static java.time.format.DateTimeFormatter.ISO_INSTANT; /** - * + * * Reason: TODO ADD REASON(可选) * Date: 2017年03月10日 下午1:16:37 * Company: www.dtstack.com @@ -47,20 +42,16 @@ */ public class DateUtil { - static final String timeZone = "GMT+8"; - static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss"; - static final String dateFormat = "yyyy-MM-dd"; - static final String timeFormat = "HH:mm:ss"; - static final SimpleDateFormat datetimeFormatter = new SimpleDateFormat(datetimeFormat); - static final SimpleDateFormat dateFormatter = new SimpleDateFormat(dateFormat); - static final SimpleDateFormat timeFormatter = new SimpleDateFormat(timeFormat); + static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss"); private static final Pattern DATETIME = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); private static final Pattern DATE = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}$"); - private static final int MILLIS_PER_SECOND = 1000; + public static java.sql.Date columnToDate(Object column) { if(column instanceof String) { return new java.sql.Date(stringToDate((String)column).getTime()); @@ -84,23 +75,32 @@ public static Date stringToDate(String strDate) { return null; } try { - return datetimeFormatter.parse(strDate); - } catch (ParseException ignored) { + ; + return localDateTimetoDate(LocalDateTime.parse(strDate, DATE_TIME_FORMATTER)); + } catch (DateTimeParseException ignored) { } try { - return dateFormatter.parse(strDate); - } catch (ParseException ignored) { + return localDateTimetoDate(LocalDate.parse(strDate, DATE_FORMATTER).atStartOfDay()); + } catch (DateTimeParseException ignored) { } try { - return timeFormatter.parse(strDate); - } catch (ParseException ignored) { + return localDateTimetoDate(LocalDateTime.of(LocalDate.now(), LocalTime.parse(strDate, TIME_FORMATTER))); + } catch (DateTimeParseException ignored) { } throw new RuntimeException("can't parse date"); } + public static Date localDateTimetoDate(LocalDateTime localDateTime){ + return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant()); + } + + public static LocalDateTime dateToLocalDateTime(Date date){ + return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } + /** * * @@ -130,13 +130,13 @@ public static long getTodayStart(long day) { * @return */ public static long getTodayStart(long day,String scope) { - if(scope.equals("MS")){ - return getTodayStart(day)*1000; - }else if(scope.equals("S")){ - return getTodayStart(day); - }else{ - return getTodayStart(day); - } + if("MS".equals(scope)){ + return getTodayStart(day)*1000; + }else if("S".equals(scope)){ + return getTodayStart(day); + }else{ + return getTodayStart(day); + } } /** @@ -168,15 +168,15 @@ public static long getNextDayStart(long day) { * @return */ public static long getNextDayStart(long day,String scope) { - if(scope.equals("MS")){ - return getNextDayStart(day)*1000; - }else if(scope.equals("S")){ - return getNextDayStart(day); - }else{ - return getNextDayStart(day); - } + if("MS".equals(scope)){ + return getNextDayStart(day)*1000; + }else if("S".equals(scope)){ + return getNextDayStart(day); + }else{ + return getNextDayStart(day); + } } - + /** * @@ -237,7 +237,7 @@ public static long getWeekFirst(long day) { /** * 根据某个日期时间戳秒值，获取所在周在一年中是第几周. - * + * * @param day * @return */ @@ -257,17 +257,17 @@ public static int getWeekOfYear(long day) { */ public static String getYesterdayByString(String day, String inFormat, String outFormat){ try { - SimpleDateFormat sdf = new SimpleDateFormat(inFormat); - Date date = sdf.parse(day); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - int calendarDay = calendar.get(Calendar.DATE); - calendar.set(Calendar.DATE, calendarDay - 1); - String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); - return dayBefore; - } catch (ParseException e) { - return null; - } + SimpleDateFormat sdf = new SimpleDateFormat(inFormat); + Date date = sdf.parse(day); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + int calendarDay = calendar.get(Calendar.DATE); + calendar.set(Calendar.DATE, calendarDay - 1); + String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); + return dayBefore; + } catch (ParseException e) { + return null; + } } /** @@ -288,7 +288,7 @@ public static String getTomorrowByString(String day, String inFormat, String out String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); return dayBefore; } - + /** * * @param date @@ -320,7 +320,7 @@ public static String get30DaysBeforeByString(String day, String inFormat, String calendar.set(Calendar.DATE, calendarDay - 30); return new SimpleDateFormat(outFormat).format(calendar.getTime()); } - + /** * * @param day @@ -330,14 +330,14 @@ public static String get30DaysBeforeByString(String day, String inFormat, String * @throws ParseException */ public static String get30DaysLaterByString(String day, String inFormat, String outFormat) throws ParseException { - SimpleDateFormat sdf = new SimpleDateFormat(inFormat); - Date date = sdf.parse(day); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - int calendarDay = calendar.get(Calendar.DATE); - calendar.set(Calendar.DATE, calendarDay + 30); - String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); - return dayBefore; + SimpleDateFormat sdf = new SimpleDateFormat(inFormat); + Date date = sdf.parse(day); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + int calendarDay = calendar.get(Calendar.DATE); + calendar.set(Calendar.DATE, calendarDay + 30); + String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); + return dayBefore; } @@ -349,7 +349,7 @@ public static String get30DaysLaterByString(String day, String inFormat, String * @return String * @throws ParseException */ - public static String getDateStrTOFormat(String day, String inFormat, String outFormat) throws ParseException { + public static String getDateStrToFormat(String day, String inFormat, String outFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -357,8 +357,8 @@ public static String getDateStrTOFormat(String day, String inFormat, String outF String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); return dayBefore; } - - public static long getDateMillTOFormat(String day, String inFormat) throws ParseException { + + public static long getDateMillToFormat(String day, String inFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -474,79 +474,79 @@ public static long getMillByLastWeekDay() { cal.set(Calendar.MILLISECOND, 0); return cal.getTimeInMillis() / 1000; } - - /** - * @return long - */ + + /** + * @return long + */ public static long getMillByDay(int severalDays,String condition) { - int dateT=0; + int dateT=0; Calendar cal = Calendar.getInstance(); - if(condition==null){ - return getMillToDay(cal,dateT); - } - if(condition.equals("-")){ - dateT = (cal.get(Calendar.DATE) - severalDays); - return getMillToDay(cal,dateT); - } - if(condition.equals("+")){ - dateT = (cal.get(Calendar.DATE) + severalDays); - return getMillToDay(cal,dateT); - } - return getMillToDay(cal,dateT); - } - + if(condition==null){ + return getMillToDay(cal,dateT); + } + if("-".equals(condition)){ + dateT = (cal.get(Calendar.DATE) - severalDays); + return getMillToDay(cal,dateT); + } + if("+".equals(condition)){ + dateT = (cal.get(Calendar.DATE) + severalDays); + return getMillToDay(cal,dateT); + } + return getMillToDay(cal,dateT); + } + /** * @return long */ public static long getStampByDay(int severalDays,String condition) { - int dateT=0; - Calendar cal = Calendar.getInstance(); - if(condition==null){ - return getStampToDay(cal,dateT); - } - if(condition.equals("-")){ - dateT = (cal.get(Calendar.DATE) - severalDays); - return getStampToDay(cal,dateT); - } - if(condition.equals("+")){ - dateT = (cal.get(Calendar.DATE) + severalDays); - return getStampToDay(cal,dateT); - } - return getStampToDay(cal,dateT); + int dateT=0; + Calendar cal = Calendar.getInstance(); + if(condition==null){ + return getStampToDay(cal,dateT); + } + if("-".equals(condition)){ + dateT = (cal.get(Calendar.DATE) - severalDays); + return getStampToDay(cal,dateT); + } + if("+".equals(condition)){ + dateT = (cal.get(Calendar.DATE) + severalDays); + return getStampToDay(cal,dateT); + } + return getStampToDay(cal,dateT); } /** * @return long */ public static long getMillByDay(){ - return getMillByDay(0,null); + return getMillByDay(0,null); } - + /** * @param cal Calendar - * @param dateT Integer + * @param dateT Integer * @return long */ public static long getMillToDay(Calendar cal,int dateT){ - if(dateT!=0){ - cal.set(Calendar.DATE, dateT); - } - cal.set(Calendar.HOUR_OF_DAY, 0); - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - return cal.getTimeInMillis()/1000; - } - + if(dateT!=0){ + cal.set(Calendar.DATE, dateT); + } + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + return cal.getTimeInMillis()/1000; + } + /** * @param cal Calendar - * @param dateT Integer + * @param dateT Integer * @return long */ public static long getStampToDay(Calendar cal,int dateT){ - if(dateT!=0){ - cal.set(Calendar.DATE, dateT); - } - return cal.getTimeInMillis(); + if(dateT!=0){ + cal.set(Calendar.DATE, dateT); + } + return cal.getTimeInMillis(); } public static String getToday() { @@ -568,7 +568,7 @@ public static String getDate(long day, String format) { SimpleDateFormat sf = new SimpleDateFormat(format); return sf.format(cal.getTime()); } - + /** * * @param date @@ -578,21 +578,21 @@ public static String getDate(Date date, String format) { SimpleDateFormat sf = new SimpleDateFormat(format); return sf.format(date); } - - + + /** * * @param day * @param format * @return long - * @throws ParseException + * @throws ParseException */ public static long stringToLong(String day, String format) throws ParseException { - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - long Date = dateFormat.parse(day).getTime(); - return Date; + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + long date = dateFormat.parse(day).getTime(); + return date; } - + /** * @param day * @param format @@ -600,30 +600,30 @@ public static long stringToLong(String day, String format) throws ParseException * @throws ParseException */ public static Date stringToDate(String day, String format) { - try { - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - Date Date = dateFormat.parse(day); - return Date; - } catch (ParseException e) { - return new Date(); - } + try { + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + Date date = dateFormat.parse(day); + return date; + } catch (ParseException e) { + return new Date(); + } } - - + + /** * long型时间戳转为String型 - * + * * @param day 秒 * @return 格式化后的日期 - * @throws ParseException + * @throws ParseException */ public static String longToString(long day, String format) throws ParseException { - if (("" + day).length() <= 10){ + if (("" + day).length() <= 10){ day=day*1000; } - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - String Date = dateFormat.format(day); - return Date; + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + String date = dateFormat.format(day); + return date; } /** @@ -653,39 +653,39 @@ public static long getMillByNow() { return cal.getTimeInMillis(); } - public static int getWeeksBetweenTwoDates(long startDay, long endDay) { - int week = getWeekOfYear(endDay) - getWeekOfYear(startDay) + 1; - if(week<1){ - week = getWeekOfYear(endDay) + getMaxWeekOfYear(startDay) - getWeekOfYear(startDay) + 1; - } - return week; - } + public static int getWeeksBetweenTwoDates(long startDay, long endDay) { + int week = getWeekOfYear(endDay) - getWeekOfYear(startDay) + 1; + if(week<1){ + week = getWeekOfYear(endDay) + getMaxWeekOfYear(startDay) - getWeekOfYear(startDay) + 1; + } + return week; + } - public static int getMaxWeekOfYear(long startDay) { - Calendar cal = Calendar.getInstance(); + public static int getMaxWeekOfYear(long startDay) { + Calendar cal = Calendar.getInstance(); cal.setTime(new Date(startDay * 1000)); return cal.getMaximum(Calendar.WEEK_OF_YEAR); - } - - public static int getMonthsBetweenTwoDates(long startDay, long endDay) { - int month = DateUtil.getMonth(endDay) - DateUtil.getMonth(startDay) + 1; - if(month<1){ - month = getMonth(endDay) + 12 - getMonth(startDay) +1; - } - return month; - } - - public static Date parseDate(String dateStr, String pattern){ - SimpleDateFormat sdf = new SimpleDateFormat(); - sdf.applyPattern(pattern); - try { - return sdf.parse(dateStr); - } catch (ParseException e) { - return null; - } - } - - /** + } + + public static int getMonthsBetweenTwoDates(long startDay, long endDay) { + int month = DateUtil.getMonth(endDay) - DateUtil.getMonth(startDay) + 1; + if(month<1){ + month = getMonth(endDay) + 12 - getMonth(startDay) +1; + } + return month; + } + + public static Date parseDate(String dateStr, String pattern){ + SimpleDateFormat sdf = new SimpleDateFormat(); + sdf.applyPattern(pattern); + try { + return sdf.parse(dateStr); + } catch (ParseException e) { + return null; + } + } + + /** * * @param time Long 时间 * @return long @@ -703,7 +703,7 @@ public static long getMinuteStart(long time) { firstDay = cal.getTimeInMillis() / 1000; return firstDay; } - + /** * @param time Long * @return long @@ -732,24 +732,24 @@ public static Date getDateByLong(long time){ date.setTime(time); return date; } - + public static Date parseDate(String dateStr, String pattern, Locale locale){ - SimpleDateFormat df = new SimpleDateFormat( - pattern, locale); - - df.setTimeZone(new SimpleTimeZone(0, "GMT")); - try { - return df.parse(dateStr); - } catch (ParseException e) { - return null; - } - } - + SimpleDateFormat df = new SimpleDateFormat( + pattern, locale); + + df.setTimeZone(new SimpleTimeZone(0, "GMT")); + try { + return df.parse(dateStr); + } catch (ParseException e) { + return null; + } + } + public static String getDate(Date date, String format, Locale locale) { - SimpleDateFormat df = new SimpleDateFormat( - format, locale); - df.setTimeZone(new SimpleTimeZone(0, "GMT")); + SimpleDateFormat df = new SimpleDateFormat( + format, locale); + df.setTimeZone(new SimpleTimeZone(0, "GMT")); return df.format(date); } @@ -777,59 +777,33 @@ public static java.sql.Timestamp columnToTimestamp(Object column) { } public static String dateToString(Date date) { - return dateFormatter.format(date); + LocalDateTime localDateTime = dateToLocalDateTime(date); + return localDateTime.format(DATE_FORMATTER); } public static String timestampToString(Date date) { - return datetimeFormatter.format(date); + LocalDateTime localDateTime = dateToLocalDateTime(date); + return localDateTime.format(DATE_TIME_FORMATTER); } - public static Timestamp getTimestampFromStr(String timeStr) { if (DATETIME.matcher(timeStr).matches()) { Instant instant = Instant.from(ISO_INSTANT.parse(timeStr)); return new Timestamp(instant.getEpochSecond() * MILLIS_PER_SECOND); - } else { - java.sql.Date date = null; - try { - date = new java.sql.Date(datetimeFormatter.parse(timeStr).getTime()); - } catch (ParseException e) { - throw new RuntimeException("getTimestampFromStr error data is " + timeStr); - } - return new Timestamp(date.getTime()); } + return new Timestamp(stringToDate(timeStr).getTime()); } public static java.sql.Date getDateFromStr(String dateStr) { - // 2020-01-01 format if (DATE.matcher(dateStr).matches()) { - // convert from local date to instant Instant instant = LocalDate.parse(dateStr).atTime(LocalTime.of(0, 0, 0, 0)).toInstant(ZoneOffset.UTC); - // calculate the timezone offset in millis int offset = TimeZone.getDefault().getOffset(instant.toEpochMilli()); - // need to remove the offset since time has no TZ component return new java.sql.Date(instant.toEpochMilli() - offset); } else if (DATETIME.matcher(dateStr).matches()) { - // 2020-01-01T12:12:12Z format Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); return new java.sql.Date(instant.toEpochMilli()); - } else { - try { - // 2020-01-01 12:12:12.0 format - return new java.sql.Date(datetimeFormatter.parse(dateStr).getTime()); - } catch (ParseException e) { - throw new RuntimeException("String convert to Date fail."); - } } + return new java.sql.Date(stringToDate(dateStr).getTime()); } - - - public static String getStringFromTimestamp(Timestamp timestamp) { - return datetimeFormatter.format(timestamp); - } - - public static String getStringFromDate(java.sql.Date date) { - return dateFormatter.format(date); - } - + } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index cda1968fc..f48e6e296 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -178,9 +178,9 @@ private Object convertDataType(Object val) { } else if (val instanceof Instant) { } else if (val instanceof Timestamp) { - val = DateUtil.getStringFromTimestamp((Timestamp) val); + val = DateUtil.timestampToString((Timestamp) val); } else if (val instanceof java.util.Date) { - val = DateUtil.getStringFromDate((java.sql.Date) val); + val = DateUtil.dateToString((java.util.Date)val); } else { val = val.toString(); } From f7af15baf0f739164fbbb761bec52028e29d73f4 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 17 Mar 2020 11:02:40 +0800 Subject: [PATCH 145/523] redme.md --- README.md | 3 +++ .../\351\222\211\351\222\211\347\276\244.jpg" | Bin 0 -> 91707 bytes 2 files changed, 3 insertions(+) create mode 100644 "images/\351\222\211\351\222\211\347\276\244.jpg" diff --git a/README.md b/README.md index db25164b6..717343edf 100644 --- a/README.md +++ b/README.md @@ -326,3 +326,6 @@ into # 招聘 1.大数据平台开发工程师，想了解岗位详细信息可以添加本人微信号ysqwhiletrue,注明招聘，如有意者发送简历至sishu@dtstack.com。 +# FlinkX & FlinkStreamSQL钉钉交流群 + +![image](images/钉钉群.jpg) diff --git "a/images/\351\222\211\351\222\211\347\276\244.jpg" "b/images/\351\222\211\351\222\211\347\276\244.jpg" new file mode 100644 index 0000000000000000000000000000000000000000..a005c552544342162169e85ddc053d9f80915240 GIT binary patch literal 91707 zcmeFYbyQnX);JnUTco&aDIo-h;tmCZBxrCe6fat&&_a>6Sa1@81b26bQfRT_4#isB zr4%d9W9FNgZ|1%Et@Zof`u+3Xnwzz9_ucp0efHgVpL4dI+v(dy00~S9sszBo0sye? zet_FKfCAv*{Ra;p+<*A+!9yIJhq(A8`1p8u_!LCM1SHfHG&Ix{R8(|~tW0$DEDTgs zPq?43fY~@WIcS-9p7O9iWo74J{|gZ;92^{cTzqnTd~$X=DmwQ6@p1bFK#F@m6WbXZ z3kbMJiiJ&zb=wJGykqhnHrCw%`1^r_``{rS_WgT*xot%P0NlTS4;vr%(S4kU*blMs z09g01?_P%#hYXjTnSzF2R8HR$T{T8Y1r&t4xsM76Y3o^hK*eQMcTj_n<19*+uY+nj zMdYnLK7M5tR)FZ}8hEvf#pf&?fOA)VD_UjWbwdA_?*DrHorS-2awjmHyPJZfcMM|P z$HIDe|KS~@e`$erj}-ep88g4!gNHx?xP{w?s;}fMS))q_6iAS~w&m?CfZ(p-T_aL} zETHbmKZyPZ_6(608Bl5#)v0M>B|s#H$cS%PmF|$Z5pgKuPqv_dGS~)qiQpmZ z#t+BU@8^?pFK3+Km62@$>WH~(e#@p*&>U&}kj>Ojd!p)P274O}m)tzs48?zahjQi> zs1=(H-_j;pwK=Bw$g!on-ZpxnfTuo8$sJ}NuOJo3MIrZOkgkrqC6^RQy1T?u6&qaS zGgbL@<_Oi8E1|W=`%BeQ&nrRs=8&Mm@hW%s_~#UEKD3oO@?nmm<^Ax|z{FxA3EH9{ z6Nz6qWv3IDHje4mxdZ*<4RNDiM3z!5DxTnbn%H9*i;82ueCd*VCQMAW!mdNos~w5T z0X9{L%#lo3t~Z96lCru?c<86iAcMr58E*mAA3@hvZUr)>0c6yl3rxJllii3wk$qc| z@%=%enIB$T=a(3(CvDPwGQZ*Z+>hiV@pZuA{S??R%Qe>N`;B3#H`I* zN;>#Bzu>IDc)RbU;FX_Q;vB@jcjLd5td%V_%{zr`ZBI&`mL4oD#EGw zAH+-3Hy|Dr{Ib?`*%Xo=_RmM{4LuC|9R=#GQDr;KJVr5GRAC3l04%yY1$`*tDG}dG zqi}>trD%#T0h&hm@vy?`c9n8~f>n>d9`+U&-QNZ+=h0Blsr)FDZVNkfGGZXntA5UDL=>x*l;*Y@nM+Am`tY^dtfoRH52s???-KB6pQ5eXAXchWd1#k#@|Qlt?`oDa!H!6c!tk34 zl~B$rvv7oV*4ESzWjZ#(6Xa-6ZHjksdJ$wtOHImfksj&y{igx`JiKuDET%!#%JGeN z$I<2R+uGWK=pgruih^^^8Y#p4%zZUUQo8J=2_b9qE+ZwZE9YQXWD-iI_Kam|L`yGDPF(ow`fcpDI?4Y~2D>6p#5r&la|sMDp9MnpLM}-}Rad zGj_TCiD5d~I5DiGq0=r(&^-t_uG=Y-af{zI@vpuGG&#{O{Wh*K>TH7+6g@ZY+7b@~ z0F=J`lko3AtjNNG>RB)S<8Dv70SbD>HL{qebWZ?=s`lCGkh zOlpaI6EPCv3$kffdWI@|I~N7_%xdC9?Yt;jmSm5yLYl+&zGTSz*Q_$P^sM))7`rGppqn5ku+0VDICU_5e6w?45hKh%ARHwg{+(5NOJK3jgy{RCRQ98< z8iHGzzxGa}fAfcRM0TzV3=Z*sA`vxJAlBuySLabL;_eVav9MI_ffiyN>L7i2MNmPG z!*%{13tD{U(+PWLurGBTB67gYC)raSPIiji|G4qGHmZgI=! z@w?M`KM8d{e5LDLmV`!89KpEg5mgyj*EW^imN!*|>V9C|Avt2v)Xpk4G=r`}1!jB6 z#I+~Et7*Di@g2#0W+nJ@xImqhGw2rZ(}y*8ZhO!zqLWkNRfwmhk@1gLy6deE#Xlh> zAa7tO-*BEpUWE&SNVV54aU+`vS;_Tqid`#v&_047MB^G9Oy{0axOYqqm>4)O;U`@- z=f$>QRMwoT?)$Kb(OzWMJ<=p6mFS{f)h3k0|IYe&OmLMdQacqZU1O!nmayoxudWbx zH&0JtJv}_d5lWWe9T6DEbLO>6ZRjy^G?ynOKBFg9=rHDbhzHKk=PeC0eQ3RuKFa

o*CzL5g5v@2^EcMEV9s<;KPZ~eLjD4%2Jew*Wk z%f3!|PxDo}6P6(JTs$~Ipr6E!Bwk*mws1mK-pM;5yACbn1nE8>lr>`&%3q>Dg)&ZFD5A$@p0s+ zMrXH$ZZw#6jc(p^O(?H)IA~xn1#{8;_Rmu5b8p9ZUeTd~LzNj*wd8agMEzi%I5B>1 zBT7aWQ@U%lH!2&Y60jfRMRKJ_Q>LN5N_u$4Er?PDTGEM(QVmr|vK@vGf_`laFnuOJ z8xOtN{9{2Fup=$F^QiDCTL;{^oK59t-N;`Kz0=|%Vw@%wI`|77yr`nk(261@VP&wKrq08hyd>HtSkp3u7 z?$7R>Bz+|}w|*trF4(udeU($tYLViXf4vewFN?kZvqR}>OUv(4wvTXE=+#g>&O*&q z*1@m8H(F|qPCpNEK27-(+U-l+@J}Q)qMoEUN zz7}EY9Q5Y$dD;2+(@UFuT6_=69Wl&qRbsLI$5}7dS{`t9HDUr{^M&{OhqX3Qx$8c^ zjtdN=F)~WvOh8~(Nv*(ckxlGExZS@{QZ3_@Om$50M!WM>)*o7aKbl|XQDH3q4jW;P z()PE`%NyIk&3XbeeuZv=gW=tSH$>)75?}p!6_9;;ao;oRaO1J)Hz37=IPW^Vi*!L( zwaNNI&|jo61w##d5y2Um5VxalU#@Ty!$fi~)d)6flDv&-EmYlb&y|Z8P>^;-#(3>9`7`{%r(;=Y zMfOY%1BMWm>4Fu!M+OsU_K342^wS*b)whTOW(yOqdk$kB#iU0p!ff#>a3?tBP2h#N zy!}(fY;%3--->ur`ElQmGjU#yhZKAUh_E*(i zH!sacXqpq=VX5~5*K6Jeb$nG|k(lyj+XV9J2u>?*k;uQ!i7#tc&>i?{n^Uclp*}oX z-I1zq*WvLphC`5Uk!@60S51#iAmU^4m!HfWp?VGSxGlM~?rHUjWBO~9`f1(G?aw7X zL`0=4DYsW|>+VVxv#1u{0=%UjN1JPmdTwoQ7NOIrcxB9*oD?=$Q@J8oi#{EYMI!3w zf|?$PjcL5+R4UhfjcsIoq{p#dozR_n3-D{pv0vxQ+%hD1F?NL$*$ss;V+z`%5#&p( z=&M;pEy(s#-X5?u=OXSYT3Ydu7y__%f26fG}a2uCM<`sL+Q|Y zj0i-NnlcUEv;3fJV;9zHjuyXc7UFoY&1%Cf0Lu9J&2dy5**HmybkX>Im{`23C0?~A z;fcfIg_5&)dUC0yq~5Onx8C`6dvZKgB}uM0p&WTeJF8)&(~!dwl!=+L6`9-d-E#nW z3LFZMv7SFn5i3a5X5iD%`@#9xP^QV9pDsnb&qVj(jvRL?ANUrqzkVTT{D&$4W{=LM zCa}z02luU$swyqNyrJbrD>?FN1J(Bf)SHx{UA8PlegkT2CyPX!kAW~C^SXNd#6BRd zd@>$&NNgd7TAkGwIPq~}{Q$2OO@uTVsjuTg(exzXj__6CGJB!K1E&EKtO6JX)%+j@5wyBa5#+R1XnFXv5iIH=U82S|?{;SkP5B6omuVU8uYCqaGdQ;7_Pl%_DZ&-q2q7o&ax^S|1ILsM= zzr*4^nwmamEJf$!hYu?LKpNHQEDn|ga1gyE{|I^>=SEOTm(Y;jJw%o6avsU_xO+ZQ$P8IWfF#^#L05SQRmQ>}7*m@z7dLlw@-!G*z0BZ7^(p0ltJ zNdTWKY%eX*!!y^3o+LDAj?FyoEZCC79PE<0W?|vo zFoC)ljTP;HnLYo!s$aXWVxlE?FnmNs>}@vzk`g!XxjOq9K}jamZv_7=8jS5iEeye%cw4K1Ro2c8wSkYaO2 z1-;WMIEmo^r-1rxhYb^galR%mg+DA2Z_%Z%ib|E3zFRq=Ka`Z;WmNOZP#JJr5(*V{ z3XxZ;$8B5bFZU|HMl$Z3jhHna)YkhpkN{|${_cuk9jRFvCO&JUrdbjtYA1ArAClsr zF^I^1Mm^o9x)ViiquYn7y$bec8eFT1V_Z=dddd^O*RLEy60`$GpT-^1^Ic}--U60X5x}`#h#dZ1Vg8D)>W`1v;Az^a=uY!+(#?Wr3s$*S zog}4NRAcupBIulykoq#>4-;O%b$7Qycb#obSirNf$?`Jbacbv#088fIW%HLacDDdC zNo+-@q@M}9PI#4b73FZ{JK|(3!cR|t7E#{9dmN=xttQC$y_5H$K%vO2^ia=|L7g;k z%D(oD`)Lr1vSjGQ4^^&|25n6s<-H4Ya6C5*>Lv4Nhp1gqKY|1bEz+}=*H-ip{HPvr z3%Dd5^pw*&E^*2E){yVEHLX?o)ZS=eBoB31up@bmUH0}Yly+@9-m&FnM{{dp4C-6H zeF;K2c72Vjw)%{Vk1HtVN1x><8BG?) zbAKY@vOvKDm}mstQCtm~zAfjY`YJQB)KaXQS73Vx6>yZ465l;p7g2{)eXbv*mxRsLb(P^WLGlYaRLx-HFV}>2>M>%Y7yM&-kp34OVZk^!1gjKXd^>v0kBxrNeIg4}ZC0yE@2Et* zRxSYjrr6a}aX%Inw-lZhY6&&15-oAsLr>V>0^Xj^E`z$HxSwOgbVlJwb_C{|%%Qmm zB_=mF4Ezzg+uEOYlj3fwp3G1fLj*2bATjhA82Olx9P z=G%nmJ6&6%Z173;UcK{zxo-g%*+YO0s-%N0;jqKvqbS-e8?wTtNKYqcdHLFcRHJ?| zGE~zoU_1OUo4Z>@P-wvdV>s~HL@|1-pOmZ={kmu;51-AJOKw+?r_gbGI_DOU+e8qO zmjh-@)6@og3Rn-Ra~iSNP<>lig2|xxf&$>~(UTa&K$9`hj%0W|`DvFL$d$i_4zV`s zMo3;Rzt3nCmvlr&67-CS1XlK*)bA)nF(nX2g!_~7U?-L24;WW{y=uOTuB4GqPlrF{Bckt=2m#Uw@VHi=iS&LS zEODgvsA3guqAvZlP1c$maPG-=3y@%|stOtDxCO}Exlx~l&j)~RYPsD`!_bsGGXOx)7;ZE#hx!;6lThSblVCt!O z+DD(meU0@#lT+Sk2a{&E&CU@2S(hD9sZH%%ZL#!u9{FxLyx?ZrJPnw>xT>Jlh*Xy) z+orQ9?BwYx-9_itV=#x5!R_B#*^2}A^UlZE-zU2z#ZPQouvrapd@@u21G+agDgCf! z;>juf_Tp6GC)~3~QCkBD(&i7cs!q+O;zz^1-7UrE}B#~1s+}TBVfKd z80QYHE@Kh8rW}?${)a(Q_EQ~F71wOqX0Xqup~tjQE%H8Emc-!`7%1y6n-ul2mx9ne z`{)N;o5H?iw}8TP9Er|><1PE0IvLLz`>rgdIBm5tZ?(wQ9G=heGoN5lY78Ws7LNZh#MG*ORsOB>mf#|AjnRAs=jYEP30RP+eJ=mYa=GWzv^Q?KH*O_aRg|$fnoDB*x*foM0Dc zF9LUFltL%!2b^?>ZW1ZGpBcHI16py)4z5_XMGA(dI~d(Zy+rjQIlk#D zJAd*5f4@-c-(oH{5X#*kc^S$tCwE8~k z$wB8L7c!CkVkym*2rG_NmYQN&+l71AXey`5bg0uDPSW33EY3aqWcn_{RyCXlu5qDb zC1Q_r=lWYph4KxFTmIw9ueE*4q$P!B2{mAS|SfIVFPSnz_$@%4Ur3C-a}Jl8r4UTduk+$|WjLr^STtUFh_4;Yx%(51py& zXu`L5yDVh+$*um#Q_D}rbY@Y5Jh+rK!mJ~ zmuwE4<0?}0r+7CkVXxfzj20$X{e*4-hufMruQjc#2(v{!6K&TXpF>nZ4%^OGelu0mP|kbf=fV%{|<9@0{vLh5YPkcpfy`zRr(b zI}D$%j8lO69mQE~i4U0TDi5DUF`%xR>DNF&64G8gtoLo_@h6%;U6h*F6KxWT(qDT* zvfO~l{;U&19dLw6gwYHK)LBq)mFrP(j4xdG7Vty$C>M?;w8GC-G|{f{Emg;>C1qdb zL0Pd6p`2&EJ`#nxQEKFLSFQAv2~@=q`}?zNnM>J1!7Gn5D9Uf z0vQUqrc~#qILEG!Fj~WTuXo9;*xU{dNQ#X_G+IlGglTD!=Hr#!yijxPn%+PyGR$$S zfbf`9Wc3yYtco@*ViPDxQwjvym(|UlQ<*Z{_YzpD+GTEB@$%;^KW0!RjU9x@ zOU)hZ=O)ZW6{6#F&j=zLRrEfyx(=o0aTwYUBru+?WUoW*C&dmElwb-G(xG4LKy**0 zNNq$T^Rl1o#PSIg7FrknDs}@KAwJ*Fmb~cS;%~B#>S`&;OSy~9j3$MZRo3u3tl_*o zSLxf~{GmOwq9OTED_T0ffeSt(SV(W*p1XIfv}vDWrqs z$aO&Cs1H;-tL|{;mbxYto!syB24HN+{NX@9d-;f*Ztn9N0;y03gxsjF+o~M-F}aZMA1QeLbdzUO>m=>E!%#R%kH9* z&-)~QA%8Hb)evyZyl$uBFA6eF3-#p9OxPxf9DBocF8gdrvd^mYGyXdqw?1(-tKW(z zdvqtDD;tByYLDuy$|jwWkSmmwBe{Nhd*h>PZcTy z1wcuuUxsdI_k=Fwv77m>Gs4@uP_97_@6A>%QGqlahlr4tvVMX!fLCA>_p zy;v^3rL>KA^AS(IYxsnTaNgUR_rC@FTAl>}ynjvp?F0YU>FJ3n_n3$r-N45enAf8! z(!(VAW3_pvLDty_AOG6HM`Tp zOPmf<3-m=eT6JY(Pocx*J7Mk9M32Te;O9xl+(vJLt}7sL5y-Vi$));wIGYPt*6j{^EK4^bV$s`ezrqure7*Ai!mi=78 zh2wruy0+k~jfY^Bd*nLrGB3mA@n!8%99s@B%}p|GQ^k`@>(4!Xn7+2aoK@8UEdrMM z<>IH8iZ&?XgN>q}wNGz}&zQgYBB2yzzvltIswr!0brH7aoZfy2M2gYIM z-Q4R|;u5>M9$sN@h{{fw*>Z+l-m?0MCR?v(+;Ns^dO7wQ3ni*)v^jvwBxd}TsCXbx zk-d_7E9ye!D&9}oB;##&Iz2r0P|Ifsk+nkgWOlXYPkyvsOr_RcVqbiP zi!2?58=f9a%3@*1Z5=p4x zy5y9FAAscfV+F%XZ=@S6ELUu|{kkB`^@EP0f{#4}pswqUb2f(JX2bKUeMrTI(%Ay5 zDRo+(UR&4|&MTU|ZpmTI2r6qnol*1^2(r5INH?#*v-fqX^L)_LN z=D5_T(grB7(*W3%h(ma7^($_P@QswJaj}m$A*l^4M-2li*kmJ!{+W4v=&!j7SIenP z%YzJOEib*WLgm{u^}4jzXKSl>NdCYRHVk9inFf{7JF37m2}{_g>!&VQtJEN3V+3I^ z)qM4H(V2GzfswO(IOSDV=O0KnK)5SjL$;FyE;jMzG0X6b+t?Un4 zKc=U@g7kK!l* zBRRw6b@iq7b@SLx)&%)7dmNj!Ejq^vl2z8xt5utuU94J2yqAq+X4(`v(m^J=2C|v% zh-kytm{c3Xv0p)8>)mc1s90i+EJ?L6##XY()|68^-*QCfY$vofSQ>{dPENxtBt%JG zWNAdHW=XhkqI%Y@_S%v!VROq=F*j<{kcR#cq6ivj?RHE1FzS_U{>@H1M0U-QVGpLP zs%|&XG{y6n9<70PzmMFvzdt#A9^uRwVkzAf`RaXz{dZR0vHU(%wI?D^i7cV79~~F> zzHimW-y9y5O{{2R8qHe@O5`;D(6;P&3#c5DfB8xCNrdNc{ty#y5l>Dn8l=u#6ai}( zrId1JhJP7iEfMgieIrCWou+)*V=!V<^Lv#!fJ5RVeqq5P8`(F(_O5yjwz0T1=r!mK z%Y}*_x9`^^7#$o-q~K>HA~E@d)TGE>JGN#v{y7L$wa54TGmGcQ)i#)QTm<-nNnPZ3 z+`(_&bC9MJ3y5@2Tdf|G&@ObL)_zuNi*L(D*8LCb`S(k!n^9f!AM{ttrrj-IZgqF9 zHo~`jUstzG+;{3QhXKwa^_m4iZx_3X{RhE1G;Kr-!(IXj)h_B9s*JyUw;NM)qNyVf z`-}isgQI|d?>}2qB@zYL3$W9^36`R5%6_9%3lp_<6lzh1S)87aoPI7=PU*;TA7SOF8@E!DG$2RA&;S=UBv+ ze9`5b{>VMZSZpzQn7fYfUrghGf0)KeRV9y9$(h%9&aA$se9+*$OnF{91ihJcc52ha zJv%VRxdqt9;h_yeiR?bnsT#cj7qCYEY4tSKfPcfGT}xQ_w;uT>p&rM7x^{XlwSthYDTB z2^)jmWoVQPSeQ+>38^!4Hb|=k`jFaZqN$>E!SfP>P^Qb=GD{c5-kQMib{=P%&VhcOm0TY0=f=a2$lE>2o{XZ$nmUz3@jh>OWVW(Bi6A)AkJ6J%%S z@emJoLj&lfd@50Cn!qf?VT#LU_s%U@{zdO0w*VZl{SN((&Ir)qcBO>9etN>`~&P^a33LHPe5>9AWU#s|{&O93)gium8c9VhiPjJryXMzxza#r

qVsND4ALb<)Gug%Ade`XySmg7 zIj0br1Pyu}YQFEO2q=v7(^HBVA>~)~N=^TKzc2bSouka2ZIcy=oSRY#Q0eM_ZTQYC z4x_u5hmz%ZdXVoRa7>yUzVM~H#wfKW0%Gy}!M9O80bPq~9d^XjH~K!)E-o$}qV#qy z-_%@vnddDy?o6OxN*BhQsSf6x{@{l%=7QRAo2~`sXX?1*=wLr1jK7`4tkW&(c7G_u zB&%`>8vEaeNaRgH6nu+hx4istdIG-iysYJM6i&#>ugJ*tODdlV?aGR|RD=h-esvV) zL3fZ;CEP22CA`>%6PM@QUp{5mdtEy_Ba=?L5=bL`DTGn8yb|+}p6)a5Z4wsee)IXL zc-cK#%uKMK!y14i3c>pKndOjfQ}OHEo=lsc6bH8JVL#{whsy_7i$vStO=kYvMCKV6)ADsF7pH(nHyHbeRYA4@i4e<2&Wra)b`#2EI^+@=4;$a7ffz( z)T7b2fTZcl>HbjJS_}ojr}lJ$`BBLnqo$LYmU!^gK#DIfeUu|k<+IB5e6Lq4SOayj zB_We8(t72&>5e;;hi`fMNMEjebd;)|5E{P)^cTsVk4i^HiUi*R&QFxDXDWs)ExSv3 zxYa_@4NEZ(T^@*)>^pnah_hYV*HvFjv8;UcWx7sQ1H;W2#M5`}d2HsBQxhbAo=>`o zD>E?IC7Aeg{I=IWRx>zwr(%}6lLE(SV`HEspAlw@-Rk?_>Ru>T z@YY6ZOyvrmiclPwogfv$UGTpGgBB zXTNsiG%G?6`Pz$>eI-deksg^;d7KkEd6&uxa-}DwmYY=3)t+!STmf5=nBvW?HY;wa-;H>T&x1{4?b;$PCwilB07GMvyRz8l+s|~ski6~yQt=vtvvKfAC`Fu10JSPGqk-@zTl^67(@uGX#91poopT{f;FSyPOl8R1DKMa>>W>vv zq?+zg02TSqszt`{NVloP*D7lp*GSjRh@dD;t%1}}If7b&KXp{C3=J+QQs&#lVKd=P^lvSe?We2O*1C=x`dQHY)L=f`Rj5;N;wUM5mUkpEVb!6O z6j6eD`iQ%={lX=nV@zIS?3A)6@@bG~?T@G9;54HPvX^euH(boTI_`Kj!ko4ypg;#A zyo!%=HtoLow(r!2$KEOLXQRGMXdCwG&V2O8=b>vLv2#uhQSPy}R#M)gNwJ-A;6k*= zE(+UVK@14h@_XMd>bEaV<;P)E2CZ04(B*e|5q~WALWwC&Q&k|C4ClEQWoR^`gIkR< z8VYM9M)fh@8$34B&Y^M-k7vcTd!5bscn9i3Fy>`Y-U%k4?ilX~w*~93cIFd76gfFG z$c?L^$=a#829j4qOt}t6M*Cz){xMLRy+YH*uqbz|C@F`1hx~K5Bh~2Km?g^a-Zo~& z&PmCNHB^h_?=K@emG~hL1@(lk((jfqnZSW;mSbm9iXFu>+}_J7vh&MW_Xbl@QT`6H zsTtb&v3Ofx1G8zp`@C9QGM$c4<|LGdNR*4+T0z8KvW%AdC7kvEL#J|j5GLi*{*CuA zl${WyCtaLr6Nll_ime)E4f$y?4zI6cj7y5Fj~@tMb99Z`U~sc<5w^ridV4Phql7{S z@px;^l3NOM&S|WB0r?7EPHghfFK*3d6FCZ`^I8s&=Ux(k>Gd%7lDSj1c}^^ImP=O; zx1wk?al%|HRmIl&*PnWa74&|#)QvYBL14v z2$GYGuS<^IvGr;)x2I)9+=Zlh5T71YhLaQO(32FNsM*5p!BeLd`sgRo=h`V5c+&#| z=a1gQSVX#g9Gbqp0N{?pyHr-*XRZ%{b{puysyB6Hng@cExj}zoG*VtWva{4z^We!A zSjOD6-)G+5WjXhdeX}gR+A`-sd7=F0FJ%N3Sn)YM@w!lze%n2}qgdo|sHgR6e9O0z zn|PyFmBT%Avn%*uPTxFxW&D(f=_pt9M@EY;*_Qv4eGwRSJ2VtTB`>O-LJjT^5MWXc{LRqz z6XSOw*^xSXe!euD{LY0x?ei_5gx+we#Z#2S^sAc_L?94otgRGJIn)`TG0vPv4fwe} z@#BrKJ@)`jbeIiWQtZm6(89RWg36b{!t>pHLO1x>H{%fHb|VhX{^PXBQ=$uFBx!^( z*v$Wrv!IKygnpU-x$V>EUBLQa5c`;_eP>|=y87orC*oVV7%Rt}4pyZ~ItU&gBwmMx z5ZnS9NbYQ^i}|a3>EHd=wFZCgs;2h&V>WIy^CeHMH*Nv>p*>hV*gJ1Ogrz7ajq4ID z-ygPeVO~wFRZ>33RZs$zG5rvlKn|K)E_s+tWdDgtc|$vvShA}+D2l#Zm(BJ8V6Dhz zzW)sH{^0>Gx^FUTEd zuT@lICEIMwzrh~v$a2&yQc|)O$izN_Kp={h5hJph|HYCFG`E?^@%}19A7xUb0k@;g zDp_pF-P|8zu?OTot`DjY%Bsy{hZ>?#Rir++{_p?J-oL)k|BI{pjxp-=Xh$w!YdH_T zY)&scVE$PycRlB`R6iUM=Lc)`Gg9Wq?C5|GG-V*^=Rq49R{K}VE$HNZ%LT)mUyA_Z zxn6B$%&rXh>!xm^m8+Lbe44jMVQxu&A$!QP9X1ca^xS2mg z9)+NQ!W?za&7sphZSSxBXV}`VxZCz8f07XtM`YWUa%}n1lk%_{5jwZl0A=WX`Q&24 znjSOqvJyO5)V-KeXT=LM!W0g5Dz5n`KlTw6JWw5ts=ugTO^|QiltI9So%TTvLBBVQ}pGBubs;EW6s=rgdT{zHZsSD&rvHsrhwd;4L6wV~zh; zwBjw`cX8V%(OZDq6M3~ly5}0os;R;3k@={JG7avVk6dXLuil1~;H|_HTyVaqO(t1F zY27{S3DO737W|a7SKD?tLUdXj>@TN0&$x1Nb5z^NXNkp*#hE@ZA1Bj~*`q5~*>J8R zX?nE-e;4%9(sp$F{wTIsG~CIpQy}hCW#ASdPMq>a>@ULvrv3+gX&aE6cK~=Sc!B%B z&?Di#=1ec8tGs$cg#t!0wZ~Qheag;xS4!P9&;H$xqyGh!{}~!D|XeqsvV=SntQYvdF1)T2#v!$3n$FVR4I>(W!9x+iT$&Q+GmLT zqxT3?3R5Z7hmf5|s)d5Q+L8&wMuZ<*O3h1hXxzH-=@^4B%qX*WcME4)#E62CK;!wb zP=0*6nO**l1)QJhj!7vNd3l{iAb&3Ao2^J>dQ6T5UP@`qig*P(=Om&(F&HH}3&Q z3q}8|5%)Sak|cIW_<1`7EBxvw@@GHS6o#*6l%5i20u(88>fEh0l?SZo5*3hntdH<3 zxc9wgv)f*xKwt-Cf8@Zx5N(a1taV6TMBA>e=2iEdt3ZglGsAlF+J2;5<#M0dO4ZJO z*qlwM)cr`uv&rGhQd&?uVRa8XVE1A;1d4{Zeas0*aT&xxh7f6*agA3?+VRTc4p3w zEtA#K(&-wo9LGk{eN?p9o#4l@k?ZJSxG!-on0A#do4FrgL>NkF8~U`sBm~iHWwX&_ zCf}$sUfFi<24p!L6fzCCC-lz}{XYqxuPjc-1o}|%RC_V5(ox+_M{gtj0C@c5H1Ve8 zTQ9?nT^sH2RD+F@vwp}tC>Y)0#lDfs72b(3&>cU}<1Xr~eaMhFLJSYW_mS$6a-1C_aQfpp`-M!p`4``dOT=i*r?JylHGHjYV^!+` zFE6WWr3HyqRb~Sn`u)sI_U{r$)P4Thi+q_*&#=4e^2afEWtrpU@j!GW1n_?=%e*Vh zRP9I3A8B@%NR;V23 zF5ptnjs5s}7oz{1cW(d6nwB$$!ADd4}Z=Sy{PXD0o z%Zz{Jv5^jB*2tY$Zi{)ia2ZqXrmV95yCfp%*-o}T1ZGxe|2e#?60P_}Mt0^0|B>A- zp#0Zcv&yJ+#?x#PM~kT+a*Q?1knFYzq1oje583zDY|Y@`=A$M0?`WV0cblo_&Rj1Z zdWL%u;j5DK#M4(2N_!0ThYK?+Db_{~49A8pC+013OzCQc=^%-`r$a@`{Fg6q?<)FL zW(smkpH*AuvGuum)q7^(-2&z&Aj0AKkHLxQXT3c38ee6t)o7G+))H?4ZN#PBH?rYS z9&kJQ?yc{S!yQi3HT6(gsUYn@Y^+onZ-;(^(#FE4(6Ww&8J5p0Mx4Qg~ZutfOj{oK+_PvtQ63w*os#HTj!cvok?~2#7 zQ)16))hHq0gmL&cZAwgvPJrMf3$cm){tphR2CCs%sepxy(ZDf)z-O!O%OY`TEOSyxjKI=+Kd|2|x_-6Bz( zxDax;4dI`_`{no87w|$%S7oxuuPXl=B5TCW@deX*#9jnx9(Mc#`T|=7Mja&C3@L1r z%Jvif1H$6^pLam;fL6b^L|krQtMSL&J&hEJLG8K2zHS?vJ7dr^PzG>MB*V%R3z*Uk zTaUYj__BnI54wm2Tf<%B#{yX{B7G&7p7y7PP5=WE?ipZFyb9{zb!XBFyD|K42+#CB zyp^1wsmD*A?cCxY&qQl&FBfeq#p+>Eu5e?af!zbMlGgXn)l|4JUYD)^S;IGf0x zGvM_z#d?kEeML42&G)<6EPSQ!tpNcP)+j-ld!*X6A3KB!$2`Gfl1%$|vclp5MCBST zgdX+QGL#$1e*f0htX4g>jL`7B%RUy}S3F5VJ-buAhXv1<35Dg+q?3ZuUq1_i2kpmG_W0_|)G~y_X@!!d z3qJ(C{OLVktPe@Vn*rGdzyX*4JCc}vsiu(cn5UGVfZ0C^1f_HpdX{1;5|{EsTXhz( z8UTc3HC^E%Q7LbEb5i%&kL>vz8n}kM+1@dZ1XDA=|M|~dfV>HQ;`^}&Cc`p*kdQjZ zm3~4(hvF6`*%AFaPiX&ZNYc>tmBzuVWAJs|K852LCe$_OHLWr3>dWi;hV}?K|8MCc zHkOiJwZo8>hlGo$*O$2bp&H%#tLp!YySEODs%!g(XXuuOp_DG^1_6~EQW^wW@XwCX|`C>OP*>Gmc6xl%la-gDZl~7vd*kF}&Dh zG%a(*dD`~#BWC4vvqGEirpCM#0Eve4I&2$Csajs|7p=U+=mbB_k|1ccSb0Ttvx22o zVf{w*i7XQQ|HIh*Z>9#=1%J38(NXc;P(A#zzc`9joGwiXMvpg!=+-c1j~B)$R8Npv$rt(P$OrbtM5Hl{dQ+7@GDkg8li8+7iUxlsPU@; z&4&5dlSI$wjW=^wQJ!tSo`D)%*ZsUf!X*q?qnOUF+1)Dic+32Q^WeG;KB?HlCyM=; zk`r$Alp}0+?ct|Ajqgd?P-4Rl75Q3;72^4lZgdbgPczs$>cqx!d`Svhej4F-7Z>+1 z*DWXGxPU10@xIVv{A`TpZl;Q#$Ux+@-Du;L`SL(%<&yT9;oYeD!`R zXT>D?8FWIBVtwtodNl0tE%O9$pPB4WFHHptPT$7avf!hQ&tc%w{K{XUi1o?(4b73^ z`=!M0ShdMjn;K})g~ny8wNqiHEi7|8)dh^*BkpL#u^eYysefTSDQ)0KyM5N#(7|xb z#m6yg-#SIwb6Xo3a`@W`7-D3)%%UWrIDw=~va?z+$N2tv4&zyi0h7+Fi>LetHD!bL zB)QQAde4xV0lKUy*xDjZfx^)VIrps(lMD66920+XWrB0Z0$ZHL4eK0H`^4}4WI zg`-E2rva;ZB$p2_^34$0_49gPRKW1Nqd45t=^W}(KA8^n<@F<1$C32?)l zSlN3~-rQ>rt#)k6a#rb)--v24gp}^Ggyq0CZ)IogScJ6`IQxr-5<3IEdokK_4@n|p zti`gl7?Kt82vn&wqAA`v$?CJjXiu8H>LMilfn}At7ghDa9W@gAB)bUBGSo~@?1^cI zZq)r}QnvcM^55~M&>ozH2imhxt!WzSVMfuU7HJut7g7{894Q-(Q5Mnq_i>xGB1Eo@ zUj7t%LYq8isO;q9@Ga#F4z+s4kdsqZ0vB?+WYX+*;%QmaSWm1FTFw_=IHV|R`+;3M zdn3ca^7Sw`#3amWo>xYN1!<~(9A>F~Q1C=XgrFp-FQJ3$*Tt`~@kLTzWI? zcd=`-7tgy1^F2hXS87PbLrHtMkbxoc_8vC@yCI%%^@-Z+ppkIFKCL5=1x6jUi zyWPuzy4S2fI>qNX$+0||TC!F)^S@9X1pdGZORad;;53W0;P=9vI&#lTsx(xNC$9M> zUtOB7ZVRwz+EiMc(6#(+mg?s-r(~Zv+{h>N>Slp&7#m>5+8Vu(Cnsw^rmH0e+ z&b^L|Z=;h!26#)Z(d$gFWuI>Jb&^)HQ-t@L7JT0uX;)=Wm*SloxGv2^i(6|RFB{t! zId$E6^G3bVj`W=?14aRNz=nkfJZ=adqL6G0eM}IdM5+-vgU@8PV)l*D`gMGiFjfQ7 zqMd4fUAy3qnv=JU5*e$+Q(IF9ko^0cO#4TkX&QE~?WWmJMQ`0oa=A^X$Q2eGW_YW# z&FV58;~m+on!beH>vH#jqUrwQ-m1i$O^zS8Su{RbU}?z+(DTxZ3C`7`x;JkLS!Mva?{6mGPvmLn{Vv+pqRM`z*R~!yNu6 zqv&2!Zv5~q^UGu0xhZ9-)|0;SFa3T65ZlT>^y=w_F^wRjs?9F07nyx~=he8T!A#uA zQpV1{V)>$7h{L9u2CFm*74dgdu7`Bgib52Mskm@P>N1^WsGgsJ^7{UPaNeBjT&+<4 z@bqG;_ms~(@#bxBwBxUan=PjW{a+CM*CP8Cka!tBUMWsgGFno(uN?Dr9d6o{GFI+u z{TIX!lLewP?bNB)L_Jm*qnOK0V4cS9K+!o;p*3R3ZY{a#_wqXH3V6#LXw;gQw1|}w zT!?$DB`@P7j!n6B$4cC6dQXY`w~c0Gl++~op+ZX?mkgGNm3o6wDoOvT%^Q2H_4B=@ zWcVt2oM8^~*OD5$%P>mX8@;fgUV)|Z=2MhMkdcU{#@jI-NzE;kns0;`0|^fucp=` zi+#$y(V$A~qf8E9y~cA|dPVuwPF`4)8%#2AVJ}3Z9t5e+pR#qn%y`=)+pr;?MXl(6 zSWMI63Gsw{aKQ5U%p;?+daJi)ZyUcjSjrZ*td>)YHzjj-&QQ(CRpTJiv)O=~cn|DD z+;1dgLZ*`Ls|~2FP*ovBz-?drbAnma59RJ*GG$KO0YL%YJStKf<9z%R46#kajmOXF zAmEBFMY-~OZI^(cAuw3*SXNn*Ss4KDR&gUz{S7cV!lqb z+G#7aG5gg1fw33j3V?sCTI!^!4hSr|Oa@DxaSW`i8)l7FOp-sd$A>p%Qe>u8A-<8I zaTT}8@ZKHYCcY|d!od-f`MkK}(p%+|)dD5LK~%dcAJ|!C4FZt8fmBUyzjMi^DZ+g3 z>Rsq_?OJ8Iu~^3xP2Or5IU=~Zt{R@u$M{Wc1;DlmT@-S!^oy{~9@f#PlkM5Pz&Y+0 zWpC!7H!052CKt|2izS|zn*@3f_5ween9|-IIP84AsCyHRp`h_HM*9kdgouNjj+0__+Vlf{VEtmNTA&luq) z3M;*0h2vO8gOV)Xk5D$h%-0JJ%&Q+=1U(;n3~JzS++OdFCb`y@?FN%dm3;raq3&zq zA%o5#V+&GqsAQfbrOm#TWU|pV7I=OCO8I%7+$qb=9EzowJq2xoKybm8w_k0r#BnjQ zY;DyOUF5pTZR(u{Z<+&3nMGnZsV1SQ?Ag!q4c8%8IM{a3a-ORxAC_~o(1#00xb3qz z3EsKWL_pm{u#JKJ|EiC&1y~JnmWQ^-q$&w`a8ET2G`%nXvcg9?#+H`gW#H!Rf1dp zxQ#8+UwAl=zWb7^33FLDRbRq)pPy}otaD>wH<1zg_{eFP*K*KAXjv~-wpRCUurz+4 zO8THTW;qna5W*y55>3YH7vCweGZk_4Du;@i9Gr4!;UCiFs-z0-eMyYRRqron-*urG z#LNAa`}^@XfTJNNTmC!er&|8Gwm{wa!SmQ-?Dun_IuGGrx*v)D|9V;KvvDBXe|o8T z_N7?Fq+CwQXR=Ash`dNxtf-^1*Aw6A)O}C-jeuN>L;1kDCr`VRysOQmB5n%CUpZkg z(V>T?K=7LqVlldGNr-mWYJ=qtJXy!M7ix-OY|DQI<)q!Yus4@Aht)!qq7|5m^0)1wgUS!1*gW9>2;Mw`6V7FT+@ z+M@=?!n#h8Pi1a?<_^EYlY;zPXvjlO+IPYE>D~vIc&A-tb6HN9%&U6cFD_qE4|Ic` z(^2}6tdjri=7OP|XF1_D(S95D;H<*(Sc+Hf7a6$Y)F7c3FC~)Py1~R35^hY!Xt_wI zy=ZHIZp_E-*N&~4IhRo)o^)@NcjA^oNQy(Je$c^*e zL*}lC=JZn9*AHSXQ)`u6j|=Sw-yMY#(%4X`^}nj8pH$p3WytF4eVrbI?t(VAX zwp88J5vNpxH{U*pwdTGLG{@;mp6){(M^-=_&7I<$tq|F3|1F|~ATPVia*dPG?m-;6 zl^9(nvTKR5(r@1wt5i_8+GtA<*1@O)HNZ$ijU0~cQ#-AFF+?JNI|5&|Bj56IAwXiw z3S{wC4e)gzH5ABpb-q?t8)h;A%K!{-m-n52MEak&`XPx;C8_Jpl+`1l1?@~d+ z+twT9Nu$L<)~U(^0RXn_@YU~9>(6X-^VcZ{Il;k~!gEUJ!HoUyDMX?9>o6nvvi#Cb zX~ARme(oQZy+M?KYk(viaP5PRETDl2=N@?1UX^TtPJBTlO6El59h?fIZPKM})3taV zW;)PNV52`@oz!;DuTj>(q;9Plkw5wIn_W!k&^i9L#Y1y1%ngA^1GUi^sxS9ffAJ9h zbfiN^EVUHmb3BNs{=adqbvmH5DEz3xKB|U4q?g(&LbLdE$-Y=!^KJ|;M|S3Q)GajsT&4RFQ8lv5OtMiBY;m=3g(}KH-l0~jkc|rtBOT0r z1=qaYHhV*`x*M>quG^v(yd2apoYCdi+%j6VVK~XCy3CMugCM#?XzgoM9z}k%jKmcF zaGmi$t;ek?2YK~4pT~4!$&n^9#};2~?qs_lPLb%c0UWtiCf7B@M(iOu5U0!S6j7;rK6$IPdJ))+d@$o`i{9KF`;@j;LOj}dp_Iv~$&WeP!6c(p2Yf<+&2f#U+a+A8)S`~nlT*ejEbU?+a%tlZg?0Axdd)6-}oj5v);R=dFW}X;<^c@>cX@rrra}DE&N7GV ze2j2SS!vTNsW4wBr|I)x%DeR|qxc5x^(D5Huop((Ikw zE_#=(-xr(8PQmWBSV5ruUam_%rR(TufkM5AE94@-pK0$u{9r2stcpjT?aZyW7lDWD zdA5S;+kA`Zm*o~FmtCk9xYhD0KDAA}Ts)3duy#{Ws>9K`1@%2>Hhd)Ew?GQ(B zmGEKq$=mA?-~=%T4V+$7)%2~UoadfOgKtC~ph6&Ie?h#^D4jyY)lTQ_{90`tn2RK* z0XxfkC)?yU{S05_emrVI#;t4101n~yw;)hM{<1bK%UT)!l)Gk0pF14TzNXyfH5;8; zV0ht`-^@M^xiGQ*R+vO_kmxAFlMxJ0P+HCDbv{FKx6HNTWm}S3DS$B) zF%mdLGySj{<&Uk8k7pC04tpBhduLoSw0x8KrG@+NV>6@$Y$dE-jHFp0zhGG(@mnPN97M~BUOg#t(LCOyY6;*g1zf@KZvNsorVS?eO=CUhF4rTU^q?J)< zRf^bD2ZsEBv}M?`mzK|w@|`Dd$cY*gO4gxvWLHyot>tbC+guwgh(8-;dS1x_FPpP= z2&Mg=2S1OC*AdOSl;R=iAKh)tLQ7;imHIz(c&BV+YZ|NANMqU z%h>n*v}n&$y1%D)^da8_ZaK16nN*mIGZ{ap63no35-lm>B&=h2clw+@5>eelTHw08 zJwIfU5+V{_a<5)|n#6D#xy_CG_Ef&+1GVIw1C48!tz~|;9nn}wjwTcF7{8Vwb77H( zXr@U_L7Ys7bn539nnNlZ+L{F;uADdNd5m8nrZUbfSOUrv6_7Kkrq)#s)l=>3<`mP; zcex!z@G-v3O|Dw2yKO0T1#@F^2gzT7v`?3GFE8uXE zHmhB!V`WM|Kl*Hk=n5oX48-3-WIrq%!rBj_`$8b7xCFs+0`XYfbEvqs87b`x0^OGa zk$q0?|K+D+d_sD_(kq&k$doM8DkIIt`Yysm!n#I{p|mRa%f}F62}F3G2W3l^dPhvc~4xY;S+5pHt-oV38)EOSSBIKyD#(t-|N z5VhcL&WLv1_!is#uLI~cKa`+HF4HtL+NTsrEp0MVX?)?93%e`z;U%6*VNd1!ke2LT z=GDCs{-Nv^Aeke`p~xdrM`-GJI4+5v+R-?i*gX(4)@{Cv!V5t<#~%NVLu!>s57u~V zjP2D;Q*CJX5bUsfse6;x^Unu?=V`*H}M+~Qn16DjT4lzO`YwC5X7;D+dYW(VLmVfx;px%hlg6N&pWw*XCbt>P8p{Cf>O|5 zUWhhAG_-l8)k{{68A^E-Yk5LvW6|YNR%|mVvqnmO;Vep;24d(}NaW9N?~u+wr`npeK4O9X(;*5VV&c9MJg3B z;|3l4P+T7I6Pr=1DaI;gEo5bp5Py`dxGeaSSRn1TD;)|!Dm7w?sNBeSmEJ$Us$QcT z6{5dpGvvf8Kj62Sit8=PqM01}6(wV~r*Y8XI(jOG;DimD8-Gp=f7eSqx{lUr&^Jb(ggZN?ITL(wB8uLc`>GzA-@Yl`hQ^o`QLu}Kcw%!{t4)p z_K-+&cjUJ*Yr=CFTj4)4NAv_9zSd_=#Cr)~v`Ot+!_SUlPsL7zaRfmumJ(zSlr|&1 z`+{H)EP!Zr=rrVNWeRn{NGCV#YsUqX}d|K`Rej;s_K>dblNRc4pzNcoEqihRzE&% z5RE%;=5j7ULk{uvTAJQq-jYB(BF$QQGNwHitN@Yi2k;;-}pr) z8b`I8eWvOZYI9)o!~dbTct7i_nO9q+*CE;`{|cfnm_RQ4c|hY6O=2c$4YrtOS!tPy z{0{^rUcvD$1SB;8bna|>9r8~|KWR9arxO-AzbvrT$u|`QEK*h*4A4kzc_({IEa5DU z?H6I*W{SID{`Mybk7Wz2wE96HsF#N?1JYxGXT9a9A;U^Of>Uc_zPxRu5Xr)8Mf99f zwWknln%VeKEWkCik@Z@xUEpei&pFpX-w!n!gnwH>q?n`@qDKv)#d74WVH#Ab(uI65 zX9fbrzE;^)^(aBFSB(QdP>-c8|Ge=a+W&U&5#o3Yc@XmJzRLU7@7#KhsTwvE+pT?Q z>)CD(A?mxAle3$b#}p$ULRFq%yfBLQit=_JoROqVN6g@QGen4ry=-xND}Q?$g`U99 zlC-0Ak5@U~=C$B1rvJrzzk5PcZfP9SuS&p|c;l`6wPb! zsaahI=AVy&wkFF7s>9LmQeXk+`n_AZqdA4^@Mu`{olfzk$vaU*UW!php~(tYuFU86 zuzC*SZ|Kzy8H@u1_}TFdOX>eCFGG!9Ay^p;uNKTu6zq2kSf_DjB{)uQj*m9P5n319 zjO%8*pdO`$EkSXxSq1$)r(edjIoe zo7H(*9sIy)zV2+eJ#|fJBRHGGh|a6d*!1Ar=Y%A7dq{;CxB`LvrNvheyx{wHJjsHq zZm;rT$h*r86MvK__2*NLfs1_>o9iaLa* zuJPotkmk3NY4zjma@dTl`8%omz~ewww!;jaAG-BY0IKJ3P^y^sSmE=zI)ewv+>zZlv~@&3!ee*ZR;4D%QY}I>J2) z`GJGZTAR-iRwUdK{?QA|G6M1NgkEa~qA1?Rv5g8q*X=3*yFAAHFNK`nsL32liCzgy z-Hq<`^(@2t%A?}N!miW1ksrhgnY;q{xi8!^Hs;L*fy;M>cvJEU|i+UfqG1ZdN$oDec=m_eDPSS z%CN)X*~!<}t;cm({4*9gM6RX?lmSmGvq?_gK@q!6* zIk#Y}npMt;uP$NT!j)P(TVlVlO-a)H=orNjU(H1rJE#Y1->2M z#bEjsqZfiwoNdF*T(U#?HlCQay*ZAu42W*}ZhGa&bzZ=-5A61KH$;Jt3m;1_SDQvd zlZ~}{Xm{`NOc1`RuhJ_%Q=qH~n_9RGdC@YqeojL|_R1J~HD%1PeE9{}eFMjXV1dC$FTf05UM-|f!Nxta|BIp68%jsgL1XRWgi`Nx;>N`-~V zoYsP#HL+{aQR7gHL6_&@}p09e`iRpN~v9<>_A_x9W2RE_bL<2D?5 z-xHn62}r(S53xoj^0HR75>EvK)QxtY`EME}M$}fyU5XQq>;|YF&@>P~ArDS+dw$O$ zIKja5+W^~?15-T137TA$p9jFLL2!Hq2cP%jkOfgV;Z}UwW?Au8{ju8ld&=UCPAh6(PFhj`16MOBd z!}o9#Y5iqx7tzjXjnEoR>$>Fhnvjy4DfHK6dR~@x7%$M#-WBmLdiTTZi9Va)c=tCV zT`XusywWKdNl}5j-sWfmWuya(MeQrXxWvrh$TlhXPIula)oNMYEDTZA3PhF!2?0Xa z$coH&bPE+)+f~h8aDRN8Q<_gVt*$BSg(mCDOYi{PoSs)BfGVL4v%o3}nDyB_EAIhm~A=>w-ifkSF zAAU-o2XE!GGUqbwkg+p+dFMN1^^z5)BS5)OTpJnAgXQ*{Lf0vMICPn0f>;CW{Iocyh1oYHnj z{-cDxOu8kbD^xZ~?(PA?OQF2uaXEkqz#6SIi2L~6Z#G#Wgf-Mfc(rI<#%SODbUElY zk*`c|a3D7XedcOhuUX)NqUla_{cH2-&BESS^Nqoun~k@?CP4k6UzM)VGY(RO1TaX` zk1<-#0l>>cYZE!cUB63fJvWE>#`7qBC=~fTg0Ok1Y)LE?!iHpJjA@!!HScjNpJ9lX zD@rVl=;#Xa8346A#j3l9jFuK)-XAUl4C2d0$I@3uz!={3rx%0a@I*(CegWa>Zo3(#A@HDZe*%*ei8*ZqQ+pzlG#3!nz_ zJO3p~-i(%ZD(H!69=VOkttmJ|F5?;@bs=xe7dd;1siI?Iiw=%HGcz7d`9J|bM4z-bFTG?xe$@JvMIgEhj3(_LMe-DH}OqeQ@)24J5;BnTj zW#5GqH=H;QZIb$@QFxZN5){n%r0jDm4&Tk(soZkaT+!dY+vYmYa{%IGkQz}2ja`cb z(mDd3IK{=eZU4ftH#k8gMhb!k@A_LFfE_}B7dp7@yalU6mJBu0p>Wu$t6NHAqUyGg ziH~;`qmHtB&vcoxu-{%mbjfF%>a^pXnIX^JA|sN7al_gJ!oVpwBP-Vp0&6-7eSHI& z69ypz>tHi>l8yX@bemtCfoej4f)t(5N}=(yptNqyL8(l|$K?Dm1re9(cvAmGH??Ja`!_Fq=xNmDW=uVdFCAbdsSI4=fEHt+|0k1>%kt$A>))YN>di$A=s_u^ke@bVk53wP?%R#CFcguXxz z!7+gSDJY3R)R=d>9H*UoYI{I%50BJDcrxS+An!5s`vn<0_H!+%dRl&_R`TrQn&9a@ zEKTnmp^edoehz}%n{9c?)DX-dDDlg+&+OlswmuRqRk0ZYb0jj@6qn(Tm)AVT(KMPdZ3%f$ne*M+5|FT3 z`o6@0uPoM2QEwCIf?F5t#W)(np2HoR$ivB(`KVSCoOHvn@RBD=H*U(-?DbPA%0)A{ z@s+dmo@SEiLhct+l&Q)oTw}h~K}>~L&S8a%buZ4XT#v#0-F?1Hx>5vGaRFXNDMDfc zTO;&Puq1YjH#T{={0R(2g;KP|Mt(OGPzDoRW`ztuAitSl2m##42lRb3)ooqGvfCAxNrYfxjdmJt%O@^W)~fcY-@=L zv%H%bAFi)3Z1Z4HsGEV)Et4pr=D)Wa(5C6-+6cSNdN)+ytB}-d^}|747UK3eiM-Do z9(|%c$3JmbK!|$|{h&HuDD{}bL(5ck7ulFJH!WnZ=V9i#1UvzfSG8n4X}Rv3OrSil zWA|rPoc%kQ7E1g{cUn&SjXngGAYcdxnrmdPIZ43W+qivLhn(feR!Ksf2cuQ`h>1ZC zGRnDlLFO4^(0ADG-&92*6VYZ)^b56Bilgye|5{%%aXbbbK{6=qH!XWn}ds^@t#Jn5?yU8M7w_A!H>Andfg~sv;zY91mVRWHCHZlonAh%THiKcZn0>bGy~> z-6ipmO-!H1J`4jXYCWoB%9~WqxlgsgrOF9ZRX*bUyWuxB(Lcj zyEDVV%nK;N&!&<0S~HG(f7-}msrQ@4$~madmEQ#KYYQGTlgzAn}s-$z%K4U24P@z@|Y-{Qmvhk%+Sv z>)a+Xp+{|5O{bUcBs$51BbvzAVG^b3OmoKq=avpBN<1#jMR$#4O#bvn1~VilC`p3k zLR~U}CVc8I2*!cEg1j0BtGdd%nn5`F8UzBe%b!q_9l$NM9dr%fDq4)whext>q>sj& z@x*n+3f|SpXrK5QsS1&djeqfrLnvdi#zGIW0xyqLNyW2mco=26=}E;3xnK_@>iIs- zuC64*cSAPG^nu|o+ ze$6FzVMa+E9#CXR`u$~agfUupjast4Mf6-3lncacuWEx@OiMl`tTZF%7P^{vWz%hY zzR@%%?j43kV~e{(3Kh}ufNHzauY3vt#=HmNPrtG&%nx77cAfAk^{%S%)#>+M=Pr0s zGa-^J9n%PS+y-Cy15x8|S{JU!r{e(j`te-@@PtJ)#}ilY;nbgypxiEs2#BIWSwZz? z`hUiPFiZGjo@$U>>xUCDrd&Hl-Y~Qkv{_b5WHtC=7z7GNA?3~N0J`v7k!re5&B#E) zp?o!_70wj?OWPbctC<26Sg1FV+%8I@_NZCpd!@4!6gfkKo&>Wb%#BH#^_awz>0=DU zp^N!llz>K-`(m288-=LnIbjI5BvA%L^egNF^{hPtuL>yT1qxAA80I70b*9#OMs;{x zksHyw3=1^n-tcW$)WQ!ybW+eXD8@JB7i~ggSHNcgJ zP;j1~Q}T5Gh#qPmAn@u;_FznZVtXd1pt!gFD@1x;8Wj6H#rG3`%54c8F}%(K@1F(l zQ|BLcRr^<^*4V?dPh>NxRJ{a<$MAZZkd*E`g6JUzV(dZujtKow*;aKY7LP9RQB(N3DQxJF3GuV>=Cu$?$ZM z8y#z3d3GkV!-7(-e-x%3fTrpt_vO5}8eF0$euX0mBr!?xHMOLy94~zqn8iPRI$nLX zZRI08zGvzFF^wI?fyzC02}OO@XYZgm=d4Mj|2#?{Vqo-YT*F{4%{%XPt6Iu^%_X8d z0+OmCD=z;>86``}K_QHvC>6fNXvKxU*x)8f`?oq^6rFRaVX?4xw!vviX-;fG-E`^7 z6}o8rnn`9pmKeA4JA$N1Ki7soK`F=xnRvvu7LM4Ef|QaPA)LV=;Q=7KduM3N|7&V& z3DONUrSQ@1{8Upz$}=T7uBO>w)0MM?fMr=aDveG(3F?t{h9p4WZI_A>JjVT?#}pP{ zlYAzZ-bRqaAGd>;|C2jyw;!cs0l4gfo$E_ICoTEpLEG%1(ZhJ5^M zYn3hTC_m3()I6T=jgfZM@SZ2VjzS>6&U&TL(!wNx0?u!Gcrc6?M0G|*aIG8vVkqVZ z8QxKN?cZ8&lCME7ADkz!W~|*_c}J)GGcUU}0UkKX0ogg_{nIlFf|F(slRNr2wSx#hZA()Mswg5vM zvi$kU_z${>C)DSK#DBznXg`C{+n^ z+frd{KMAYkg0#Ek%~CEZW{_4EL&afja19C8v~2k<-B021gdGElD_Hnl6-LC*8G`j+ zLRfxr=S4*(VCH!=(GIOLL;>1)zz^Z5L$gwA*gI8aI*SH~!E=rl%G#C2bN>%;!~exE zrR@KWKug1y)nJa)%`Sq!G?_Fl=|Z_KHvgjXyLH{Ed{ux2BvjWGTwU0g7tJy9_$kFn`&jO3n~Q!ifL z?|{%kHrx3D>i9cttUs#SNyL7N_M2~({SxiR%!}EOc!H=Rpl#M^z&q=^v3B~T?)$W` z>3+!)L*PaQC#p`P4IvPz`kRXloj4lw+a~pR{3{fjXXFJ(6JSN~D!&A_cAon-VxC?p zuQMtI0K#fsa}5k(ce$7r+OO8I6zf>NrkPi}fhiElpA<+AG@1t!425gSSGU+acum(1 zO`2+^Zt~^~JS~&DFHX$5$Q~~vSZA|m^TSWK_E(j~6U+Ufwk8LxCgq`~g2=Jwi^e18*HBcT zr~i&Qwq>sl0LfLE3r>%nI21oAKyV>Li$dj_m<>?}NY+478zu{-TSxOGW?8G^#oFbQ zfAANCkPSnYky}qi%=vn9YI5p0ykblH+LAFC34Zsheyhr}#)3bfzl^yPh@Db)hbJsT zS@kj5a0nYDjc^)tTwOZed>F=bPXw6h}2sq zL*K(`xo=zPg;&S|v@iOcMQip70c~}Bl~Lz?$f@F3?@GLIKLP;fgoxj;Y|6T+0Z0k1 zAD9(f21ASN7}U~BAR>7 zbVSaSp}t1~(J5ib~e*2f2nQ}PK!p6*iLH53+lni1M(q!HbXmf5@pz{jP$)-E$%fSAD%p+|y zYHXlYk#aXbC65;_6f8|;A|h2YQ9IP%tzwcBmzu~5BgbGtJvs-iZDVwm<=9hTefx*si8Rot^?9(AH`P3mKs=1vjwqi$rmjRFa}t#3!C+yawda`pbvXu7`5X z$T=mB^R)cVgb4N3rn@54D< zii!&L0b}g~jQXed(Z~>tU6eI}Bev++K}V`#o9XU|)CwC?6JL?C2>L=DH?9E;fBMc7PAYc9)Ox%3So+cWTL1IH)TS3$0 zUYqsdKOn=%nip=rO+P1rNNvQH`tWvoJR6~qEB~1H`^DpU!ZL03@H&W=!+$U1K2M#v z&xQ6DZzok$!>eM7!lj>xRDI^b_MbsnOgargk&Ron9!gIhuqi{z0Kr*qv03}66_T$X zek?YA5H8of74JNmkee6sIJ(=21QQ+4{&QWijTlBH2UGUf zB1$)(Us>p0W@GU4R!vRO!6N63z({3I^G4*Erl(!Lm@3A=b3uPXGw- zY9E0o^K{X1YC!!UWRh6iH~ljfM#ThDfVp68Ku6=@SvazywmTjteeV|5eWYV9xiq17 zB^4yIQ<7euO)LgC;g{;;;$z}rWINCm2n1w}((T}xssSbu;c4#b8KEq?pKqs(QCi%K zajnynHhm}M3G#BFK7P0EY}dx_?TV&~H=nqMx9eCs2(|-cH zuUxH!#5U~2t4RUudUEwx2{jGc`vSy-9IND$I+V7{J6Cw33GCwDj(oc8pu#qxRn9xr5q+J?Kvusq;3U)dTK08 z#DQS#eYnHJ^qB6K{drOGUCZy)-fwdX_bT!8Qaa=kSgUKs(8CjRo{)oJG1Pwlw~GXp zDDNL-(d3!uMr|AqmZyuHc;iwtu#-BPKXE*Bx|X*$_4vEeix%TA=by`VUP)~q8#KFz zs=kmN{gs&E+N5xHx}CFJ6~G06%fu$TiIEJL4>A4)0dXtxtH1qwe$~7OHVq5RdDB!1 zb9W0HTCmV)1Z*mCaS3atEEt>Pa+#!Kjxa1{g-1H{Em_aB`pDS71d967iIV z)>yQdnzS$nIPfqn7;OBVGsamAYTNLe6R~PIKLaQdPj9z|4AjkOc$5}Z?91N8Wf{7|Yx(`yJK3^)z)LJS zV)b_q10FEvCnJ$dF{YgU$*o%+;Z*!T%@Qb?e`23719T2Q=MP}7X-Dvz40UNZCwDKu zw+Gt9m?nm4Cs3{3q|Uy5dW9Y1Tr|gW*xh3Z>#ew~#*x~4f4tXnw6(3YY8v3cndm@= zcMy;R8a9Yz-!-5Hp0i}llV?VxC*33{N{5LjPDF$gMCBsalFKS>mVKmzF464VH7S@p zS~l`A)$cFJH>MxiDQeO(ggXSfA?B0F@6bN^<8QVvma?^Ja5ViKmg%#H?`@y?S-c0C zIPa;tp~LsU*abmVsDU$>#D~7f-63I{;dFlMPQ0i>e`o&7%-1>>*S?sE(xlJ6@X%!t z?L^MY6p06aqX9>iul9GH;Fs!Pi;Z!uHi(Wc!{uQ5T93mhMO^lNa>g*v;ZH&}!d(h2 z5wRc7{6-=`I|Agi%(Jk9Ag!!Lt9Iq--u@`vp~e;F+k%&W`q^YT|;qM!y|wg=bGh|83hHSP0f(vBUS~a5XOCcJiniNDgqaVb$p^o z6qVm#?E`ovU;DS$yacF~*{HL%#FzvqFnaqj8()h*r5gD6{n(pHEdciF-@4trYA{xj zQF+Lm9bgAr;V~rGZ!$KFjo$t)|DyFUwB>@1CLM+B+&=IBv_!XVl3vx6@c61DjiG?w z6QB!=wU2M&7gMO!bhy3DyK+H=y~OqJh1bhqS?wpWH`iE6tXx-1s}g58>e|P~k>@o- zmghJ1lkQ4O}V(WjTLqYI-mTUyBg|t5`qWejqp`TU z!tkTRpCxTPon#nwlp(kX4Cek%EjZw&0IsbnH*pasVWdbu2;}1AKEDRku_qmqmL5c* zUp1wFV()UbIl{le_pk?R_>ez{vhh6Uj|1(lwrv=RnzBU&JPwMo>CB>|9RmzeqAcUG z>FDV8lNvJ+D)}X3#Kc%-!OQH)-wg>n3h|n}M|u)*yM_+GT7TGxk0glDIR%XjCi=3mAUWahE*nBPm!I}6C#5UN zDdnndKfSf{xprS#T6D^$D*Cf}oQbsb-B@Z25byBgn+4mso~XJ^MD~^NdsP$14~lAQ zXBKmDYXJLDG|AK{IVT^RK@CarBP$dJWzJO}NOmy_gL z6rX?H!>Dza<%Ew(EYJ!`C?-ea*hP%23NunWym7z4G(;M+cx-@=*PO6Ue1eewLXWtv4b6^Nxy?aiS$IlCXPvE%8?vQjS zAFLC3f1^SI;M#OY)bKV2j6p7(z3l*tfirViT%7-cE2}ee^#MgfEt~GThf($c&Blj` z5qc(X?Rx=Pjl1mr@2}!RR{w&4IJkvC1gmg%Ui;O8%zixnDP|Y5sux{i(bPj`27#j1 z=cgmT$L%96)|Qzc-5pfAS1N~$1PO6*uT8X`3BeU(l%h6>Oa5q<@b9e{&}9oYwX=&c z%N1F_*U5AOjEv%2i`QbR`#0cZthj4xGrxs(bUTny_M+3Y=b<~#SJGjH@otl(G>cl3#+5; zlVP}IzfZtTkW2@14wY#j(`8ek5o9e$WEH+bk;cSlZGiP)l}en-$+_zlBQD)mq-rxajL zer{>;SSDI|ezhVPo>sc(gT_(elE|Cl#Bhk$A#M6K@(#$FKn9R3iVXH{V`g_JwH{OJ z)SW?l75*NREEN}+HG6V2*%<3RFrJ(t5RGv1TxQ)Hx`zY7_U25{TP`hW3TcXSL!1*D z@s;W|%Myy1J)m;??YUbtx&n*AjlAO6a5f_owN4DUP|46>tibLuL;RPxL$qX^lYu9WXSgbD}UUdSnn=M>i4fK@m_+YgEY5#mW+J-&=sp7)WI zszPD-fZQ z-xkAivOyvU%Pzp?hS-8bp`RT`nAJuM#UZ+#9N<;mR{I#uw(0^SV=0;gqv1USdUyX; z>rq0mW9}W*tD1P9SRBo^o_p+m;@Z|cGvgq|hoEW=8Dq0kv=XTC=QC_XJZ51^7Q*~C zxJ*#AY|0W97oY*sW`NTj)n5>hq4H=Rk0BbCE?i7@tRcSh9{M}0(dA%zLZPXxMjamX z@!>B+pX_(QAfWmU$~NV#xmn($V3I*fst6jH$$cS`50F@5&qypU<}V8KDH|!Z;h$je z-u3Yoz?KL%-B2*cLk4(0KyXxC5U*YXmi{Op^4$3;;D5L+Xa)VNy#mSKyr7UCpEmO_r-pvAc5D3x4Y@Y0qi@_K3V{HY z*I=id)1-iEN0B>z>|4}+t&rt;y3|-k^k+xjw#9}44{tWi}F-Y*Il2-ei3f=f&alomhI)6P=6_ufSI!qj*8?dnN6e-`uq>Gi^W z{gG3ebZM+uLZ9h&NgTWVQSIZqRyiYX<8qbVKwM~~pF0jb{`3dGaJ+O77(|vj`${P7 z{2e#+p!hd|dEDg$w^nhMC~Cdx%CgyMhUq?%UINAJtnVPzS_(bzzWz|%20Z6eW6zd* zNA*ASIhCdp~2*HlHi|w4@!3b6Fa2z8}SFB)yo!T+hvH8AP#B=$b`9+ zViCD(t(e%Rre3Gif+yp;RfWC3$_>E+9ke)vLB`1S|A)7?4vS)a+_-lM1*E%?jwP24 zQQD=uLs~jT5mZECNu?WUKyMK7C2(mNt z%ri63eczv3;9ADkEu1o$8139QHj){gqI$J#re6df2PUz6qDZ@YgB)Y%fp9#<9EL{o z0KI!!W0VL%tC4#YP~vpj=Vk}oc9Lrc#~a&d7Kh2_@f>6Iv$cwX<>!v$z(q^(Ms-Fq z1_ts!4+FvFKG42G{Q_d`1@0$|^FkKFBKJGUzP;vPiuemy0{TjLuojFTWScfQOIw_G zTxL3GN9S|&fJ9%g6!*|FUxjt4{b3eGrJ)&QjD-7~Q4XQzpi25IK}8NRjmu5??v8C3)VQ3%2d`nxT!vo|>8Ux~wlRdT{H1PQ`N ze=E8Hh-8PZnfDSsUR-%<63>TEsBy4+wRjG46+rErdy2tY6o_TS%dyT?7Mj}ezS;`?WBD2DX00>&)$76ljo|0Okw*Kq zI&;I0m>5?#=FYE|#C5FctxDeo7HO+)HZ{B$c~Bw$Y+U6q=u2eCL-XpJuK$h&^H2cr zOwkYBSkJ1iv}a)9P&#psg81~~)?!adUa_RPDV0sksC$WSXz4fubcFol>yP03I6SPI z`_Z>(yux$$^i#%VREzXAr2(hc`DQjXWv9;tUY~Q*nsq~z-}Esrzk*)-3XA=WPJu6i zB7NjF2+eVF36G=+3F9|sk0@j*e83Y}CZ9v9n=opAk|(cn#71{RAcUNU>unvQh_Z8= zZ3vl`NNg`WDxtN`s_|g^^l*oe7c^@88rcU;I@?xZhuj+q?irg<$F0`P^FWX^i*r)@pzpyYr-+l+Jsd$IqM=`edi_CDGMLu8w7pMB;rK>?U6C%e9Ot3 z^rbxk_j7#a;geAb{&Fs;9k-Ukn;(V*u%njb2!rCcj`g;k!4PPfL&;%6Wa6!)Y;x0Ko3>PY)BZ3S!8>!+1Gs?8*}3NL`V+vwg0DeEZM*Hy{lE!C8rtO>KO4Qjm{2K+Q$(v|6( zVqN$;x>mvw30qM&71s>`Aj4GC)vvC5cqt$W0f4Lg7s|=ymn=+NXO0<&=XLAxXdh?Y zEim%BaVRzG9nIi#GO(lK$Wh=E{NoE(xyCp9Y7NhdM7gtjY$8wcXk8(*&j4(?It^g( zKRbb^1(p-Qo(9CxL=u|mRJdE7&g5qp}mFQx3CIww^aEb)5j8MY6h`etZW+^S0Eu|M`a$cNq+nnXL(x zZojF{HB=~+n0}MOxk8(P(bUZ@VAd>t7i{MS?I$E?C(gDIgw)gFZa%3B@-cO->1gR8 zN-;0^f}^Q%2GknBqSjZCV9ExFBci+*IC7OnxDz)WuRrC@3gCwc#D%?ba=K*R7e?p- zO5~byQp-J#=F1}C?)(*M54@#q2HNH{Qd58cID(Bowj8t2AVQAvG?l?R!v(S-&!vI5 zI?B@o#MR-bxOxkO0BaL{j7>Os3tz0k%ItM%37#6w-tyCWm)O1U-ys12zal-MJ~z3{ z;|w%QjEZmJ64EzYsIRzL7+PmYWED&ahoL>dVdCrKbd=&{5?^TBpUuy_Ir5UY$XZ_0 z;Hf5p;Rggd#Dk5G2KqoC#Q1Iq#7o)Z;hXeEv6rW~pIXQO&yFE1<4&Qa5;|DBl0-SY ztJ3|7vD*YW(G{6arDJE)S3O&Xw&GuJT@CYEgkUavzosgnyPn8-1RFGqeDCT3WvktH z_n3WWrk7Tdh*)tvz$FNYTA2O-9zV+B1Gez7Fx5L~RecE@n%vi+lyA}2CnS?*wL)pR zv5`l~${PM1(xds`v%Fn~s#-e5TRIh?fjuAv8fqK~J_cXb=mFmYkHn}Ghd6ZC;P9~& zpa;^+oL*jJV4h7CzUMoI-@$=nm%B^xSVPF~PF*9rD1O?9ym=Uw0#f1LLa0439^Uo0%1W zXX+}M0DpC*cC(}X(cIv)H!UVB`_RkV z4SpxzAtxMfYhO*Z1b@S}%?ET#OE*rsn@M!)W1?8Ez`J)BTkN~3o=pf1&zuR`QAZim zU|UUCG*gV8#H)mlbr=oz%zAg98Ey82|e?F zz|_E4vr}SHtGeIf4?7-hx^v3{CC$Gs2c&u1C&!l^Mu~b#8!H{fx1~|iQTDx=f`EoH z_%j2K>BJ}Xfb*TE02HmQYyLnph;;^+)`h))yPO`TEW9xL$sgg+n@=6hgjJbOadL^P`faZUW`n*Te3@g;LU6Gl%ghwWK~=zCCe0+*9x(@ zPJjGwQq{TCLt5v<1B7~{mR2T$BRE;f+$>8uOlQ5_ZRaSG>`pZ+qQ$x8tPMXlLa`q8 zg^ZrA9=IWZ=&o&UF;=OwVpc`9=-N_OiZ-jug^GC zz%W(a{djSoV&4g0&C;YUdmR`Vzz=|b_2tu?^XDft)O+OQTG!#7&ojXl!apEw=yO}V zTUAxnir^kD!Cyi0VdL9{?CaXF%7Vz?UOR8w&1=uyB`SEmDdDr~z=|PgN)W+9(~5sb zI}{#(|1Ju_nLBMGlQm*fMdT(HpYXVp{*eJ05Y7hj>+z`ky2X+)#@`%my@Eoqg`sPf7^5Uq zj}bpyNpT>4p)?~Wm2Cs3yDIsZ2F-#3w zjE>ToYW1zQrWHGBtvN|$Uh%`V>^^A3#*+B@Z<`RHfok9- z@K^YxvvU*iDyjx!WL4DDMbdp~85lJHD+&&a*M~gZSuqMo40vMw_Hd$!w9MMEGoxD_ z5MscpjsDv?^fwR_jHGcJFi@n1dpe^a9%Rq{_j||)vRy)mx~_H)xOg_qxL74B*YG3G zYY$heiZ1frqvN4y;^BEi6bt9@!fnk92++7)HaD!E0~2^-2Frg|Yy~0)#`GjGQ*`l0 z5l^~iPVuOj@Rjf>W`y9{LZ@P+i;5OtM8+UHbl<%WVmtR6;}!(b(KbyYuFa#%&z=^)B}ABViBBtrY8q`PYQQ1nu5=)|AuRb$rTw# zw*9M%2N|K4ifp>dCiku2cqWDf%hq#g%{c{x)A6y-H^JRbU-291UY~W$Kt!!umV)~I z&Gw~LtHkoj61L*uPgz$f{*QGXq)&1jYCD&m{N47+!&uV`|T9w=ENEPJihU*VZMHJ1ZGw&BZJxx-3~+ zUY)vXh~N%b5$B-6$YMA!sKCYlRm*UUFzSs!J|5SWQb3wR@Hv9g7XwOb-wUiNfdq(4 zQ&Y@R3+bo<3{>cW&byAWCa3)R)ZEd$c)orP-GZT2hw<>`(DXtk(8XFaN$Z~dvn!yZkp{LcnE%XyfA=sO_1DxM_3Yc z)X1chzj*w;M4z`fa$k6ss(nVt9kQYbp?L-DZoI1;58U5{@L$Oq;?g|O%Fk*chJjjk zMxd4*%JPfA$R{U?3q0CN%2e6;y!HItvm1pdWGODME-pq6Z9^;S$@U?5>WN~n#PH6kH#Paf1SU0MyTr&|5_)J1!$fwj$<3C} z%^`~--3O1!VDa6d*TgBR@(2S91cD-C5=1Qjo~sN_`^po)?XEE?Qsf`|a1rQ_H5jGB zXY{MRtCK@2G9Xdf7g#)Wt;6#Ae>i`|(JdLI{&6+2(@&H}>o$wh2Fr@~JguqX8)j4# zEGC^DVA&_*UQ!qTzne`M^Nt`T9{c-6YHP>JED`eT*M|V9gyehg3LtY3Ou~-byvb7A z-r3RIL4YUqMV>*O8r}rYlH#mFp`>+n3DBYudkIa#SQO?#M~}&!abp`VkZB+`VoSaC z#oLVK>>b$?qUkFA6!?cDvUL}d#Ti~>oh$#T6P7cX<7a+c#L_5xTKUrcC`W0m(X5V2eW;H4PcG#Ci8;w}6dHt76`chVLVCpsF zCW!_u60U6cUdTqYN+MIRp}V~GtFv{4^Pq|l@#5%Hv2%sO*IesQ@U26VwyqHInCDhh zXs(j`UL zKwR5&95HF6tI&;S$U!?7G+c<-qWNTPsRQc8N0@QgI$s5)IMC=CB0?+eiN4?PdC3Nl zC%kUiXFWTe3BkUX*&b&i;yIU_MHgPQczx8a_TlY-V2gK_Gbg{Fhz4cF>)b+^0P=q! z<>TB&m*I%H@MO}+bZEiSrN$*kHhBhL4Z1DS4TPS|dI=pD4t;1NwX7Cs?f z&8EoddlpIq%tC<@=#7=QI8<`gYDzic<*q;0zTmE!j;9eRKTMFeBCG z$tnLDT80HW8DnwyPC2~QLs_D|O70gaDox0goA%05ZA*E?193iKA$rt1p-^_@|973Y{Z*VQDs@?qn;nFK;@`TFA`OsyP@?hFuIp zos$232g8sgebnnR^KunN+?l{bbbBcUMYSWq!yiVXitCfgAn3TcV4)TN6V$57 z2s#uE1rJVdmB18i>^x{DpAZciJ8=3ijj}Xv(Txq!BmZasAc_FjnK?1QmOm8{Kvi&0i1si$wi%*S$l>d2k0qLegvNpF7LCZ*lss}xA^286$X(xU^ zIq?~-DtEwY(RI_b4sMr?;&n4sO~)&$+er_dzm@31%9T;=1)D{f8p7|F_YGmreZT!7 zf*K~jdzsh(kLnf2pWmN|Gsd)Ob(k1ng31QlqV~3OBXw^wWx4Q5afGfQt<~#S1ouJk zEcc&6FM!A@%34sUlB)PKm?5nEHFkoTXw#B+qTA`B*nTN_{RpS&EV+h1f&%jGUsio< z%7FkfDqyDq9rYUl$r?anpZ>{>Z;k>Ml^g9NW!?s4kj7n*f3Bgh6+*KrbE=|Z$oWxH zJ2^0)5&>v`pd0=N1R&gpC7%VUXp&QbaVmxIZ?}Ql#;OM*1`1H0qVb@HiXj`1pK7b=GhBnE$kL zC$PV8U+}{_%T?a*T>j<#WRD(fA5CyVmFECWka90^?nK}#%(cM=RO1qkQ*ST$AOqV5 zI9Ei;<2@dIuTKMvMbBI&TKEXgVf+Ub>R0;WR%s@q|$^S>Peixl1nVk^*VKETBS$f0VfE5c?d!jjWiYU- z@T9R>#6$k`wiaAbNkeuH(1^NdX3GIhjum;9C|A%a0QK#SL+2Q3=Rju4F8B1dba(3~ zzKYaB98#jthKzd@q$N2C+#^xN$%#*3?*L*c=>F#x0yo=h$Ko<&u0kczSzPI>ahLR) zC4IQ@as=jrXud$SIWX3NBf1zpm#7$ZHsbELIsUE(`dHZh8T)^amw+5UDL8~UuiJ5) z^E{gWZXVRAUp6&PT^Z}H?xY^%b>DZykWsRKBRk(rE;?U3|2z&m1%cOTC2nThw-3~@ArDUfMBA#>P}GvybKN1PleE6=NXHm}DtB|f*WAhL$g2<<>87Gl!7rDa4p zz$HGaIE+hZPb#CTX+Xz6cWPvR`26vY%G&`yAcmLvE|t-iV=e<>mDjP{5Wk>j*fO-o zXNOx)#`h3iI;pLpHA>0bKd1>8nV(s_Ux%}RxjE;xh&J&~Xm230y?SA*7QxBkVKHFq_~;pastC{09atE=PD~JKu^#sm}QO64skz?0#JR z{cHWey#L_VJd_4)V7{1MWI0*F0<3->KMKJy8EZ~SnL3xSI2cR5yr@a?1PEp)txy1u zkXdYe1m&1%HuW>8{n;lH2oh=MnLGY~gjf1{r#kuYf9@ zMQf3`pgfPii}hvvk~@LGr4200|TuYS%Qa)a|acv>M1BLoR9*%=g>-r>IJx3 zvN3MXi$Xg*#+l7p4~ik0~`Z7FdHV?JqP4KhS(7kr|Q9c7`9u zEdUo;RvaLng{#S2$<|_->{Gg}2VXS=?DWv-{{HWEEUH-h2Ya_+mT=T^{ta)RoG}b_ zmK-!^k(N9-XwCUcm*l5^zkhe3CXHS_vAy-(c6gIUhH>HZ-gi<0ny>Qn01=C1MI+kc z-}MawpW@;yY8B<;;n%-tF|1v^2g_QFQvxH;xy;WPXs=v#cf0}U#ToGAhVnT3{awOj zPILt9F0tS|HC(TI_j)aF)D7!7Mm`=!d>`bNYwQ~>z#QF8ZDK?01hBm%*_ z$2kICX9IE({=_~vo#5wWtsDH)PuVl^WO*)G?we7@T`rlEMuc6zgZ|~0!Ow%Ui0SkC z&(i~zqz$_$Iu7?_Dh_q)>NiT$lU8PzR%$K#0V%RzC~s(W*PSnEenMUg8hMU<_o$b3tu+YIL`sr!c;h8$qQhxDE}Cl%fYu zy3R*in)C-6?l-HR3UUu;G`-FQ#Og~p(4ji4EH3pEt>8fBTQp1#Hh_cc!O)CpP1N|T z4-n|5ps+hdp#QhPDv@8WajfrkLwL{?!(m{pJ5cFTABj9q$=lZy|3J|9g1Q5F? zhur9gK_v{bQBLX%%3c^ukJR+$;;+)VF%tQ*MW0?pcw z04X&peP5}D>_Ijn{)a8DaTDu5Asj+;h*~f57%9xBoZ_kW@w7727h}%Xs$+3RA8B&0Cmi1x8F)EAUQLn14;z22Sv(TK9kFx1|Qp2v^IMAyQ zAVe5@Q3u}ngo6hj6KeIH{EXa_rEAO82!H^^EmWsC~o@7~O|6UZ4p zdfooH>73}JF_>3#vNfTmp%X&ndRQR&VR6RlN-qZ=gHbR4ACNigx1<@$47uVwt^pEs zr$}Hw>pCBd8jMwg3-Ko)I*i;yi>v2q$}>AvdQhTi&8LCxg^l{@{=wx{kYy28D+fN# zs4JRqr)!?cKa&37L{9p{woI+y*AGIA&6j(;mii*6Li#DgGLTOeim(l1#OhY47}I~%)784q9*7xiMxh5p zl#*V-fgj}0_eR`Kx$ubkJ$gLdRmS~?VMLSw3iLRAZkTct`0vIM{nkH>BOO-s7``r> zdZ5xsYkNkyP|<9bvRimXFpXR1}$o6HBSgjF?8QFmaj`oBCaj%j>A&jYs5*sO^Lw_NphXWk6;4HI4g)^NYC^0B zKeb(=_#BI#`h98Tdp84eDHB_t-8I$`IOVX(tfLwjPV=Rh;)KM)dbsB^A2I&3x=!=8 zWFzUA;l)$!{sX&uE(_Xt{c(sn)uWsw4*ZMhRgw%6)Vd@GVz`CgvBZCg4 zbkMEL-p-SnQq$KIacp^;@U>5_&`mC4nMI)DatcCRn_i00vDzcRPbHhksCBoaeHAb ztiOQ@{M)AY_^CdMm+W`Zi?*UH;*nJq<55nD*Hi7-?LPMcEp{KejNkE<%{efh6%77N zCa|9qzxqg3BS36e47_E;Ey6$&c-cW36jVdwx9uoP7%UIYZLoayG-^0-^+xrJ%JM`BL?4FZ;sQP z#G5JU4&~~jVLER|M@9`)73tPT6S}oKziMNtWfz5jS1lir;F7OA5}rRHK|8ovNycev zw{79qbYADa{+g3tp#P=NE$bDG^f(oFR>P2UmbWZXV+r?$IJ};GV{y#yCtw~r3S>(# z`%j<)Tu{@eZG=D$u?mzXDJwpw5N~rNL-|OOPVPeujV4WlBnpc2+JQ$;x32WC*o=*Sf;>Ej zE_y;<(LA>!7fq~rNQsV2BH?L`Y};ZkaR0<%@=(A#2(}fg4X%bC3~2tZ(`tj+kSXmX6SYTmrb^Mg-SDLTI}%t5D9$l<1NW z(328B_&mK3vmQeTJUU)veXsRNJ3vt&xx7WI*O2hut-SjI0TY`n;t=Mpm{QleYMDLl zbeAEabm!I=0^_L}#lwT$Dx`9f=vA~eKC((XD4iH`QfrBW<5_)`X_pHM2Er+p44Tg< zo53}~F#kjfM@h7QDqF=W&oH>T^>nx5z!q&*ctspQG14W#`P{56FyQp&$O-i z7IEy0KJN+obCua@1b|U#bR}MKX2r@|jUBkjC60fi@(`Sf^%eeu6&17W-BSrFnTrDD zO_t)zd2AX>vnywqmqZQ97+-eJ)GhC&XnLO2)NUg-(T8Hl1w=M+;^$=Fa2XrPTFvgz zw*p?2;uPJ$0^ngazP-LY?9Y#vQ+~Ea4DwysiB>SDXNhs)ex+92lmX>K_SVD5_Ik}9 z5VVH-{5k}uVtpd=OaeP{fu_88^9t~I*;Fk(XO~(81lSD<*yGG0xW=_ryZ%vp;uV0| z=JtDc@9`%ELbpQqD>c-s{yvMm(e(T1b<{cR-A{e8)w2W zjS0W%p5P2)V2V*v!kVr_AphV2x*djFuofnhtm+cDTy3|l{khw+F`L@Wz(O?$&Z4)^ z<3V?U;BF=j7V)N>mUXI6xhN;p>74$a66lNEcUr{IfjMwd40#1oMlYtRTNhrxT|mA# zs>3<|GEyc@CgXmwO+y*XrNnD6G@U9PZ3=o2j+w%am?drs`e=%H7_XjTv z-fCxF@iIpuI6Cn40b#MU<OOq>z3NFXSTUECj z2+;->2qgf(iMVnIAJHFtGpu2_i}^a&U-)MZnCFXQ_Kk6)ra&;l&A8t~Nf(If8 zy?Ftt_`SX78+776oQu3T~{?w;cgQD#we_5>PpBzli6&$Oagky;y+=gFtdBq zqTfpjnJdhUhG=ka4>D)nDYp&1J+w!|Qj^)L76E}4nVMH=j7``bG8Xs0k|;HfqL=Ie ztLG^HOgt<2X>26WchyR#zU!~+&02I!5&gF;B@ zj^BYTKzZWm7{)z$?ssabWevl&?$x5L*Z#K?eKYwc63#4pPWxR%lt{^!@K{F)ltVCl z0=&52e=_Df;4Q|XPcEwJt{qmY=G7Ow0?WtGPjWg2kQ33owJnHEc4a zZZQuubXV(!BCbU7FsnB`rb*5nL~zxHmO3f7g1)j|Iy91S>^hB}dM>QmjaYrFMvd2) znf5v0$ZMb7D`5n}81Y|-Eju7IKOmaBhl*?FO7EJd3&<>ya@x4+$7eMK?P$WD@oxd` zW#G95pe-n*;wOuj6T3==-1#`e6OBD?x{tWoB|9Z^-qs$v*4+TX`SY+$&;zFi01@$z zFZO#JqJ^$8-$ix=?C}pTz8&PY&CoL}-C^hgMGa!of=d>Wp5lThCyg1~)aHs(#xoI!~Z2*COTxAPY>?BGvbKcD8 zv}JPO-gEo8CFunQOAQ19U&}~~t8N@oB+}QO87Q;VlEzciG};$Jr1uxZ<*xun?8G{T zGG%fZBnn>_I*s(xV{zI!Yeb!FFOaF$?#aPv{-8)G)x~ty>FhhzyBLu&EcNdb`qqxv zg(XYHtp8h661)0KDxD}vup?%aQJejxMe&p!11`ITHfwYg_e*B25FP$uRWo`2%sJ2} z&}ob{yG8XW0LB~^ibYvZz?k!C!|R)qQjt)y#x47TVzX-yD%PtqH-)-r9_qS58R(60 zTzOW9Uc7i1vI_s=;D`y8`$Ob3BG0^H<(SaS)4IXIPOr5Y{rJA;69mOG`z)(LCM0&D zHJGIw(+@UH`M}cq5sai;t2fq`J{qU2yy)wK;E?BU-|AQzf7eR~gq7a1hzbWF(;8g;XR6z2 zk|Vv@%*eGy0<#mp5_}3)J_Z)h#Kl7-@5+`fl&r^ar3LwUY-*>tmTy^s{5l22Afl@PHJwFb&;0E52} z82o-_h2JGUH`k86z=8{tfKQ;;=#VPeD)y=msZuz$j6|{qKA~p zgN7;WPm4t@uoV352k(a*7v_ais|}h`e|$>qt|r$6^zv|si^EEQNhl2GhL$KtKvW9B z`6)NajNTLdrbdTvEXADfqgPF7liOQt6P>n@eK8~4#tf#BR^9xaLBox_uz^PY1kz)IUf z7a2G>XrBCxRNV;MybcV_e|kl48{s(1#2BT%QS@RV_#%qDV8yRgIv_6p8uo0E3IeIY z$hmptvlp^0GlBGezTc-)^jZXdogAp}{@kI{kc-x)#(dxK{q$MJ+iz%9A&kaTIK(;b zB!={~|8o0^6Ik^jHQ#u$dmKT*nGPQgmAf(Z+{2=RtFUQs%>zjf!S7l}*6z-Z-fSrQr!C)AXRd~$yGWHw0ahL@QD2`)u7Vwk64wSA7 z7nDOt^HQfjg-Rb=ZaH}NfwU5iX%?K}TCQZMVB9O>@Vb|=8|83qtX4yN@?G1c!aJw; zk0=O`Djv#&0up(zf?(6Sz6bZ8pCF7u0{Xt0G1}DWib6Z;XVx#74^W^CJ$Gc)mm#o= zA9h1>jePF-%yW6Nb|Ql4h27f%dxa%d@n%G9Q`1qj2s#ukf)9e#adZyzzK-0y@mz3` zQ*v3M)2-Hu7XukZOUb))UIc?Fipp(7t>DQ|Q2s%`3|VVJJ-gB}((;DUZNoyp(Ph?c z!vJV-MP^*C;aynn~IH82eHiOeR%Lr-Ob#0=?xy z7eq}5&eI`L0i&2LV$%uP#z1P%kKD|!6y)^18cSYF6i=G9R9_$(Q_SbeN>h`(cmK23 z*NCd0<>3qaw_v#GBAll0k(_PZ>MC2u9MDq&Fw=s@AT@K$;ttMK>~XNoZ;J?gf9hzx zg28XiHW8%Q^av*GPapgbqx8`8$TRb{ER)}v!Xg}mUMo*OZg)X6)U>!PD2ak)D$6a9 zBhQMagtYx+2bOvK*JVTwr|=|Ua| z&T8&P3fFM!X=)>fW8cWWrTv1^XdKZSdxT%OFQ%QluU<{Ngq+;hXeIUDI`WL*yqmBm zP~Q~PW|!&}0{Tm8u^w`<<|&X<(%BCos`>-Ff*9c9#N{*;+znU|&-M?lJpP1cSGfYx zfEmcY9LuwU9TmE~9b#76t7DH0ZcU~J)NO4jHIHXBj;9#dF4TVjO+%f=|E13mXbObI zeoSCOo=!7LF^mY8;mMcq4g#VOj{|NeyY>TV0nC8Ls(@HsQdovxh?aOYdiv9CIbjmg zyQ_w}&TNF!NM!qsk;`b`ZU?ZST6Qdj`BvP{#KP2D$7VS|f` zu1jZ5WbuS(t~iR6TYp0vBewOnfWI3C=2{vCDdE{h@0yeW;^kD`GIPLy zJYeK!@F#zE0^%MI85^>p>O3(dctX-;0;eKyNgg&nh#?APXiM01~mPWIYB&$twdQaAE7G@2xPWCA) z$-EbK=5{uq?W#{o`sV0-U~R{svWYH%N5SQVl_Q6zNfwV zs}Dc$`3PTd{!U2qozOSd9COQ!UxOh#^(m- zHfTL1E+?$L@D@Qnlh8JnJ>kY(rKjN{_-e?6e2{xN3iKo^R|vh)=JK9#u3 ze9|_LQ(`qz)d3reQ)r<8=H8K(6GJ^vr(X>3C?0qmb$myl?9hg~-RA`FDqv*`U?IZ7 z=jDU(D=>2)o&`8&cyd{aU+fGa1j@Tl8s+FtkhdOtq|1MG{hF$quIUK>Ozx-I<$NqY zp19TD5;Z7p{=Qd(Mdz1GGpFAUgmeef2$}9MdMK{@UDS**gYm0dhgZe*fL_5zOlRIEV~wnp$IGZxQEicfvt-o9q;n#*Nohmv3H< zKkpb_8)u?eX;U`9_Jsok5Kdtw0i-lqVkLSDen3gh&lCcdTi0XDGO6vA)+T3Cr86XN zg;K)yD3=awRLL*spV{_1ZTYu3tq68>O6kC7IvcAigmK!60Za7@Ij;{Tts>@c{^qB8#ht_c{Z~!eBJsoCn@w&lyr`}6^>G#VsbxyIfHG5fY6DZeg)Mv4Tg@VSG zB5^|EZZ3J>t?I@uk}?4%H?Gv;dw6PfNQY3g_}2xh9EUtVrH@^flgbhom-E+lxwNJ8 zS>rAEuL|fN=x}Sjr#s+m?rNkl_~P9<-}*T^W)~ht29nlsWHY}2f}(h6Tsa=J1RWVl zkd%n!C8}&a=QdyeWIp3|s?@w8bqPZTl!A4F`76{uPz#XU1x;k!;uq{`dzXSO%a%=- zz18!Y#o?`O%ZckCF9Y9YLLDf2kw)wYJ2C))?cA?w1DCnRA7)YaCFW8Jbao{!-<(bN=%Euf4v&;xlOKGoa5E~z` z9vvQ79S$RMJDpiUU9mQQSvmdXwS)@I$eiu_w(;4|Hn@U0r=nu&2gs&X|%{=uLM&uhjg}Ian zbyYRSHT-Wkn(RqbWU~Pi1XLnt*zuGDXO7qU>CCj&xeD>wG;VdL!sANF1-|}eTey5u z^;h+!@NE+1TI`6>Bp7J{!K zKzPle?bWUtKi1m4W;@!`QkdWfYs6LK_w~p_tmx5Lm}YW?~R9{{|NM_y9Ew^JlB%q zPPkR1>fUW(j(wX;yo^_6uV$Ndd}3!7X0$t(H0~ORb{0m5Wp}6@zXRCukP$+r9Hw&= z^M;GGMXFTW6|>hXFYTSa5zA(@#I_)5(UdWi|HUWurjODQCeZ2vOT_3jY+TJY_=3@s zTBnw-@7Jz0>y)l_261q-?)JhRkEd$cO=|PhkzbvYs@aKPH~la@oY=LxeLHszuc%h~ zE%e0pT*7WkF*Bn$%EUn^!oVf&md|HAHa=h+>O{3&<(G*(ccyhNMiEyN86V%C%oq}w zrdF-(A=TEXatX9y>0nu)yH#ETC<UE;=;z@TDhAt| zorXL89R6RVRRnw_TfO{YTi13V)=w$a^|F86QglT3nAdGVha6>%M7Bghjv>ny&JA%7#?La5X;qeq;G?n zQBbz*Hxp%9JA?t(X8e&cj)D~Jx~Be^_(D5cKwf0gZQ5~?siFiry3~WH`!O+Kr~qhO zjKxj3Hr(lO&$t^VRM+MAhMS~086bs(b4;ts$$`trBn&mR9N&{!gd&UFilUrGb5l^( zAf9im{i`XzIGi{`ZGo{`EnZQy;vXDH9jI_5QJji|IBMQa)rvZe3pwt*!LF_V(t*BW zEt5x%3E#*at~XvP6%UTEVv1aGFNm>SGh5@|Kz$RSUbdbrV8Ys`;SRZZdL+t%Rp=arKD z8XMxrN*%kMmjJ7d5y^cL?yS3Aky)hDL`~dk8=M^VSMAWYkwq36;OPAM1+92uep%VtYXjPhS@Ln{0Wl5a7bOfQ;!jmm8QIS>j5@_csq$=S zjz9cRE-uLni%RnxwemVe(}7n3>TDQVIzcsF`8nJeE_GL3fu0`_>W86fRfrqvX*8)C zjz`(9K`=}yIwZD)^e-zOpjdLMgT=T*Rmzl1Y)qOwqro-)aJM{rc7T^Q>0)XC|FujBs|~>z#pt}17yjfd;6Y@u(7a1PgL}2xKV+AEnRtkXX)xq3RHIiHx1BT0CC46 zPuE@oIsR@RTuOMUgII*E43wCu{flfuOg%LBQn(J#9RaT9i< zEir?*GZ{qOg`+7esBNhu|HTd%ex(Q^#wizI6x;Oa`=^JYcI8gl?6aD&gPJGf+pja~ z{ynP(GCDR{yH7O05WSkmE)Q34Sp>LIn!1ILoJgO5+(Q$~98)a*w5AbW!n#9wTW! zhWSbs(fVt>U2%_P@~Kyqb$ERc4Dl~Be?X$X(4-f_=3L)g{S47ReXv{*Om;Goa+$?Y zcSDi^Smqc8&K``$6k8{FC|2^Lw8|aBoiR8cbx*2d>1(0j5xJqgeQ(VP8zqdcWDdec z!mBsLOa@*utEmp=CGhoAKad!$yH*^m>V7~q!P5rUmwyRc?Vq^x0=-o!W8+r|C&eNU z<5p`~b}479u)H!wFubE5!|c1bKoAX1OQrehL}^uo5u#+(EpgtJ+{dD%h{wyOeZR8; z=Uy1*diyKp5A82~aKKh6@N>nl8d%_*G(s3h>UH#}3r$(`rN_zXrS}%XcY7=2A>HvC zBf0`l=DyAy+oD_k8BnoB51t@ws0m~HpXxzyvlsvQ{CfN{SFz%>{}s7>OJ8of%zeK_ z^TG`|5==p#hR+q1f4`23?T#Z~NTN3o@BRUy+oKT;zp@py zKINUeV0z2U!9BKQICSK-fUm;a>eALRldn+K5ta1k$iIe=Z& zy2M`3W=y@75mQ#Lr0}<< z7QjPshzygexVz|$H|qPTh5yB%hj6T8fJ|>Kb4K=N^V4G6%l2W|z=|4-j~O3i*gF(b)7`=2-+l;3Gj8Lv_Br{oQlHteVqe(c(O#+_Z8%-P-vZ%M^D3Z7GDHke;LS7QE> zw~Hb07?ot2HYaECNj|zl_5{Wj>V&A`Rk-~5e*8?R^;tCA0n^|BbqrIYPE1ZFN-s@E z*cYQb5f?<){knVvbTmBj5l!g|8#tMQ;s?FaUIHjz(()_2@ zLkru%4L@N6e%0epRwSp*r9Zxb{LI-5+kVCZ_yFsD_|-1Xf+b*`~UH`Yn-FA?O7J0Z5!^SWTp7fBTH6wYIwpa zS@)~%=XViEFk3-FY%|!WKER@9ti9$(HIPEq*IqU`cUq|g#nEW(f~oGFOLj4#p8E6y z^0eg*RPPX2)HBYiUb64a|1>!EBnc5~57R%lT{?9inSdz(VHm-vToq0D0dz%;mtu_fIH^tp z-W#Sunc>{wwPd#jK~urVJ#~3-H$iC}TbMrjoRpE+U&>%OKX~z@0ct=9Hg}D^t@zmd z^j-18Yd0Gm7*Q#~!-k9B1^VjpAG}gHb@1j&H~-hh9-v7=1wQw%L%H^Vq>{WS?**;L z#!Ecgl4n`niMFCnbmR|>dVn0!#3fBH;IA8@&@D6KHiUs^9_2=T$Y6C8fl_ zi%ZOKrL<4n+q=O5L8JGtsL=WU{!@!$d`m<@U{jr1EZFc8W&0;j58gWn=X}6Tj;%Mb z{FCjJtU~W&yb`QdopSiDdUxGFN=SgYJGcUV_-?;=nI^IB51J`r8Olo3cjqO%&+xr> z9SWKM0jaa=#&Z_|qas9#nD>x82)=dqF|M`L9xu6M<&!Hn#i5kV=5ohkhD5<7(G ziO%H-3NWQ@%;?--m3>PGjm(K}z6E1U)`S*ge0|tK7fcrUu1Zd(nN82qQ@y2D0(FiB zln(^h3;_J%^@8dU=zqYK#c0CG_0hDYGWjuvF7Wk}92q%L38N*fo?m=I3O*g#K~4h4 z5rM5jPX4Tdd}o+?(3L^dfb-1< z-9ZoqGW>~%{%^kAvGHg4JVwvO2_(10+>M|TNr1ocxZ{Dx7j#%$`*>irQ2&#re?n3T zG{0jD)2^qhxr@pN9_a9YvAp5fm7^LnJE76~gu z+c5;sU%T@#%NBe)&-kh>s`=WK1~)be?N8}EK2{|YH2*H~iF^f5XasH%gijdM*TEte z$o&(1aEVCNqJ^RwKpvF}3E%DFdqO&%de$&m{Zhx_>m)`rF5Ah1a^De@LHl}zN=;VH~cwFXiM^Er8q!3d!8B{k_?X46s3aH>?E z+Q2+;;$Wh8zm%&(pjz8@OfLs$IVKG&j5h$~TwN&LpbbUU;2z;|{g`0EsJyHmxLUvg ztheMG>!5paI2WGa`oNnTdx;o|B_j_Dvc{CM@D zDG(y5eCKUscR@kYr<{NlQJ!OyV`q~6?rg&7lrKoUT+Sa!)Z{IAHs%ocC8u|C3Y2?KQK$p2YQclTNMn^ln;^+6wwKIa}X_0Aq-Rg(mb6pJQ zK=hgL3Z8;Bd3m#mf@FSS$%1P(scp=vD0ES`O-VGms}vuPa#Gq*l5jvQEO<6YGsIx(&IZirl(lSpJN@#fR-kYdM1~} zFUG^Lj$~09T-PWqg2Y>xL-&N65Kw5#tV_Nu1@!W0y64@^?+Lu_|H|WK5W!1bes115 zNt!i1>1mp_)H{>!BcL*0G&IDS-N)HT(1qZP_!Uv+!{0QvU!t1jBh4bUZC#Wr=FKhp z2;kKO_6G%D0ukb$F0_Yz8@d-F+AimGlUx9uz`ei5lQL*tF3$%%t&JBkalf||wtdpH zjZ<|!ad_f6Uh=!Me3iKR?@%#epx-e;_veByc}f4lz3xm`F&gWFsc37dMITB%-@xnK ze~o=P8gT{U@%)z^?)vns8N+!5+S$7bU>~e{3hlV0J)b2!$qEv7jilYrunnt%X#kDX zynk-htGbiD+Xxk#3bM@k*OZMK;5ZL!yCNJC)RS!^5BMhHL#tq_mdON1olJ)XmeF5W z?_v?gRii!qaPc>U&-_nl3DwTg?^Q9u7b0Iq-uic)419#wA^Ui|;FA4&RHVN!dtXFx zcHlQk@mVi;A@2dVeFM%9^#2_A0Z1H2OK++@27jqoQsn+?LeO6Tg-E|06^IY2=<5@j z$;GH#v8%3G@ThFIn$^)W58;%9kntX$Q>Tnfh1%3{-cm2%xc<{|Kr!D`&PEJ4Hahw6 z1K&Rdu^c^~5x$qbD^y}H-WLdKx+xZ7Emm$wN!>K+IWGrJGz-KJoqscs{=1?6_#*l7 z67r+LiT6(_6vu9%zb2CHj0``&>n&$1<4^;8nm^z6WVl0bj@uT%Y;x2As2ipl zy$P>t{%|W3+{HT_XcQyG`5rx`i{nt#XCvvE^SjWSHxW8JC?{-ddFv8zNJo_xwSTU< zz+;B9-u^Q7-gzTSM7BD8E2eLi_%bB8=lmzgZ=znD@$XE&2N!OV>-Od6x01@bil>M`{*)6AE9|s)M_Nu4TPFqH~bX`KeT}a!w%{L)N(O9%txZX-v#g4Zs%#Yq7 z7&!B~X)HZwbFpCjq$dOTB%=k)~W%EEdK%Qw(ZfSUFvnlG@KgRV?wQTxyfQAaa>5{&p? zMDZ{@JDT*JlEa^1hKPUN0=m7wW-0{vWK%l&;6&i$_E^l$ek1$MA@&Pect&4h^lc|> zG5yLjQ6?DwdAj3;vc+$Be zy@5F%T-uKw24DhErzEVTb z_S7GM6w3DB6S{wO=@8>?KMk8t6ppgal>n9l5YW?|(TL{-LSdfb9D6;3zs&5%$rzEk z{|*uGTL%SqeP=9@rJWWxmdHsGdlAQU<$P;4fENW~U2TG|CLUdL?xcSrXK+DzQ_!V6 zw@Rwuve*AqsZB2OuTg&XH&zYYjWqWmS1xDScvotz_B*Je3|e2oRj|rF|L^b;((A?m zUc#LtKOOt}WqWsz(8M)ph9_o+q6ffh#A?=axN`>(+E{OaL>3hi=IcT-k021fKiiT%4;8eA*Mpb69EpiLf-);fBBoRBo)w!AD|-( z8F%vJT}{I?MM>(oQI?kH=U&&V!!@QK20)VmL)59;AcCV2QhW1g8<@!r{ZUc=zWvvN z^_}0v0T%V&ZLg<=eiH+JAp;41=JWh~NjeVD-PpU}G=~j$z3SdGeQn$kh_vykeZ?of zC>DahZUu^t*2oB#%;&FJni)v==cFWM-9kB0m6XV!0vnu64Nufn!t$R!=!?0I4_H~i z`;gjy`@+Bb>pvU@Ng5+Eq=(AT{2e^c?&c zEh+Zoms}(I+rblX&J)pU)XM7WB;(G_?X#5+wGo>e1WYvPTucb=Qavg6wy1jkvV>HdL#UqekO^8d%3wuu_ zg7XCksH)t8iiK62J=f+S6=1%#v37Y>A zW%PUTAB9m%h)ef_=gUdr5ECV(uUCEoK&qO4_Z-LuluEHwdCyqJ{2?E#{VFXY*2ViG zq?ona)1R~Io|Xevel|6nUi4Jn1Fnc|1&m6c$b*nue=OjD3Lbz!g50O9F_{$alM&oH zrJ=gLw2SaM=HV2;PtcU^vZZnxbe0Kk_Tm4?F-A5{_voV!$v{kJZ&&FT>7{z$h(6T4 zH3j{V6y)xfG18W)tcEPzeU=W5SO@xR_8rn#JbHi}@2@C6cjE2MU+F3#_%Gq>1HCt= zTo|jTNtM#5vw#qJY&pMdBj?Hd84Z*RL<2z+k7tKg`>x#e%z{-W$)yb1Zd9NJ-1b_4 zgT;*bOsdvn!5{IS9<%EN#!LUk$`WXbw=!M(701RvQ3hN}fB*U`(JKFX$0M&y%U7Zv zYQ!3Zrl+Y|Vy&^p{Hq)S&T_$R>>I}_q&M&aVYe%YUKKAI|3yLdKl(VAH3d*tu0V~! z*}h0n5jjuJl#P_1+Is!%;`-T5a0%$IG%I`r3L`(M_=tH@&Fk9ZMI4~!!5ev3DlvTe zv|}-SG3zry^Y^;Q85eI20@`%P5cd@Y0+N>2y9Q8P^OGxj3+VMTnz>GIBWU^YU z^OUBRsc>lQMtHU2d!6W?Q3CNlqXb-!7u|D^0Z#Ny`XY^0O8_#xWdu{K43W@8+4;;g z9bO&11TvAAAFe~_4vQPl>TTH!3~%%Yx-^B_{?w(z%e>n{4!ldCrcnk;KyaP3r<%#Gya?j`B|FUGXefDb|NgIxaLI4LAn$ zgHfCG?Cs=FOXL^v_6zz;Aj=znP0#>-)6rV2))6Z7L|i22=~{E^aFUFyrfbAZ^W^4% zFAC{@(jwRZTEv-I&@s>jjG_FeX+9TyjTO!Jc#UmCNngnf3E33>l$kqD|7G8lAKyb>;diuhfJP(^K8qqDsH=FyKQRchsp6W6$S#h3g%}N|BFh&Am>rp9@I+n-rqQQ$mgzEH3c{$;@dKQ1a$ds4s7B-gHd? zh!bdP=X`~Sr0?n0&4vC5UsH;^gA)CRSwX-FISmsP{uvP@{s84h9$rswKfOc@<>Esa^9fMx*aMXv>-pk_lGYX{>8I*G z8>6#U6sg-ZVxAO0Km@cmZ+8m26DNSIB)>6+k6ggTXMZi7fE{>*86&x1pr-H_eCIA3H-Orc}8Eax{0 z<+x|v_Zp=tUtRgl`C&3<7S?zE=fFXbZ{T|cKB0S)M9;%zidC~P+jZ6NDYV+Ejvlon z_V_>U*0C5P_WN714a_6P=f+e!E?g1+_#whTC6!=G9nj-LQJ|A2jdeU6vD$NOaireu zYt$dhO`7)eAIH7dxq{w>$Nt5U{Hf<%=fxs5;YbQ&&1bH>G+qGVqug^q4xBwv$j0sw zv-9U3^WS&>ZGr!`z<*odf4c=f`;Cc?yz+B!!GKGV+LI`i4m;}?Fy9YXt5{hX5oGo7 zM~%buODg;513P%IHf6DOL(^xIHmR?N@kzJasz%|{OYEkUQFh7Hz#s5zOp99YRO`0) z=(LHaWKU3S{LIku&i2f~%lfZk&Tl7p4p!VQIr`dOEz2)V3;b#qZ8La(Xe57WudlCX z@?x&jjT!j%=+m^Ui@^nz*IX{F8g}(ujw*S#h4nZEPAX|oQWhGkO9sLsoD2z^Ip=Gz zkja>#J7Ly6>(>kUGj!-rXwpzGs=Kdu@n+rEA?48c)No);S7q8*Q~!XY#2-sa_khE8 zj86S5cbbwA#6cRSkwd~C7z6ajpSgh zyVrf=OuE-8o$h|?dHVaWqGSYLU!JX-f@jWtCu&)MEKTbZfP7~oLM~Lj4tb@wtEWw8V-at@4zfbM{A6# zbX>xoWDQ=2)=;Qw7O>Afm~QgPHD>4XtaZ7F$i6>~C__gZ1tOQ)xiYhFpfs(Jo}i=Cd#9J^ZLRTYAQk zy}QBXotL3ba&!1QPc}PFErF?9J&-oxjC+~S_$r=1q}pSWl=a+FLoc`*)JBTEKC57R z4ixstp~v+))_*N{kdo0oGfh0F?agQ9$bQK?@f*wT<~Tbsp$YUDSw6AgvcsfrsEm`R zm>)O7-Biev>Dm?f&|)YJ@uO#cme;3r%fDo|P+K6fBc3GDutuj=&@Nc8h{oLsjl1tJ zeF~wWe#V$_T0$Vj3V=!gT;|CRAK6s3c^W#IxbVB-I{kIt>^7mxiYHpv!6{%4)|A3) zrA}u615mP_>zbD~$pHFgpst}JDfc_;xz;_R;<)4(W^*oG54vZ)xTO)pC|l9{sf|_c zu3*1210Di&*zdk)S0<(CaPzXaa`Kk9cG58#O{^DKCU^7-Z+7N&NHrw~&}e&Dtn4X3 z6*oaM0?C$LjBgpACU{RICW1cKb8GrVY-1vkta=Pt+OsUPLM_D4f;8T5{-&`R4{**d)&@B{4lRgoP_x-CiwHuS_bUt5e?X6BF`DWk=&k zP>4Xn=pr2qE5X1YdGcePPRki-R!hQroJinVx2lR}Uaac<)FuhAqE@nSn~IxkJE;t@ zUzd@R)#1l5%gm^!cHWFroKx|{BwRey#s+S7y1{_uoV${RwsxY2jIrPz-IwlyJ>M3+ zyd-SFV>D7?roZN<+6Ymu`o`eJEBc}{d1G~9Pl={!(p9YlGVg8tY*bqUN&~5GN263a z1@gugK4QQ_R zv}ZI40S}+hbetk+OMBJx`l07L!CevODPI@Q#@;bbd9%C%?zPZT2QFGE3juLmQcfsZ zH=v?wn^OQXeV8O{8VW}sLMX*cmc*xUZTc+CGZMs+C3(i$EU#+K*1c=pJPuA_l?}={ zHrXlLcj|?U#;|!;OY&pgyBXP9myu+#=P%P$*cr}Kb!&qoWbPZ)a?0@EDwZ7RkCSPk zaZ93joZ_OcgEo}lkjEId1%*-7nqg*0ub$wsl4BTL{yNeonSnFjDqWjNd2Ml~|1ESm z@;+LOySl)jOVE9dNr}mK(2uO?RYP`#of&%wA8mj{dIebu!Q@JVjXW)r!Sl~+P-vC6 zQ1OaHO^M`N24vfl(v2?N!O<0Iq`p<8gayjKQ@&IxfI}o~Wl}muyt0%bfFveCPi#Ov zTDL-enth`F#;jDPV=G9l&01?(CATG}o(b({#tYwF%# zz`S)cV*+QU9sCcyw4P-_nJVSTJ5t`jt!|E{^uW!lD7%89XUwrpTb#P;%W;nuxY=^H zFE$F)H;n2iLcaS@*Oh6rljz_gp@tHQC)f=Hcqb8I2(hSC@f1sDpFK%T1sn!5eo`;^ z#wZ;deClFnL~6nEz~w?rk<{cZTW#L$Oovt@+^1CgAE1Uy>I2y(b=n;7&}bjIK+gGZ zP7W6g#~VMVt$jHB0eVvX@U_sa<0!JBhv(_j?}vNbEUZ~NdG@hv?Z_!@e_V>{u9|ue zCyoj&Y=Nv2T3j-_oiX-i5p4AIW3W-e?&-T9vXQGdrGzXXy$4=vk>rSruGYKZ=c;wj zKE90CLuhBOXBVl~k0_G(`JV+w7NbOUWZ|exx8{5SSO|HRI)W={z!}}}pbBvjozLjH zgQffd3KUx{{zT0pq^kR+j9!OWp7{hCg`9O@H5qtOn2QcJj=iH5H;%Q8KwbF8AiAWt zw?LnEJ|FSQ6CTa@!l1c$jsfj1LJRfvT|epBW1XevXi(HF;czWdn)C6P1#ft3RF;RO z#s2%DzPlpPGVf!yL%!j(E%pUJl@~l=TIQ`jr1u#genWGIIBkz1w~X?Qm`d4y;eNS;|>lrgf>gMz64woUkxTp#3J+hmfTMM3fh}IOV#u0DvuY|b3Gm$X= zE9pFx4>WKQQBGub_G?#X?0+8q(qA1uK*_Q6>*w=zz)Tlq4;5s^+lIWVW-UP#$W6&< z#M@(x7)|1eSM+!>)V?yOgrH$9Bl3-@dFmH=59{gOLt~a$Z^k_M0s1z|S{Affh7+S! zHGfOQTz07?tcd(Q%ExRtf9m(S%pip2E*JbohtjR*|0yhSx?BN;C=aHx1MsGaf-=SHZ zgl#+wq33Mvml(5@zv|^l7gM?O-q>csBqc|hyfle$G)f%7YQ>@bN>RUhg_B8iJ`XI*x{{T%4TJ<*<3rGkp|q|5!BTrmUeZ(fO2 zs@nCfJm`CY#HyytpVxY>nSTZRTng#9yC0jbz}y07jD0?POZg)sZTPwYQlrXS9~ajh zykPZmKzidv$qmN))Mctg_5E4Qw{mjayO7zAOi+2)wTQ@(DCx#B%&cLu^^zh@Yud|lAAWelJ!HwUZM?=l2!_GM!d_RK_=sa|d>%gqx1_ke zd|S?6Y4lPgz3NURY>L`TJkstHqSR%L-hXyiP{mWcv6XNWrd3qmDwn9cauRi>p$$VT z5TDfJz`@D2%puH_Zu*`Ib_>pVIKwsIO_I)^*^cGTL_BPbjkWq{zhpO*vES4fx}=t$ z7oX5AVb+PAiS=^232k^;F0BbQ^@QX&WjrOw6uRo%){r%3Ggsuuoje#`TZ$1$V>+e}rL6^FWr{j!EvLj&8=KsP0xH8cl3jA{Zdocw zol;vEROU`Ohh?PKJ(GoPvZw1PESoLz5sU3?!y@47?NaFOSVp3SC6K^DX zGCd+&OY^G3?DKMy@%5>7#Pn_pMUQmuOb^o!&?bc!-S==x>UUfIgA9&se$YihFQw^7 zlS-4A3^@_Y$#vuI-U+gU(H&4cZL(5~YQegg|mPIVNT2 zA7Mm6r-+1$U%eI{l&1oM`grXrAa*DNL-Cwz_iMH%Z9(9?>tT8amiMV!vu;nn{{a$8 zOP-7tA1M$3VAfRz3ONZqd!>8I4t7TaJ-<%c8}j# zrAVWWgT&cgusF$J*-9W#?aC$_&K04re?YF%1ZzsBnO7TWTpu~JuDZry?D8dHLXq2Z zqmTr;jKnOi<~**>sG@7^s^;VV$f!1EfB@6Qe5h$X{VXy`_Dfk(R8Ue1GuE|$JHt7C z&Tgur`8zP%tKVR{I#Ird5>7+8_TP%T;6wX1(zeA{NdT4H?x;wuxosA)6I7eFHMVmZ zy`-*VslY=+BmRIk)r$2c99O{ofhO@L@6f?)QNg?%v>F@C+Z7)w-q&Y~>qg5e`N0sI3EyDm;2I2j@wy}3lPJ!8NxR%C;}yr>)F-*~|qo9Pv2J~2Pz z$`aB`5jks6j%>-G)`lj_6hSwL>U2J!hz}I>CqRBMu$+^<7kX^I&?5jJ6IO zMEzHTQd3h#V{zJIk z>qeUR?BBPVMeLHi>k|W~6uCuOUtj(ziLLHNFb$}q{4(?j?@eD39kw;ztFx8ZlSrul z5_XwXPuF7w_vjNP!?29X?F`Dhh>I8?t$s|A{{$)wb7LytZoYD5&4eM^wsxs(xe>y~ zo}M!Ve@6ELycFge$p#zDk}lmXC_c*_;kJ+FQ+3fFTGLV`P)C(hWpucXWzbW$qk5=F;(*rPLW1)m%48(pr*qv&2Lg{lUzq}G}>?GYlh3`S$1m~>agyV1N6 zF9+SlhLZ^MCY#3_a7oEXIe*3ac?lYB_KDiDiTS%I-QmyuUH(@*o3PGUA*ak}z*pJy z=5VP)atG<&7T9gr@>trA--&WbS(yiS*#pI>`h5+U1HLLU?kRuUkgFikqJ^6UzrF6|#f{DI~eX~5|)pH)vPLznWd0iRj>JBEzF(k@t z3V$(`Sjvk40~L7=9~g5+E;xxz788Dy9#}n5Tm}P}OY;=qt2Wvs3RXk1oC>k5`qe?y zVt>B%oBxYjKb5>UN3U#b-R{(|6bA=WQkIx{GD+o&y65#1)Gh%z`qf_;GI&WAgn_w% zqd5AHxxrwFgRt2Lo7t#kGY*prCIj9COhF^oZ?PQG83oI_=`^G|6bi-DH?N^_=ps|? zWw$E0p@2n#Ys2goQ32}mpsm*0^@YT2lE#>($r0N=8ygz{RT&u>p6`JLfq4J7YePIrCNC%_!Z%^ zCZs!DFy*hA7XD7XOi@}vwQIXtZ0OnV^$tzm-(muJMnA`t4~(8Sfxl>h+>Pt|5>K2zow50k6lOwHsAc`{Z7_&()vb>1xAA--p=t6?3YBHrdLSMq5KE731YKETD=G?eUTLRfIo_T(*urREZP3jCImdax?i>sP=n zrHQ;V(pzPV#ya*HyTRakP^$)q!qI+=b5lvRiFr~oSF$Fnpjacustm%Goa^YFmodrJ zb#IJn<}B&81XXk?Mp1~60S6Ml64gjj$;48^8*-DoB34x1lw)4|6s!nKn+7Hch?7f2 zMwH%QQEbQa0tsvZ@Sr+y^MTQwGr3CE4VDE5vKNY0B0Ke&9~ExaCfOGtP+k;%q{=p? zSMALB#J{Ywkqwi)k>EAgx7qYk$(RwT^tA12BgoZ_f55J~5&qMF8)K3qoy?0%(5UVg z*zlW|x|1kl9Fs>m`PJbEXmIM&tjO-O@2r& zjMXwyy)oAPs+OLqX85()3bq5{r}LI~tJuqY+`sY{WTlF+%~5m7i6aLqGDcg$n0PEO z`IQHGXz@b!nCQ$a@>abD`PfLrh}`&^9&%TVIZm5AVJg{YoBob<8XLf$zcFs)NEQotlAVxGf;Cpu|)2KuW zjvL{C;8Jdv){?g=;An;7s+~UU5gH3drCDk5kB)j4FKIuNtW^ppdxnE2GGF}hbBg4 zji&JF%eF`=18C}l=mZWo4cqsi6TIw8`bI5CaV=JLTy@Ppbm9$=N1<^JEQ5%7(84>5 zyq5_dWOu968takn22-Vn$iAutgCT1&(o`J-=W4Ymo@VG*y{JhP8wQA3|D<`;;_`R@q89&8%)<>d5fB&_w0jj~nk+JRMNu z`$0+KTdFVF?tkVFY6;Js-Y%Pc*lJ z3bBZWhean2Efak=8EX{|Q|}Whf+xv$+1wU+hLsv*UK-xG_3AQ{SZenrUoE$NCqY=# z4Nrr71UTbLhy&Zk7vdWy6)fFfUyY?2Tk&ig1>0@$j>8lANv;VD*f)}wVWBB+cn7(h z+|v1H$YT7|u)z^B^&Ghx4y7^o)YSiS1^G!jMxNPY?$jU3VVE3GvXrqthtUc&3el4RV9E9KE89sV%Ea=^ZYH zT3m}xz20fb&5$hwrn^|mN#SJByQ>X$l1qXG@B-7#ux)1nFxZ*empG@U<X*30FW zOr6O(s;DCXHe0{#{U+3^AvRfS73pEDrN-sjU1iH)G8vsHMTVxoDY%0?%w11+OQ%Y` zWcCf+Yh<`%jQ&Qsyt!ozEAug&sE0O#AuQB(5wLanRzKoXoy*{;8ynYO(?8Y5fN@qQM3c9`Zx# zdiH(5B46=<=TyB*{0%gJns*jTgVj_`cTh=Q%@`A_#(DX5Z~tv794JZFC`RB3<(Ti? zllM_Z#zI869+Md-?)+p4hXS|y65LkFM*|7*nfFZ3v?RY_Vj4x#Bw9X zWEX|V6J0K+yI}@KYY=mSGkkO%92xAU%eJK0iX0K%dQ;o4Rn9Fb9LBs6KE~SVTwRQ0 z1EEz0tPh~rE=y{i(b5l;SCmJ=w*p? Date: Tue, 17 Mar 2020 11:09:07 +0800 Subject: [PATCH 146/523] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 717343edf..c32a83f6f 100644 --- a/README.md +++ b/README.md @@ -328,4 +328,6 @@ into # FlinkX & FlinkStreamSQL钉钉交流群 -![image](images/钉钉群.jpg) +

+ +
From ea559d900e54e082d22464b3bf1e50b74d408b4f Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 17 Mar 2020 14:02:16 +0800 Subject: [PATCH 147/523] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/redis/RedisAllReqRow.java | 17 +++++++------ .../sql/side/redis/RedisAsyncReqRow.java | 25 ++++++++++--------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 69a99b32b..0d257c5b9 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -30,6 +30,7 @@ import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; import org.slf4j.Logger; @@ -93,15 +94,15 @@ protected void reloadCache() { } @Override - public void flatMap(Row row, Collector out) throws Exception { + public void flatMap(CRow input, Collector out) throws Exception { Map inputParams = Maps.newHashMap(); for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = row.getField(conValIndex); + Object equalObj = input.row().getField(conValIndex); if(equalObj == null){ if(sideInfo.getJoinType() == JoinType.LEFT){ - Row data = fillData(row, null); - out.collect(data); + Row data = fillData(input.row(), null); + out.collect(new CRow(data, input.change())); } return; } @@ -117,13 +118,13 @@ public void flatMap(Row row, Collector out) throws Exception { if(sideInfo.getJoinType() != JoinType.LEFT){ return; } - Row data = fillData(row, null); - out.collect(data); + Row data = fillData(input.row(), null); + out.collect(new CRow(data, input.change())); return; } - Row newRow = fillData(row, cacheMap); - out.collect(newRow); + Row newRow = fillData(input.row(), cacheMap); + out.collect(new CRow(newRow, input.change())); } private String buildCacheKey(Map refData) { diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 768c0cd4e..f2e164922 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -38,6 +38,7 @@ import com.google.common.collect.Maps; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Collections; @@ -120,14 +121,14 @@ public Row fillData(Row input, Object sideInput) { } @Override - public void asyncInvoke(Row input, ResultFuture resultFuture) throws Exception { - Row inputRow = Row.copy(input); + public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + CRow inputCopy = new CRow(input.row(), input.change()); Map refData = Maps.newHashMap(); for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = inputRow.getField(conValIndex); + Object equalObj = input.row().getField(conValIndex); if(equalObj == null){ - dealMissKey(inputRow, resultFuture); + dealMissKey(inputCopy, resultFuture); return; } refData.put(sideInfo.getEqualFieldList().get(i), equalObj); @@ -141,14 +142,14 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except CacheObj val = getFromCache(key); if(val != null){ if(ECacheContentType.MissVal == val.getType()){ - dealMissKey(inputRow, resultFuture); + dealMissKey(inputCopy, resultFuture); return; }else if(ECacheContentType.MultiLine == val.getType()){ try { - Row row = fillData(inputRow, val.getContent()); - resultFuture.complete(Collections.singleton(row)); + Row row = fillData(input.row(), val.getContent()); + resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, inputRow); + dealFillDataError(resultFuture, e, inputCopy); } }else{ RuntimeException exception = new RuntimeException("not support cache obj type " + val.getType()); @@ -164,14 +165,14 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except public void accept(Map values) { if (MapUtils.isNotEmpty(values)) { try { - Row row = fillData(inputRow, values); + Row row = fillData(input.row(), values); dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.MultiLine, values)); - resultFuture.complete(Collections.singleton(row)); + resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, inputRow); + dealFillDataError(resultFuture, e, inputCopy); } } else { - dealMissKey(inputRow, resultFuture); + dealMissKey(inputCopy, resultFuture); dealCacheData(key,CacheMissVal.getMissKeyObj()); } } From 5be698479acdf67e81c786e002009fefff848009 Mon Sep 17 00:00:00 2001 From: chaozwn <326747337@qq.com> Date: Tue, 17 Mar 2020 16:01:45 +0800 Subject: [PATCH 148/523] =?UTF-8?q?=E8=A7=A3=E5=86=B3cep=E4=B8=8D=E8=83=BD?= =?UTF-8?q?=E6=89=8B=E5=8A=A8=E6=8C=87=E5=AE=9Awatermark=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/exec/ExecuteProcessHelper.java | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 058ef6097..442ad8ceb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -80,9 +80,10 @@ import java.util.Set; /** - * 任务执行时的流程方法 + * 任务执行时的流程方法 * Date: 2020/2/17 * Company: www.dtstack.com + * * @author maqi */ public class ExecuteProcessHelper { @@ -131,7 +132,8 @@ public static ParamsInfo parseParams(String[] args) throws Exception { } /** - * 非local模式或者shipfile部署模式，remoteSqlPluginPath必填 + * 非local模式或者shipfile部署模式，remoteSqlPluginPath必填 + * * @param remoteSqlPluginPath * @param deployMode * @param pluginLoadMode @@ -187,14 +189,14 @@ public static List getExternalJarUrls(String addJarListStr) throws java.io. } return jarUrlList; } - + private static void sqlTranslation(String localSqlPluginPath, - StreamTableEnvironment tableEnv, - SqlTree sqlTree,Map sideTableMap, - Map registerTableCache, - StreamQueryConfig queryConfig) throws Exception { + StreamTableEnvironment tableEnv, + SqlTree sqlTree, Map sideTableMap, + Map registerTableCache, + StreamQueryConfig queryConfig) throws Exception { - SideSqlExec sideSqlExec = new SideSqlExec(); + SideSqlExec sideSqlExec = new SideSqlExec(); sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, result); @@ -254,13 +256,14 @@ public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrl } /** - * 向Flink注册源表和结果表，返回执行时插件包的全路径 + * 向Flink注册源表和结果表，返回执行时插件包的全路径 + * * @param sqlTree * @param env * @param tableEnv * @param localSqlPluginPath * @param remoteSqlPluginPath - * @param pluginLoadMode 插件加载模式 classpath or shipfile + * @param pluginLoadMode 插件加载模式 classpath or shipfile * @param sideTableMap * @param registerTableCache * @return @@ -293,7 +296,23 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment if (waterMarkerAssigner.checkNeedAssignWaterMarker(sourceTableInfo)) { adaptStream = waterMarkerAssigner.assignWaterMarker(adaptStream, typeInfo, sourceTableInfo); - fields += ",ROWTIME.ROWTIME"; + String eventTimeField = sourceTableInfo.getEventTimeField(); + boolean hasEventTimeField = false; + if (!Strings.isNullOrEmpty(eventTimeField)) { + String[] fieldArray = fields.split(","); + for (int i = 0; i < fieldArray.length; i++) { + if (fieldArray[i].equals(eventTimeField)) { + fieldArray[i] = eventTimeField + ".ROWTIME"; + hasEventTimeField = true; + break; + } + } + if (hasEventTimeField) { + fields = String.join(",", fieldArray); + } else { + fields += ",ROWTIME.ROWTIME"; + } + } } else { fields += ",PROCTIME.PROCTIME"; } @@ -329,7 +348,8 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment } /** - * perjob模式将job依赖的插件包路径存储到cacheFile，在外围将插件包路径传递给jobgraph + * perjob模式将job依赖的插件包路径存储到cacheFile，在外围将插件包路径传递给jobgraph + * * @param env * @param classPathSet */ From 14446f4dd844b1bb9da6c412e3556e135bafcba4 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 17 Mar 2020 16:22:07 +0800 Subject: [PATCH 149/523] remove hbase jdk tools --- hbase/pom.xml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/hbase/pom.xml b/hbase/pom.xml index f64ffab1b..835457e29 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -29,13 +29,6 @@ 1.0-SNAPSHOT provided - - jdk.tools - jdk.tools - 1.6 - system - ${JAVA_HOME}/lib/tools.jar - org.apache.hbase From ec2b84e7e1fa0c54aa7602923768b86671ab79b8 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 17 Mar 2020 20:49:15 +0800 Subject: [PATCH 150/523] kafk update mode --- .../format/SerializationMetricWrapper.java | 11 +- .../kafka/AbstractKafkaProducerFactory.java | 31 +- .../CustomerKeyedSerializationSchema.java | 35 +- .../AvroCRowSerializationSchema.java | 346 +++++++++++++++++ .../CsvCRowSerializationSchema.java | 349 ++++++++++++++++++ .../JsonCRowSerializationSchema.java | 234 ++++++++++++ .../sql/sink/kafka/table/KafkaSinkParser.java | 2 + .../sink/kafka/table/KafkaSinkTableInfo.java | 13 +- .../KafkaDeserializationMetricWrapper.java | 10 +- .../flink/sql/sink/kafka/KafkaProducer.java | 6 +- .../sql/sink/kafka/KafkaProducerFactory.java | 4 +- .../flink/sql/sink/kafka/KafkaSink.java | 18 +- .../flink/sql/sink/kafka/KafkaProducer09.java | 5 +- .../sink/kafka/KafkaProducer09Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 21 +- .../sql/sink/kafka/KafkaProducer010.java | 6 +- .../sink/kafka/KafkaProducer010Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 21 +- .../sql/sink/kafka/KafkaProducer011.java | 6 +- .../sink/kafka/KafkaProducer011Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 20 +- .../source/kafka/KafkaConsumer011Factory.java | 2 +- .../flink/sql/source/kafka/KafkaSource.java | 4 +- 23 files changed, 1067 insertions(+), 86 deletions(-) create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java diff --git a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java index 3a5af18b1..8802198a0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java @@ -24,6 +24,7 @@ import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.Meter; import org.apache.flink.metrics.MeterView; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; @@ -34,11 +35,11 @@ * author: toutian * create: 2019/12/24 */ -public class SerializationMetricWrapper implements SerializationSchema { +public class SerializationMetricWrapper implements SerializationSchema { private static final long serialVersionUID = 1L; - private SerializationSchema serializationSchema; + private SerializationSchema serializationSchema; private transient RuntimeContext runtimeContext; @@ -47,7 +48,7 @@ public class SerializationMetricWrapper implements SerializationSchema { protected transient Meter dtNumRecordsOutRate; - public SerializationMetricWrapper(SerializationSchema serializationSchema) { + public SerializationMetricWrapper(SerializationSchema serializationSchema) { this.serializationSchema = serializationSchema; } @@ -57,7 +58,7 @@ public void initMetric() { } @Override - public byte[] serialize(Row element) { + public byte[] serialize(CRow element) { beforeSerialize(); byte[] row = serializationSchema.serialize(element); afterSerialize(); @@ -79,7 +80,7 @@ public void setRuntimeContext(RuntimeContext runtimeContext) { this.runtimeContext = runtimeContext; } - public SerializationSchema getSerializationSchema() { + public SerializationSchema getSerializationSchema() { return serializationSchema; } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java index 88c2ca939..ebd313b29 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java @@ -19,15 +19,18 @@ import com.dtstack.flink.sql.format.FormatType; import com.dtstack.flink.sql.format.SerializationMetricWrapper; +import com.dtstack.flink.sql.sink.kafka.serialization.AvroCRowSerializationSchema; +import com.dtstack.flink.sql.sink.kafka.serialization.CsvCRowSerializationSchema; +import com.dtstack.flink.sql.sink.kafka.serialization.JsonCRowSerializationSchema; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.formats.avro.AvroRowSerializationSchema; import org.apache.flink.formats.csv.CsvRowSerializationSchema; -import org.apache.flink.formats.json.JsonRowSerializationSchema; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -51,42 +54,36 @@ public abstract class AbstractKafkaProducerFactory { * @param partitioner * @return */ - public abstract RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys); + public abstract RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys); - protected SerializationMetricWrapper createSerializationMetricWrapper(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { - return new SerializationMetricWrapper(createSerializationSchema(kafkaSinkTableInfo, typeInformation)); + protected SerializationMetricWrapper createSerializationMetricWrapper(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { + SerializationSchema serializationSchema = createSerializationSchema(kafkaSinkTableInfo, typeInformation); + return new SerializationMetricWrapper(serializationSchema); } - private SerializationSchema createSerializationSchema(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { - SerializationSchema serializationSchema = null; + private SerializationSchema createSerializationSchema(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { + SerializationSchema serializationSchema = null; if (FormatType.JSON.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { - if (StringUtils.isNotBlank(kafkaSinkTableInfo.getSchemaString())) { - serializationSchema = new JsonRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); + serializationSchema = new JsonCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(), kafkaSinkTableInfo.getUpdateMode()); } else if (typeInformation != null && typeInformation.getArity() != 0) { - serializationSchema = new JsonRowSerializationSchema(typeInformation); + serializationSchema = new JsonCRowSerializationSchema(typeInformation, kafkaSinkTableInfo.getUpdateMode()); } else { throw new IllegalArgumentException("sinkDataType:" + FormatType.JSON.name() + " must set schemaString（JSON Schema）or TypeInformation"); } - } else if (FormatType.CSV.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { - if (StringUtils.isBlank(kafkaSinkTableInfo.getFieldDelimiter())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.CSV.name() + " must set fieldDelimiter"); } - final CsvRowSerializationSchema.Builder serSchemaBuilder = new CsvRowSerializationSchema.Builder(typeInformation); + final CsvCRowSerializationSchema.Builder serSchemaBuilder = new CsvCRowSerializationSchema.Builder(typeInformation); serSchemaBuilder.setFieldDelimiter(kafkaSinkTableInfo.getFieldDelimiter().toCharArray()[0]); serializationSchema = serSchemaBuilder.build(); - } else if (FormatType.AVRO.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { - if (StringUtils.isBlank(kafkaSinkTableInfo.getSchemaString())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.AVRO.name() + " must set schemaString"); } - - serializationSchema = new AvroRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); - + serializationSchema = new AvroCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(),kafkaSinkTableInfo.getUpdateMode()); } if (null == serializationSchema) { diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index 498766564..cde8d1b1d 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -2,18 +2,20 @@ import com.dtstack.flink.sql.format.SerializationMetricWrapper; +import com.dtstack.flink.sql.sink.kafka.serialization.JsonCRowSerializationSchema; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.formats.json.JsonRowSerializationSchema; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.concurrent.atomic.AtomicLong; -public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { +public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { private static final Logger LOG = LoggerFactory.getLogger(CustomerKeyedSerializationSchema.class); @@ -30,38 +32,41 @@ public CustomerKeyedSerializationSchema(SerializationMetricWrapper serialization this.mapper = new ObjectMapper(); } - public byte[] serializeKey(Row element) { - if(partitionKeys == null || partitionKeys.length <=0){ + @Override + public byte[] serializeKey(CRow element) { + if (partitionKeys == null || partitionKeys.length <= 0) { return null; - } - SerializationSchema serializationSchema = serializationMetricWrapper.getSerializationSchema(); - if(serializationSchema instanceof JsonRowSerializationSchema){ - return serializeJsonKey((JsonRowSerializationSchema) serializationSchema, element); + } + SerializationSchema serializationSchema = serializationMetricWrapper.getSerializationSchema(); + if (serializationSchema instanceof JsonCRowSerializationSchema) { + return serializeJsonKey((JsonCRowSerializationSchema) serializationSchema, element); } return null; } - public byte[] serializeValue(Row element) { + @Override + public byte[] serializeValue(CRow element) { return this.serializationMetricWrapper.serialize(element); } - public String getTargetTopic(Row element) { + @Override + public String getTargetTopic(CRow element) { return null; } - private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationSchema, Row element) { + private byte[] serializeJsonKey(JsonCRowSerializationSchema jsonCRowSerializationSchema, CRow element) { try { - byte[] data = jsonRowSerializationSchema.serialize(element); + byte[] data = jsonCRowSerializationSchema.serialize(element); ObjectNode objectNode = mapper.readValue(data, ObjectNode.class); StringBuilder sb = new StringBuilder(); - for(String key : partitionKeys){ - if(objectNode.has(key)){ + for (String key : partitionKeys) { + if (objectNode.has(key)) { sb.append(objectNode.get(key.trim())); } } return sb.toString().getBytes(); - } catch (Exception e){ - if(COUNTER.getAndIncrement() % 1000 == 0){ + } catch (Exception e) { + if (COUNTER.getAndIncrement() % 1000 == 0) { LOG.error("serializeJsonKey error", e); } } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java new file mode 100644 index 000000000..34fa22c99 --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka.serialization; + +import com.dtstack.flink.sql.enums.EUpdateMode; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.SchemaParseException; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.avro.specific.SpecificRecord; +import org.apache.avro.util.Utf8; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.formats.avro.AvroRowDeserializationSchema; +import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TimeZone; + +/** + * Serialization schema that serializes CROW into Avro bytes. + * + *

Serializes objects that are represented in (nested) Flink rows. It support types that + * are compatible with Flink's Table & SQL API. + * + *

Note: Changes in this class need to be kept in sync with the corresponding runtime + * class {@link AvroRowDeserializationSchema} and schema converter {@link AvroSchemaConverter}. + * + * @author maqi + */ +public class AvroCRowSerializationSchema implements SerializationSchema { + + /** + * Used for time conversions from SQL types. + */ + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + + /** + * Avro record class for serialization. Might be null if record class is not available. + */ + private Class recordClazz; + + /** + * Schema string for deserialization. + */ + private String schemaString; + + /** + * Avro serialization schema. + */ + private transient Schema schema; + + /** + * Writer to serialize Avro record into a byte array. + */ + private transient DatumWriter datumWriter; + + /** + * Output stream to serialize records into byte array. + */ + private transient ByteArrayOutputStream arrayOutputStream; + + /** + * Low-level class for serialization of Avro values. + */ + private transient Encoder encoder; + + private String updateMode; + + private final String retractKey = "retract"; + + /** + * Creates an Avro serialization schema for the given specific record class. + * + * @param recordClazz Avro record class used to serialize Flink's row to Avro's record + */ + public AvroCRowSerializationSchema(Class recordClazz) { + Preconditions.checkNotNull(recordClazz, "Avro record class must not be null."); + this.recordClazz = recordClazz; + this.schema = SpecificData.get().getSchema(recordClazz); + this.schemaString = schema.toString(); + this.datumWriter = new SpecificDatumWriter<>(schema); + this.arrayOutputStream = new ByteArrayOutputStream(); + this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); + } + + /** + * Creates an Avro serialization schema for the given Avro schema string. + * + * @param avroSchemaString Avro schema string used to serialize Flink's row to Avro's record + */ + public AvroCRowSerializationSchema(String avroSchemaString,String updateMode) { + Preconditions.checkNotNull(avroSchemaString, "Avro schema must not be null."); + this.recordClazz = null; + this.schemaString = avroSchemaString; + try { + this.schema = new Schema.Parser().parse(avroSchemaString); + } catch (SchemaParseException e) { + throw new IllegalArgumentException("Could not parse Avro schema string.", e); + } + this.datumWriter = new GenericDatumWriter<>(schema); + this.arrayOutputStream = new ByteArrayOutputStream(); + this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); + this.updateMode = updateMode; + } + + @Override + public byte[] serialize(CRow crow) { + try { + Row row = crow.row(); + boolean change = crow.change(); + + // convert to record + final GenericRecord record = convertRowToAvroRecord(schema, row); + arrayOutputStream.reset(); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + record.put(retractKey, change); + } + datumWriter.write(record, encoder); + encoder.flush(); + return arrayOutputStream.toByteArray(); + } catch (Exception e) { + throw new RuntimeException("Failed to serialize row.", e); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final AvroCRowSerializationSchema that = (AvroCRowSerializationSchema) o; + return Objects.equals(recordClazz, that.recordClazz) && Objects.equals(schemaString, that.schemaString); + } + + @Override + public int hashCode() { + return Objects.hash(recordClazz, schemaString); + } + + // -------------------------------------------------------------------------------------------- + + private GenericRecord convertRowToAvroRecord(Schema schema, Row row) { + final List fields = schema.getFields(); + final int length = fields.size(); + final GenericRecord record = new GenericData.Record(schema); + for (int i = 0; i < length; i++) { + final Schema.Field field = fields.get(i); + record.put(i, convertFlinkType(field.schema(), row.getField(i))); + } + return record; + } + + private Object convertFlinkType(Schema schema, Object object) { + if (object == null) { + return null; + } + switch (schema.getType()) { + case RECORD: + if (object instanceof Row) { + return convertRowToAvroRecord(schema, (Row) object); + } + throw new IllegalStateException("Row expected but was: " + object.getClass()); + case ENUM: + return new GenericData.EnumSymbol(schema, object.toString()); + case ARRAY: + final Schema elementSchema = schema.getElementType(); + final Object[] array = (Object[]) object; + final GenericData.Array convertedArray = new GenericData.Array<>(array.length, schema); + for (Object element : array) { + convertedArray.add(convertFlinkType(elementSchema, element)); + } + return convertedArray; + case MAP: + final Map map = (Map) object; + final Map convertedMap = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + convertedMap.put( + new Utf8(entry.getKey().toString()), + convertFlinkType(schema.getValueType(), entry.getValue())); + } + return convertedMap; + case UNION: + final List types = schema.getTypes(); + final int size = types.size(); + final Schema actualSchema; + if (size == 2 && types.get(0).getType() == Schema.Type.NULL) { + actualSchema = types.get(1); + } else if (size == 2 && types.get(1).getType() == Schema.Type.NULL) { + actualSchema = types.get(0); + } else if (size == 1) { + actualSchema = types.get(0); + } else { + // generic type + return object; + } + return convertFlinkType(actualSchema, object); + case FIXED: + // check for logical type + if (object instanceof BigDecimal) { + return new GenericData.Fixed( + schema, + convertFromDecimal(schema, (BigDecimal) object)); + } + return new GenericData.Fixed(schema, (byte[]) object); + case STRING: + return new Utf8(object.toString()); + case BYTES: + // check for logical type + if (object instanceof BigDecimal) { + return ByteBuffer.wrap(convertFromDecimal(schema, (BigDecimal) object)); + } + return ByteBuffer.wrap((byte[]) object); + case INT: + // check for logical types + if (object instanceof Date) { + return convertFromDate(schema, (Date) object); + } else if (object instanceof Time) { + return convertFromTime(schema, (Time) object); + } + return object; + case LONG: + // check for logical type + if (object instanceof Timestamp) { + return convertFromTimestamp(schema, (Timestamp) object); + } + return object; + case FLOAT: + case DOUBLE: + case BOOLEAN: + return object; + } + throw new RuntimeException("Unsupported Avro type:" + schema); + } + + private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType instanceof LogicalTypes.Decimal) { + final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType; + // rescale to target type + final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY); + // byte array must contain the two's-complement representation of the + // unscaled integer value in big-endian byte order + return decimal.unscaledValue().toByteArray(); + } else { + throw new RuntimeException("Unsupported decimal type."); + } + } + + private int convertFromDate(Schema schema, Date date) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType == LogicalTypes.date()) { + // adopted from Apache Calcite + final long time = date.getTime(); + final long converted = time + (long) LOCAL_TZ.getOffset(time); + return (int) (converted / 86400000L); + } else { + throw new RuntimeException("Unsupported date type."); + } + } + + private int convertFromTime(Schema schema, Time date) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType == LogicalTypes.timeMillis()) { + // adopted from Apache Calcite + final long time = date.getTime(); + final long converted = time + (long) LOCAL_TZ.getOffset(time); + return (int) (converted % 86400000L); + } else { + throw new RuntimeException("Unsupported time type."); + } + } + + private long convertFromTimestamp(Schema schema, Timestamp date) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType == LogicalTypes.timestampMillis()) { + // adopted from Apache Calcite + final long time = date.getTime(); + return time + (long) LOCAL_TZ.getOffset(time); + } else { + throw new RuntimeException("Unsupported timestamp type."); + } + } + + private void writeObject(ObjectOutputStream outputStream) throws IOException { + outputStream.writeObject(recordClazz); + outputStream.writeObject(schemaString); // support for null + } + + @SuppressWarnings("unchecked") + private void readObject(ObjectInputStream inputStream) throws ClassNotFoundException, IOException { + recordClazz = (Class) inputStream.readObject(); + schemaString = (String) inputStream.readObject(); + if (recordClazz != null) { + schema = SpecificData.get().getSchema(recordClazz); + } else { + schema = new Schema.Parser().parse(schemaString); + } + datumWriter = new SpecificDatumWriter<>(schema); + arrayOutputStream = new ByteArrayOutputStream(); + encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); + } +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java new file mode 100644 index 000000000..903395f9d --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka.serialization; + +import com.dtstack.flink.sql.enums.EUpdateMode; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.formats.csv.CsvRowDeserializationSchema; +import org.apache.flink.formats.csv.CsvRowSchemaConverter; +import org.apache.flink.formats.csv.CsvRowSerializationSchema; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectWriter; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Arrays; +import java.util.Objects; + +/** + * Serialization schema that serializes an object of Flink types into a CSV bytes. + * + *

Serializes the input row into a {@link ObjectNode} and + * converts it into byte[]. + * + *

Result byte[] messages can be deserialized using {@link CsvRowDeserializationSchema}. + */ +@PublicEvolving +public final class CsvCRowSerializationSchema implements SerializationSchema { + + private static final long serialVersionUID = 2098447220136965L; + + /** Type information describing the input CSV data. */ + private final RowTypeInfo typeInfo; + + /** Runtime instance that performs the actual work. */ + private final RuntimeConverter runtimeConverter; + + /** CsvMapper used to write {@link JsonNode} into bytes. */ + private final CsvMapper csvMapper; + + /** Schema describing the input CSV data. */ + private final CsvSchema csvSchema; + + /** Object writer used to write rows. It is configured by {@link CsvSchema}. */ + private final ObjectWriter objectWriter; + + /** Reusable object node. */ + private transient ObjectNode root; + + private String updateMode; + + private final String retractKey = "retract"; + + private CsvCRowSerializationSchema( + RowTypeInfo typeInfo, + CsvSchema csvSchema) { + this.typeInfo = typeInfo; + this.runtimeConverter = createRowRuntimeConverter(typeInfo, true); + this.csvMapper = new CsvMapper(); + this.csvSchema = csvSchema; + this.objectWriter = csvMapper.writer(csvSchema); + } + + /** + * A builder for creating a {@link CsvRowSerializationSchema}. + */ + @PublicEvolving + public static class Builder { + + private final RowTypeInfo typeInfo; + private CsvSchema csvSchema; + + /** + * Creates a {@link CsvRowSerializationSchema} expecting the given {@link TypeInformation}. + * + * @param typeInfo type information used to create schema. + */ + public Builder(TypeInformation typeInfo) { + Preconditions.checkNotNull(typeInfo, "Type information must not be null."); + + if (!(typeInfo instanceof CRowTypeInfo)) { + throw new IllegalArgumentException("Row type information expected."); + } + RowTypeInfo rowTypeInfo = ((CRowTypeInfo) typeInfo).rowType(); + this.typeInfo = rowTypeInfo; + this.csvSchema = CsvRowSchemaConverter.convert(rowTypeInfo); + } + + public Builder setFieldDelimiter(char c) { + this.csvSchema = this.csvSchema.rebuild().setColumnSeparator(c).build(); + return this; + } + + public Builder setLineDelimiter(String delimiter) { + Preconditions.checkNotNull(delimiter, "Delimiter must not be null."); + if (!delimiter.equals("\n") && !delimiter.equals("\r") && !delimiter.equals("\r\n")) { + throw new IllegalArgumentException( + "Unsupported new line delimiter. Only \\n, \\r, or \\r\\n are supported."); + } + this.csvSchema = this.csvSchema.rebuild().setLineSeparator(delimiter).build(); + return this; + } + + public Builder setArrayElementDelimiter(String delimiter) { + Preconditions.checkNotNull(delimiter, "Delimiter must not be null."); + this.csvSchema = this.csvSchema.rebuild().setArrayElementSeparator(delimiter).build(); + return this; + } + + public Builder setQuoteCharacter(char c) { + this.csvSchema = this.csvSchema.rebuild().setQuoteChar(c).build(); + return this; + } + + public Builder setEscapeCharacter(char c) { + this.csvSchema = this.csvSchema.rebuild().setEscapeChar(c).build(); + return this; + } + + public Builder setNullLiteral(String s) { + this.csvSchema = this.csvSchema.rebuild().setNullValue(s).build(); + return this; + } + + public CsvCRowSerializationSchema build() { + return new CsvCRowSerializationSchema( + typeInfo, + csvSchema); + } + } + + @Override + public byte[] serialize(CRow crow) { + Row row = crow.row(); + boolean change = crow.change(); + if (root == null) { + root = csvMapper.createObjectNode(); + } + try { + runtimeConverter.convert(csvMapper, root, row); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + root.put(retractKey, change); + } + return objectWriter.writeValueAsBytes(root); + } catch (Throwable t) { + throw new RuntimeException("Could not serialize row '" + row + "'.", t); + } + } + + @Override + public boolean equals(Object o) { + if (o == null || o.getClass() != this.getClass()) { + return false; + } + if (this == o) { + return true; + } + final CsvCRowSerializationSchema that = (CsvCRowSerializationSchema) o; + final CsvSchema otherSchema = that.csvSchema; + + return typeInfo.equals(that.typeInfo) && + csvSchema.getColumnSeparator() == otherSchema.getColumnSeparator() && + Arrays.equals(csvSchema.getLineSeparator(), otherSchema.getLineSeparator()) && + csvSchema.getArrayElementSeparator().equals(otherSchema.getArrayElementSeparator()) && + csvSchema.getQuoteChar() == otherSchema.getQuoteChar() && + csvSchema.getEscapeChar() == otherSchema.getEscapeChar() && + Arrays.equals(csvSchema.getNullValue(), otherSchema.getNullValue()); + } + + @Override + public int hashCode() { + return Objects.hash( + typeInfo, + csvSchema.getColumnSeparator(), + csvSchema.getLineSeparator(), + csvSchema.getArrayElementSeparator(), + csvSchema.getQuoteChar(), + csvSchema.getEscapeChar(), + csvSchema.getNullValue()); + } + + // -------------------------------------------------------------------------------------------- + + private interface RuntimeConverter extends Serializable { + JsonNode convert(CsvMapper csvMapper, ContainerNode container, Object obj); + } + + private static RuntimeConverter createRowRuntimeConverter(RowTypeInfo rowTypeInfo, boolean isTopLevel) { + final TypeInformation[] fieldTypes = rowTypeInfo.getFieldTypes(); + final String[] fieldNames = rowTypeInfo.getFieldNames(); + + final RuntimeConverter[] fieldConverters = createFieldRuntimeConverters(fieldTypes); + + return assembleRowRuntimeConverter(isTopLevel, fieldNames, fieldConverters); + } + + private static RuntimeConverter[] createFieldRuntimeConverters(TypeInformation[] fieldTypes) { + final RuntimeConverter[] fieldConverters = new RuntimeConverter[fieldTypes.length]; + for (int i = 0; i < fieldTypes.length; i++) { + fieldConverters[i] = createNullableRuntimeConverter(fieldTypes[i]); + } + return fieldConverters; + } + + private static RuntimeConverter assembleRowRuntimeConverter( + boolean isTopLevel, + String[] fieldNames, + RuntimeConverter[] fieldConverters) { + final int rowArity = fieldNames.length; + // top level reuses the object node container + if (isTopLevel) { + return (csvMapper, container, obj) -> { + final Row row = (Row) obj; + + validateArity(rowArity, row.getArity()); + + final ObjectNode objectNode = (ObjectNode) container; + for (int i = 0; i < rowArity; i++) { + objectNode.set( + fieldNames[i], + fieldConverters[i].convert(csvMapper, container, row.getField(i))); + } + return objectNode; + }; + } else { + return (csvMapper, container, obj) -> { + final Row row = (Row) obj; + + validateArity(rowArity, row.getArity()); + + final ArrayNode arrayNode = csvMapper.createArrayNode(); + for (int i = 0; i < rowArity; i++) { + arrayNode.add(fieldConverters[i].convert(csvMapper, arrayNode, row.getField(i))); + } + return arrayNode; + }; + } + } + + private static RuntimeConverter createNullableRuntimeConverter(TypeInformation info) { + final RuntimeConverter valueConverter = createRuntimeConverter(info); + return (csvMapper, container, obj) -> { + if (obj == null) { + return container.nullNode(); + } + return valueConverter.convert(csvMapper, container, obj); + }; + } + + private static RuntimeConverter createRuntimeConverter(TypeInformation info) { + if (info.equals(Types.VOID)) { + return (csvMapper, container, obj) -> container.nullNode(); + } else if (info.equals(Types.STRING)) { + return (csvMapper, container, obj) -> container.textNode((String) obj); + } else if (info.equals(Types.BOOLEAN)) { + return (csvMapper, container, obj) -> container.booleanNode((Boolean) obj); + } else if (info.equals(Types.BYTE)) { + return (csvMapper, container, obj) -> container.numberNode((Byte) obj); + } else if (info.equals(Types.SHORT)) { + return (csvMapper, container, obj) -> container.numberNode((Short) obj); + } else if (info.equals(Types.INT)) { + return (csvMapper, container, obj) -> container.numberNode((Integer) obj); + } else if (info.equals(Types.LONG)) { + return (csvMapper, container, obj) -> container.numberNode((Long) obj); + } else if (info.equals(Types.FLOAT)) { + return (csvMapper, container, obj) -> container.numberNode((Float) obj); + } else if (info.equals(Types.DOUBLE)) { + return (csvMapper, container, obj) -> container.numberNode((Double) obj); + } else if (info.equals(Types.BIG_DEC)) { + return (csvMapper, container, obj) -> container.numberNode((BigDecimal) obj); + } else if (info.equals(Types.BIG_INT)) { + return (csvMapper, container, obj) -> container.numberNode((BigInteger) obj); + } else if (info.equals(Types.SQL_DATE)) { + return (csvMapper, container, obj) -> container.textNode(obj.toString()); + } else if (info.equals(Types.SQL_TIME)) { + return (csvMapper, container, obj) -> container.textNode(obj.toString()); + } else if (info.equals(Types.SQL_TIMESTAMP)) { + return (csvMapper, container, obj) -> container.textNode(obj.toString()); + } else if (info instanceof RowTypeInfo){ + return createRowRuntimeConverter((RowTypeInfo) info, false); + } else if (info instanceof BasicArrayTypeInfo) { + return createObjectArrayRuntimeConverter(((BasicArrayTypeInfo) info).getComponentInfo()); + } else if (info instanceof ObjectArrayTypeInfo) { + return createObjectArrayRuntimeConverter(((ObjectArrayTypeInfo) info).getComponentInfo()); + } else if (info instanceof PrimitiveArrayTypeInfo && + ((PrimitiveArrayTypeInfo) info).getComponentType() == Types.BYTE) { + return createByteArrayRuntimeConverter(); + } + else { + throw new RuntimeException("Unsupported type information '" + info + "'."); + } + } + + private static RuntimeConverter createObjectArrayRuntimeConverter(TypeInformation elementType) { + final RuntimeConverter elementConverter = createNullableRuntimeConverter(elementType); + return (csvMapper, container, obj) -> { + final Object[] array = (Object[]) obj; + final ArrayNode arrayNode = csvMapper.createArrayNode(); + for (Object element : array) { + arrayNode.add(elementConverter.convert(csvMapper, arrayNode, element)); + } + return arrayNode; + }; + } + + private static RuntimeConverter createByteArrayRuntimeConverter() { + return (csvMapper, container, obj) -> container.binaryNode((byte[]) obj); + } + + private static void validateArity(int expected, int actual) { + if (expected != actual) { + throw new RuntimeException("Row length mismatch. " + expected + + " fields expected but was " + actual + "."); + } + } +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java new file mode 100644 index 000000000..bfe801d52 --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka.serialization; + +import com.dtstack.flink.sql.enums.EUpdateMode; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.formats.json.JsonRowSchemaConverter; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Time; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.util.Objects; + +/** + * + * Serialization schema that serializes an object of Flink types into a JSON bytes. + * + *

Serializes the input Flink object into a JSON string and + * converts it into byte[]. + * + */ +public class JsonCRowSerializationSchema implements SerializationSchema { + + private static final long serialVersionUID = -2885556750743978636L; + + /** Type information describing the input type. */ + private final TypeInformation typeInfo; + + /** Object mapper that is used to create output JSON objects. */ + private final ObjectMapper mapper = new ObjectMapper(); + + /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone, without milliseconds). */ + private SimpleDateFormat timeFormat = new SimpleDateFormat("HH:mm:ss'Z'"); + + /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone). */ + private SimpleDateFormat timeFormatWithMillis = new SimpleDateFormat("HH:mm:ss.SSS'Z'"); + + /** Formatter for RFC 3339-compliant string representation of a timestamp value (with UTC timezone). */ + private SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + + /** Reusable object node. */ + private transient ObjectNode node; + + private String updateMode; + + private final String retractKey = "retract"; + + public JsonCRowSerializationSchema(String jsonSchema, String updateMode) { + this(JsonRowSchemaConverter.convert(jsonSchema), updateMode); + } + + /** + * Creates a JSON serialization schema for the given type information. + * + * @param typeInfo The field names of {@link Row} are used to map to JSON properties. + */ + public JsonCRowSerializationSchema(TypeInformation typeInfo, String updateMode) { + Preconditions.checkNotNull(typeInfo, "Type information"); + this.typeInfo = typeInfo; + this.updateMode = updateMode; + } + + + @Override + public byte[] serialize(CRow crow) { + Row row = crow.row(); + boolean change = crow.change(); + if (node == null) { + node = mapper.createObjectNode(); + } + + RowTypeInfo rowTypeInfo = ((CRowTypeInfo) typeInfo).rowType(); + try { + convertRow(node, rowTypeInfo, row); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + node.put(retractKey, change); + } + return mapper.writeValueAsBytes(node); + } catch (Throwable t) { + throw new RuntimeException("Could not serialize row '" + row + "'. " + + "Make sure that the schema matches the input.", t); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final JsonCRowSerializationSchema that = (JsonCRowSerializationSchema) o; + return Objects.equals(typeInfo, that.typeInfo); + } + + @Override + public int hashCode() { + return Objects.hash(typeInfo); + } + + // -------------------------------------------------------------------------------------------- + + private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) { + if (reuse == null) { + reuse = mapper.createObjectNode(); + } + final String[] fieldNames = info.getFieldNames(); + + final TypeInformation[] fieldTypes = info.getFieldTypes(); + + // validate the row + if (row.getArity() != fieldNames.length) { + throw new IllegalStateException(String.format( + "Number of elements in the row '%s' is different from number of field names: %d", row, fieldNames.length)); + } + + for (int i = 0; i < fieldNames.length; i++) { + final String name = fieldNames[i]; + + final JsonNode fieldConverted = convert(reuse, reuse.get(name), fieldTypes[i], row.getField(i)); + reuse.set(name, fieldConverted); + } + + return reuse; + } + + private JsonNode convert(ContainerNode container, JsonNode reuse, TypeInformation info, Object object) { + if (info == Types.VOID || object == null) { + return container.nullNode(); + } else if (info == Types.BOOLEAN) { + return container.booleanNode((Boolean) object); + } else if (info == Types.STRING) { + return container.textNode((String) object); + } else if (info == Types.BIG_DEC) { + // convert decimal if necessary + if (object instanceof BigDecimal) { + return container.numberNode((BigDecimal) object); + } + return container.numberNode(BigDecimal.valueOf(((Number) object).doubleValue())); + } else if (info == Types.BIG_INT) { + // convert integer if necessary + if (object instanceof BigInteger) { + return container.numberNode((BigInteger) object); + } + return container.numberNode(BigInteger.valueOf(((Number) object).longValue())); + } else if (info == Types.SQL_DATE) { + return container.textNode(object.toString()); + } else if (info == Types.SQL_TIME) { + final Time time = (Time) object; + // strip milliseconds if possible + if (time.getTime() % 1000 > 0) { + return container.textNode(timeFormatWithMillis.format(time)); + } + return container.textNode(timeFormat.format(time)); + } else if (info == Types.SQL_TIMESTAMP) { + return container.textNode(timestampFormat.format((Timestamp) object)); + } else if (info instanceof RowTypeInfo) { + if (reuse != null && reuse instanceof ObjectNode) { + return convertRow((ObjectNode) reuse, (RowTypeInfo) info, (Row) object); + } else { + return convertRow(null, (RowTypeInfo) info, (Row) object); + } + } else if (info instanceof ObjectArrayTypeInfo) { + if (reuse != null && reuse instanceof ArrayNode) { + return convertObjectArray((ArrayNode) reuse, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } else { + return convertObjectArray(null, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } + } else if (info instanceof BasicArrayTypeInfo) { + if (reuse != null && reuse instanceof ArrayNode) { + return convertObjectArray((ArrayNode) reuse, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } else { + return convertObjectArray(null, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } + } else if (info instanceof PrimitiveArrayTypeInfo && ((PrimitiveArrayTypeInfo) info).getComponentType() == Types.BYTE) { + return container.binaryNode((byte[]) object); + } else { + // for types that were specified without JSON schema + // e.g. POJOs + try { + return mapper.valueToTree(object); + } catch (IllegalArgumentException e) { + throw new IllegalStateException("Unsupported type information '" + info + "' for object: " + object, e); + } + } + } + + private ArrayNode convertObjectArray(ArrayNode reuse, TypeInformation info, Object[] array) { + if (reuse == null) { + reuse = mapper.createArrayNode(); + } else { + reuse.removeAll(); + } + + for (Object object : array) { + reuse.add(convert(reuse, null, info, object)); + } + return reuse; + } +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index 7520de512..a42473420 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.kafka.table; +import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.format.FormatType; import com.dtstack.flink.sql.table.AbstractTableParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -51,6 +52,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map kafkaParam = new HashMap(); @@ -58,6 +60,8 @@ public class KafkaSinkTableInfo extends AbstractTargetTableInfo { private String partitionKeys; + private String updateMode; + public void addKafkaParam(String key, String value) { kafkaParam.put(key, value); } @@ -70,7 +74,6 @@ public Set getKafkaParamKeys() { return kafkaParam.keySet(); } - public String getBootstrapServers() { return bootstrapServers; } @@ -103,6 +106,14 @@ public void setFieldDelimiter(String fieldDelimiter) { this.fieldDelimiter = fieldDelimiter; } + public String getUpdateMode() { + return updateMode; + } + + public void setUpdateMode(String updateMode) { + this.updateMode = updateMode; + } + @Override public boolean check() { Preconditions.checkNotNull(getType(), "kafka of type is required"); diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java index afa950c5b..f08287cec 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java @@ -77,7 +77,7 @@ protected void beforeDeserialize() throws IOException { protected void registerPtMetric(AbstractFetcher fetcher) throws Exception { - Field consumerThreadField = fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); + Field consumerThreadField = getConsumerThreadField(fetcher); consumerThreadField.setAccessible(true); KafkaConsumerThread consumerThread = (KafkaConsumerThread) consumerThreadField.get(fetcher); @@ -115,6 +115,14 @@ public Long getValue() { } } + private Field getConsumerThreadField(AbstractFetcher fetcher) throws NoSuchFieldException { + try { + return fetcher.getClass().getDeclaredField("consumerThread"); + } catch (Exception e) { + return fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); + } + } + public void setFetcher(AbstractFetcher fetcher) { this.fetcher = fetcher; } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java index 71a9cc386..1cbbeafd9 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java @@ -23,7 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,7 @@ * * @author maqi */ -public class KafkaProducer extends FlinkKafkaProducer { +public class KafkaProducer extends FlinkKafkaProducer { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer.class); @@ -45,7 +45,7 @@ public class KafkaProducer extends FlinkKafkaProducer { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] parititonKeys) { + public KafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] parititonKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, parititonKeys), producerConfig, customPartitioner); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java index f3a2f40f5..6bf9014df 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java @@ -22,7 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import java.util.Optional; import java.util.Properties; @@ -36,7 +36,7 @@ public class KafkaProducerFactory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 7105bc037..71e938ba5 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -31,6 +31,8 @@ import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.table.utils.TableConnectorUtils; @@ -56,13 +58,15 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< protected Properties properties; - protected FlinkKafkaProducer flinkKafkaProducer; + protected FlinkKafkaProducer flinkKafkaProducer; + protected CRowTypeInfo typeInformation; + /** The schema of the table. */ private TableSchema schema; /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; + protected Optional> partitioner; private String[] partitionKeys; @@ -97,7 +101,9 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.parallelism = parallelism; } - this.flinkKafkaProducer = (FlinkKafkaProducer) new KafkaProducerFactory().createKafkaProducer(kafkaSinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); + typeInformation = new CRowTypeInfo(new RowTypeInfo(fieldTypes, fieldNames)); + this.flinkKafkaProducer = (FlinkKafkaProducer) new KafkaProducerFactory() + .createKafkaProducer(kafkaSinkTableInfo, typeInformation, properties, partitioner, partitionKeys); return this; } @@ -108,9 +114,9 @@ public TypeInformation getRecordType() { @Override public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) + DataStream mapDataStream = dataStream + .map((Tuple2 record) -> new CRow(record.f1, record.f0)) + .returns(typeInformation) .setParallelism(parallelism); mapDataStream.addSink(flinkKafkaProducer).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java index c815e134a..bee1865dd 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java @@ -23,6 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +38,7 @@ * * @author maqi */ -public class KafkaProducer09 extends FlinkKafkaProducer09 { +public class KafkaProducer09 extends FlinkKafkaProducer09 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer09.class); @@ -45,7 +46,7 @@ public class KafkaProducer09 extends FlinkKafkaProducer09 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer09(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner,String[] partitionKeys) { + public KafkaProducer09(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.orElse(null)); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java index 7fb3909ee..ee3423b07 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -36,7 +37,7 @@ public class KafkaProducer09Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner,String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer09(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 0a991a8ea..e6dbdf3d3 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -27,10 +27,13 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.table.utils.TableConnectorUtils; @@ -56,13 +59,14 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< protected Properties properties; - protected FlinkKafkaProducer09 kafkaProducer09; + protected FlinkKafkaProducer09 kafkaProducer09; + protected CRowTypeInfo typeInformation; /** The schema of the table. */ private TableSchema schema; /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; + protected Optional> partitioner; private String[] partitionKeys; @@ -101,8 +105,9 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.parallelism = parallelism; } - this.kafkaProducer09 = (FlinkKafkaProducer09) new KafkaProducer09Factory() - .createKafkaProducer(kafka09SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); + typeInformation = new CRowTypeInfo(new RowTypeInfo(fieldTypes, fieldNames)); + this.kafkaProducer09 = (FlinkKafkaProducer09) new KafkaProducer09Factory() + .createKafkaProducer(kafka09SinkTableInfo, typeInformation, properties, partitioner, partitionKeys); return this; } @@ -113,10 +118,10 @@ public TypeInformation getRecordType() { @Override public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) - .setParallelism(parallelism); + DataStream mapDataStream = dataStream + .map((Tuple2 record) -> new CRow(record.f1, record.f0)) + .returns(typeInformation) + .setParallelism(parallelism); mapDataStream.addSink(kafkaProducer09) .name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java index 3cdc45dec..3936575ef 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java @@ -23,7 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,7 @@ * * @author maqi */ -public class KafkaProducer010 extends FlinkKafkaProducer010 { +public class KafkaProducer010 extends FlinkKafkaProducer010 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer010.class); @@ -45,7 +45,7 @@ public class KafkaProducer010 extends FlinkKafkaProducer010 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.get()); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java index c44a9fe86..e0e023586 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -36,7 +37,7 @@ public class KafkaProducer010Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer010(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 34ea8fc5f..ac5a11810 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -29,6 +29,8 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.table.utils.TableConnectorUtils; @@ -61,6 +63,9 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< protected KafkaSinkTableInfo kafka10SinkTableInfo; + protected RichSinkFunction kafkaProducer010; + protected CRowTypeInfo typeInformation; + /** The schema of the table. */ private TableSchema schema; @@ -97,6 +102,12 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { if (parallelism != null) { this.parallelism = parallelism; } + + typeInformation = new CRowTypeInfo(new RowTypeInfo(fieldTypes, fieldNames)); + kafkaProducer010 = new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, + typeInformation, properties, + Optional.of(new CustomerFlinkPartition<>()), partitionKeys); + return this; } @@ -107,13 +118,9 @@ public TypeInformation getRecordType() { @Override public void emitDataStream(DataStream> dataStream) { - - RichSinkFunction kafkaProducer010 = new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, - Optional.of(new CustomerFlinkPartition<>()), partitionKeys); - - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) + DataStream mapDataStream = dataStream + .map((Tuple2 record) -> new CRow(record.f1, record.f0)) + .returns(typeInformation) .setParallelism(parallelism); mapDataStream.addSink(kafkaProducer010).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java index 7880dd419..429d21a79 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java @@ -24,7 +24,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +38,7 @@ * * @author maqi */ -public class KafkaProducer011 extends FlinkKafkaProducer011 { +public class KafkaProducer011 extends FlinkKafkaProducer011 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer011.class); @@ -46,7 +46,7 @@ public class KafkaProducer011 extends FlinkKafkaProducer011 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer011(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + public KafkaProducer011(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java index e2272b16e..0cb11da82 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -36,7 +37,7 @@ public class KafkaProducer011Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer011(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index d7807a935..835941ca3 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -31,6 +31,8 @@ import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.table.utils.TableConnectorUtils; @@ -61,13 +63,15 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener protected Properties properties; - protected FlinkKafkaProducer011 kafkaProducer011; + protected FlinkKafkaProducer011 kafkaProducer011; + protected CRowTypeInfo typeInformation; + /** The schema of the table. */ private TableSchema schema; /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; + protected Optional> partitioner; private String[] partitionKeys; @@ -102,8 +106,9 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.parallelism = parallelism; } - this.kafkaProducer011 = (FlinkKafkaProducer011) new KafkaProducer011Factory() - .createKafkaProducer(kafka11SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); + typeInformation = new CRowTypeInfo(new RowTypeInfo(fieldTypes, fieldNames)); + this.kafkaProducer011 = (FlinkKafkaProducer011) new KafkaProducer011Factory() + .createKafkaProducer(kafka11SinkTableInfo, typeInformation, properties, partitioner, partitionKeys); return this; } @@ -114,9 +119,10 @@ public TypeInformation getRecordType() { @Override public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) + + DataStream mapDataStream = dataStream + .map((Tuple2 record) -> new CRow(record.f1, record.f0)) + .returns(typeInformation) .setParallelism(parallelism); mapDataStream.addSink(kafkaProducer011).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java index 3f804fc6c..c20f0678b 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java @@ -31,8 +31,8 @@ /** * company: www.dtstack.com - * author: toutian * create: 2019/12/24 + * @author: toutian */ public class KafkaConsumer011Factory extends AbstractKafkaConsumerFactory { diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 11be1898a..2f760bdf2 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -93,11 +93,11 @@ public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExec //earliest,latest if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} + } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) { try { Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); Map offsetMap = PluginUtil.objectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); + Map specificStartupOffsets = new HashMap<>(16); for (Map.Entry entry : offsetMap.entrySet()) { specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); } From 674d2d94a6e9cb5acdc014d6c7b5ad4fcaacef79 Mon Sep 17 00:00:00 2001 From: todd5167 <313328862@qq.com> Date: Tue, 17 Mar 2020 20:53:56 +0800 Subject: [PATCH 151/523] Update README.md --- README.md | 34 ++++++---------------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index c32a83f6f..491888c90 100644 --- a/README.md +++ b/README.md @@ -6,34 +6,17 @@ > > * 实现了流与维表的join > > * 支持原生FLinkSQL所有的语法 > > * 扩展了输入和输出的性能指标到promethus - - ## 新特性: - * 1.kafka源表支持not null语法,支持字符串类型的时间转换。 - * 2.rdb维表与DB建立连接时，周期进行连接，防止连接断开。rdbsink写入时，对连接进行检查。 - * 3.异步维表支持非等值连接，比如：<>,<,>。 - * 4.增加kafka数组解析 - * 5.增加kafka1.0以上版本的支持 - * 6.增加postgresql、kudu、clickhouse维表、结果表的支持 - * 7.支持插件的依赖方式,参考pluginLoadMode参数 - * 8.支持cep处理 - * 9.支持udaf - * 10.支持谓词下移 - * 11.支持状态的ttl - - ## BUG修复： - * 1.修复不能解析sql中orderby,union语法。 - * 2.修复yarnPer模式提交失败的异常。 - * 3.一些bug的修复 # 已支持 * 源表：kafka 0.9、0.10、0.11、1.x版本 * 维表：mysql, SQlServer,oracle, hbase, mongo, redis, cassandra, serversocket, kudu, postgresql, clickhouse, impala, db2, sqlserver * 结果表：mysql, SQlServer, oracle, hbase, elasticsearch5.x, mongo, redis, cassandra, console, kudu, postgresql, clickhouse, impala, db2, sqlserver - -# 后续开发计划 - * 维表快照 - * kafka avro格式 - * topN + +# FlinkX & FlinkStreamSQL钉钉交流群 + +

+ +
## 1 快速起步 ### 1.1 运行模式 @@ -326,8 +309,3 @@ into # 招聘 1.大数据平台开发工程师，想了解岗位详细信息可以添加本人微信号ysqwhiletrue,注明招聘，如有意者发送简历至sishu@dtstack.com。 -# FlinkX & FlinkStreamSQL钉钉交流群 - -
- -
From f683c37c78758926225333c451adf943bc6a3e70 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 18 Mar 2020 15:56:31 +0800 Subject: [PATCH 152/523] =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E5=BC=82=E6=AD=A5cac?= =?UTF-8?q?he=E7=9A=84=E6=8A=BD=E8=B1=A1=E5=92=8C=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../side/cassandra/CassandraAsyncReqRow.java | 87 ++++--------- .../flink/sql/side/AbstractSideTableInfo.java | 13 ++ .../flink/sql/side/BaseAsyncReqRow.java | 114 +++++++++++++++++- .../sql/table/AbstractSideTableParser.java | 7 ++ .../Elasticsearch6AsyncReqRow.java | 63 +++------- .../sql/side/hbase/HbaseAsyncReqRow.java | 50 +------- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 77 +++--------- .../sql/side/mongo/MongoAsyncReqRow.java | 56 +++------ 8 files changed, 210 insertions(+), 257 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index e1ed6d8e1..319e82ff3 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -162,61 +162,13 @@ private void connCassandraDB(CassandraSideTableInfo tableInfo) { } @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { - CRow inputCopy = new CRow(input.row(), input.change()); - JsonArray inputParams = new JsonArray(); - StringBuffer stringBuffer = new StringBuffer(); - String sqlWhere = " where "; - - for (int i = 0; i < sideInfo.getEqualFieldList().size(); i++) { - Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = inputCopy.row().getField(conValIndex); - if (equalObj == null) { - dealMissKey(inputCopy, resultFuture); - return; - } - inputParams.add(equalObj); - StringBuffer sqlTemp = stringBuffer.append(sideInfo.getEqualFieldList().get(i)) - .append(" = "); - if (equalObj instanceof String) { - sqlTemp.append("'" + equalObj + "'") - .append(" and "); - } else { - sqlTemp.append(equalObj) - .append(" and "); - } - - } + public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { String key = buildCacheKey(inputParams); - sqlWhere = sqlWhere + stringBuffer.toString().substring(0, stringBuffer.lastIndexOf(" and ")); - - if (openCache()) { - CacheObj val = getFromCache(key); - if (val != null) { - - if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(inputCopy, resultFuture); - return; - } else if (ECacheContentType.MultiLine == val.getType()) { - List rowList = Lists.newArrayList(); - for (Object jsonArray : (List) val.getContent()) { - Row row = fillData(inputCopy.row(), jsonArray); - rowList.add(new CRow(row, inputCopy.change())); - } - resultFuture.complete(rowList); - } else { - throw new RuntimeException("not support cache obj type " + val.getType()); - } - return; - } - } - //connect Cassandra connCassandraDB(cassandraSideTableInfo); - String sqlCondition = sideInfo.getSqlCondition() + " " + sqlWhere + " ALLOW FILTERING "; - System.out.println("sqlCondition:" + sqlCondition); + String sqlCondition = sideInfo.getSqlCondition() + " " + buildWhereCondition(inputParams) + " ALLOW FILTERING "; ListenableFuture resultSet = Futures.transformAsync(session, new AsyncFunction() { @@ -242,18 +194,18 @@ public void onSuccess(List rows) { List cacheContent = Lists.newArrayList(); List rowList = Lists.newArrayList(); for (com.datastax.driver.core.Row line : rows) { - Row row = fillData(inputCopy.row(), line); + Row row = fillData(input.row(), line); if (openCache()) { cacheContent.add(line); } - rowList.add(new CRow(row,inputCopy.change())); + rowList.add(new CRow(row, input.change())); } resultFuture.complete(rowList); if (openCache()) { putCache(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); } } else { - dealMissKey(inputCopy, resultFuture); + dealMissKey(input, resultFuture); if (openCache()) { putCache(key, CacheMissVal.getMissKeyObj()); } @@ -265,13 +217,30 @@ public void onSuccess(List rows) { public void onFailure(Throwable t) { LOG.error("Failed to retrieve the data: %s%n", t.getMessage()); - System.out.println("Failed to retrieve the data: " + t.getMessage()); cluster.closeAsync(); resultFuture.completeExceptionally(t); } }); } + @Override + public String buildCacheKey(Map inputParams) { + StringBuilder sb = new StringBuilder(); + for (Object ele : inputParams.values()) { + sb.append(ele.toString()).append("_"); + } + return sb.toString(); + } + + private String buildWhereCondition(Map inputParams){ + StringBuilder sb = new StringBuilder(" where "); + for(Map.Entry entry : inputParams.entrySet()){ + Object value = entry.getValue() instanceof String ? "'" + entry.getValue() + "'" : entry.getValue(); + sb.append(String.format("%s = %s", entry.getKey(), value)); + } + return sb.toString(); + } + @Override public Row fillData(Row input, Object line) { com.datastax.driver.core.Row rowArray = (com.datastax.driver.core.Row) line; @@ -306,14 +275,4 @@ public void close() throws Exception { cluster = null; } } - - public String buildCacheKey(JsonArray jsonArray) { - StringBuilder sb = new StringBuilder(); - for (Object ele : jsonArray.getList()) { - sb.append(ele.toString()) - .append("_"); - } - - return sb.toString(); - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index cacba74fd..0a6babc8a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -27,6 +27,7 @@ import java.io.Serializable; import java.util.List; +import java.util.Objects; /** * Reason: @@ -55,6 +56,8 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements public static final String ASYNC_TIMEOUT_NUM_KEY = "asyncTimeoutNum"; + public static final String ASYNC_FAIL_MAX_NUM_KEY = "asyncFailMaxNum"; + private String cacheType = "none";//None or LRU or ALL private int cacheSize = 10000; @@ -71,6 +74,8 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements private String cacheMode="ordered"; + private Integer asyncFailMaxNum; + private List predicateInfoes = Lists.newArrayList(); public RowTypeInfo getRowTypeInfo(){ @@ -156,4 +161,12 @@ public void setAsyncTimeoutNumLimit(int asyncTimeoutNumLimit) { this.asyncTimeoutNumLimit = asyncTimeoutNumLimit; } + public Integer getAsyncFailMaxNum(Integer defaultValue) { + return Objects.isNull(asyncFailMaxNum) ? defaultValue : asyncFailMaxNum; + } + + public void setAsyncFailMaxNum(Integer asyncFailMaxNum) { + this.asyncFailMaxNum = asyncFailMaxNum; + } + } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index a0deb6b74..ae987e142 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -20,24 +20,36 @@ package com.dtstack.flink.sql.side; +import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.metric.MetricConstant; import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cache.LRUSideCache; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; +import org.apache.commons.collections.MapUtils; import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; +import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry; +import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback; +import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collections; -import java.util.concurrent.TimeoutException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * All interfaces inherit naming rules: type + "AsyncReqRow" such as == "MysqlAsyncReqRow @@ -126,7 +138,6 @@ public void timeout(CRow input, ResultFuture resultFuture) throws Exceptio if(timeOutNum % TIMEOUT_LOG_FLUSH_NUM == 0){ LOG.info("Async function call has timed out. input:{}, timeOutNum:{}",input.toString(), timeOutNum); } - timeOutNum ++; if(timeOutNum > sideInfo.getSideTableInfo().getAsyncTimeoutNumLimit()){ resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); @@ -135,6 +146,105 @@ public void timeout(CRow input, ResultFuture resultFuture) throws Exceptio } } + protected void preInvoke(CRow input, ResultFuture resultFuture){ + ScheduledFuture timeFuture = registerTimer(input, resultFuture); + cancelTimerWhenComplete(resultFuture, timeFuture); + } + + @Override + public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + preInvoke(input, resultFuture); + Map inputParams = parseInputParam(input); + if(MapUtils.isEmpty(inputParams)){ + dealMissKey(input, resultFuture); + return; + } + if(isUseCache(inputParams)){ + invokeWithCache(inputParams, input, resultFuture); + return; + } + handleAsyncInvoke(inputParams, input, resultFuture); + } + + private Map parseInputParam(CRow input){ + Map inputParams = Maps.newHashMap(); + for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { + Integer conValIndex = sideInfo.getEqualValIndex().get(i); + Object equalObj = input.row().getField(conValIndex); + if(equalObj == null){ + return inputParams; + } + String columnName = sideInfo.getEqualFieldList().get(conValIndex); + inputParams.put(columnName, equalObj.toString()); + } + return inputParams; + } + protected boolean isUseCache(Map inputParams){ + return openCache() && getFromCache(buildCacheKey(inputParams)) != null; + } + + private void invokeWithCache(Map inputParams, CRow input, ResultFuture resultFuture){ + if (openCache()) { + CacheObj val = getFromCache(buildCacheKey(inputParams)); + if (val != null) { + if (ECacheContentType.MissVal == val.getType()) { + dealMissKey(input, resultFuture); + return; + }else if(ECacheContentType.SingleLine == val.getType()){ + try { + Row row = fillData(input.row(), val); + resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); + } catch (Exception e) { + dealFillDataError(resultFuture, e, input); + } + } else if (ECacheContentType.MultiLine == val.getType()) { + try { + List rowList = Lists.newArrayList(); + for (Object one : (List) val.getContent()) { + Row row = fillData(input.row(), one); + rowList.add(new CRow(row, input.change())); + } + resultFuture.complete(rowList); + } catch (Exception e) { + dealFillDataError(resultFuture, e, input); + } + } else { + resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); + } + return; + } + } + } + + public abstract void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception; + + public abstract String buildCacheKey(Map inputParams); + + private ProcessingTimeService getProcessingTimeService(){ + return ((StreamingRuntimeContext)this.getRuntimeContext()).getProcessingTimeService(); + } + + protected ScheduledFuture registerTimer(CRow input, ResultFuture resultFuture){ + long timeoutTimestamp = sideInfo.getSideTableInfo().getAsyncTimeout() + getProcessingTimeService().getCurrentProcessingTime(); + return getProcessingTimeService().registerTimer( + timeoutTimestamp, + new ProcessingTimeCallback() { + @Override + public void onProcessingTime(long timestamp) throws Exception { + timeout(input, resultFuture); + } + }); + } + + protected void cancelTimerWhenComplete(ResultFuture resultFuture, ScheduledFuture timerFuture){ + ThreadPoolExecutor executors = new ThreadPoolExecutor(1, 1,0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue()); + if(resultFuture instanceof StreamRecordQueueEntry){ + StreamRecordQueueEntry streamRecordBufferEntry = (StreamRecordQueueEntry) resultFuture; + streamRecordBufferEntry.onComplete((Object value) -> { + timerFuture.cancel(true); + },executors); + } + } protected void dealFillDataError(ResultFuture resultFuture, Exception e, Object sourceData) { LOG.debug("source data {} join side table error ", sourceData); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 39b72c22f..83a06d256 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -114,6 +114,13 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0){ + sideTableInfo.setAsyncFailMaxNum(asyncFailNum); + } + } } } } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index e8164edb2..f8eb66860 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -81,39 +81,8 @@ public void open(Configuration parameters) throws Exception { @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { - CRow copyCrow = new CRow(input.row(), input.change()); - List inputParams = Lists.newArrayList(); - for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = copyCrow.row().getField(conValIndex); - if (equalObj == null) { - dealMissKey(copyCrow, resultFuture); - return; - } - inputParams.add(equalObj); - } - + public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { String key = buildCacheKey(inputParams); - if (openCache()) { - CacheObj val = getFromCache(key); - if (val != null) { - if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(copyCrow, resultFuture); - return; - } else if (ECacheContentType.MultiLine == val.getType()) { - try { - List rowList = getRows(copyCrow, null, (List) val.getContent()); - resultFuture.complete(rowList); - } catch (Exception e) { - dealFillDataError(resultFuture, e, copyCrow); - } - } else { - resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); - } - return; - } - } - BoolQueryBuilder boolQueryBuilder = Es6Util.setPredicateclause(sideInfo); boolQueryBuilder = setInputParams(inputParams, boolQueryBuilder); SearchSourceBuilder searchSourceBuilder = initConfiguration(); @@ -135,7 +104,7 @@ public void onResponse(SearchResponse searchResponse) { RestHighLevelClient tmpRhlClient = null; try { while (true) { - loadDataToCache(searchHits, rowList, cacheContent, copyCrow); + loadDataToCache(searchHits, rowList, cacheContent, input); // determine if all results haven been ferched if (searchHits.length < getFetchSize()) { break; @@ -154,7 +123,7 @@ public void onResponse(SearchResponse searchResponse) { dealCacheData(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); resultFuture.complete(rowList); } catch (Exception e) { - dealFillDataError(resultFuture, e, copyCrow); + dealFillDataError(resultFuture, e, input); } finally { if (tmpRhlClient != null) { try { @@ -165,7 +134,7 @@ public void onResponse(SearchResponse searchResponse) { } } } else { - dealMissKey(copyCrow, resultFuture); + dealMissKey(input, resultFuture); dealCacheData(key, CacheMissVal.getMissKeyObj()); } } @@ -177,7 +146,17 @@ public void onFailure(Exception e) { resultFuture.completeExceptionally(new RuntimeException("Response failed!")); } }); + } + + @Override + public String buildCacheKey(Map inputParams) { + StringBuilder sb = new StringBuilder(); + for (Object ele : inputParams.values()) { + sb.append(ele.toString()) + .append("_"); + } + return sb.toString(); } private void loadDataToCache(SearchHit[] searchHits, List rowList, List cacheContent, CRow copyCrow) { @@ -238,16 +217,6 @@ public void close() throws Exception { } - public String buildCacheKey(List equalValList) { - StringBuilder sb = new StringBuilder(); - for (Object ele : equalValList) { - sb.append(ele.toString()) - .append("_"); - } - - return sb.toString(); - } - private SearchSourceBuilder initConfiguration() { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.size(getFetchSize()); @@ -258,7 +227,7 @@ private SearchSourceBuilder initConfiguration() { return searchSourceBuilder; } - private BoolQueryBuilder setInputParams(List inputParams, BoolQueryBuilder boolQueryBuilder) { + private BoolQueryBuilder setInputParams(Map inputParams, BoolQueryBuilder boolQueryBuilder) { if (boolQueryBuilder == null) { boolQueryBuilder = new BoolQueryBuilder(); } @@ -266,7 +235,7 @@ private BoolQueryBuilder setInputParams(List inputParams, BoolQueryBuild for (int i = 0; i < sqlJoinCompareOperate.size(); i++) { String fieldName = sideInfo.getEqualFieldList().get(i); String operatorKind = sqlJoinCompareOperate.get(sideInfo.getEqualFieldList().indexOf(fieldName)); - String condition = String.valueOf(inputParams.get(i)); + String condition = String.valueOf(inputParams.get(fieldName)); boolQueryBuilder = Es6Util.buildFilterCondition(boolQueryBuilder, new PredicateInfo(null, operatorKind, null, fieldName, condition), sideInfo); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 24ee01786..007de8e15 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -123,55 +123,17 @@ public void open(Configuration parameters) throws Exception { } @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { - CRow inputCopy = new CRow(input.row(), input.change()); - Map refData = Maps.newHashMap(); - for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { - Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = inputCopy.row().getField(conValIndex); - if(equalObj == null){ - dealMissKey(inputCopy, resultFuture); - return; - } - refData.put(sideInfo.getEqualFieldList().get(i), equalObj); - } - - String rowKeyStr = ((HbaseAsyncSideInfo)sideInfo).getRowKeyBuilder().getRowKey(refData); - - //get from cache - if (openCache()) { - CacheObj val = getFromCache(rowKeyStr); - if (val != null) { - if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(inputCopy, resultFuture); - return; - } else if (ECacheContentType.SingleLine == val.getType()) { - try { - Row row = fillData(inputCopy.row(), val); - resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); - } catch (Exception e) { - dealFillDataError(resultFuture, e, inputCopy); - } - } else if (ECacheContentType.MultiLine == val.getType()) { - try { - for (Object one : (List) val.getContent()) { - Row row = fillData(inputCopy.row(), one); - resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); - } - } catch (Exception e) { - dealFillDataError(resultFuture, e, inputCopy); - } - } - return; - } - } + public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { + rowKeyMode.asyncGetData(tableName, buildCacheKey(inputParams), input, resultFuture, sideInfo.getSideCache()); + } - rowKeyMode.asyncGetData(tableName, rowKeyStr, inputCopy, resultFuture, sideInfo.getSideCache()); + @Override + public String buildCacheKey(Map inputParams) { + return ((HbaseAsyncSideInfo)sideInfo).getRowKeyBuilder().getRowKey(inputParams); } @Override public Row fillData(Row input, Object sideInput){ - List sideInputList = (List) sideInput; Row row = new Row(sideInfo.getOutFieldInfoList().size()); for(Map.Entry entry : sideInfo.getInFieldIndex().entrySet()){ diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index e534a85f9..e3200c293 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -128,26 +128,18 @@ private void connKuDu() throws KuduException { scannerBuilder.setProjectedColumnNames(projectColumns); } - @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { CRow inputCopy = new CRow(input.row(), input.change()); - //scannerBuilder 设置为null重新加载过滤条件 + //scannerBuilder 设置为null重新加载过滤条件,然后connkudu重新赋值 + //todo:代码需要优化 scannerBuilder = null; connKuDu(); - JsonArray inputParams = new JsonArray(); Schema schema = table.getSchema(); // @wenbaoup fix bug - for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { - Object equalObj = inputCopy.row().getField(sideInfo.getEqualValIndex().get(i)); - if (equalObj == null) { - dealMissKey(inputCopy, resultFuture); - return; - } - //增加过滤条件 - scannerBuilder.addPredicate(KuduPredicate.newInListPredicate(schema.getColumn(sideInfo.getEqualFieldList().get(i)), Collections.singletonList(equalObj))); - inputParams.add(equalObj); - } + inputParams.entrySet().forEach(e ->{ + scannerBuilder.addPredicate(KuduPredicate.newInListPredicate(schema.getColumn(e.getKey()), Collections.singletonList(e.getValue()))); + }); // 填充谓词信息 List predicateInfoes = sideInfo.getSideTableInfo().getPredicateInfoes(); @@ -161,46 +153,23 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce }).count(); } - - String key = buildCacheKey(inputParams); - - if (openCache()) { - //判断数据是否已经加载到缓存中 - CacheObj val = getFromCache(key); - if (val != null) { - if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(inputCopy, resultFuture); - return; - } else if (ECacheContentType.SingleLine == val.getType()) { - try { - Row row = fillData(inputCopy.row(), val); - resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); - } catch (Exception e) { - dealFillDataError(resultFuture, e, inputCopy); - } - } else if (ECacheContentType.MultiLine == val.getType()) { - try { - List rowList = Lists.newArrayList(); - for (Object jsonArray : (List) val.getContent()) { - Row row = fillData(inputCopy.row(), jsonArray); - rowList.add(new CRow(row, inputCopy.change())); - } - resultFuture.complete(rowList); - } catch (Exception e) { - dealFillDataError(resultFuture, e, inputCopy); - } - } else { - resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); - } - return; - } - } List> cacheContent = Lists.newArrayList(); AsyncKuduScanner asyncKuduScanner = scannerBuilder.build(); List rowList = Lists.newArrayList(); Deferred data = asyncKuduScanner.nextRows(); //从之前的同步修改为调用异步的Callback - data.addCallbackDeferring(new GetListRowCB(inputCopy, cacheContent, rowList, asyncKuduScanner, resultFuture, key)); + data.addCallbackDeferring(new GetListRowCB(inputCopy, cacheContent, rowList, asyncKuduScanner, resultFuture, buildCacheKey(inputParams))); + } + + @Override + public String buildCacheKey(Map inputParams) { + StringBuilder sb = new StringBuilder(); + for (Object ele : inputParams.values()) { + sb.append(ele.toString()) + .append("_"); + } + + return sb.toString(); } @@ -230,16 +199,6 @@ public Row fillData(Row input, Object sideInput) { return row; } - public String buildCacheKey(JsonArray jsonArray) { - StringBuilder sb = new StringBuilder(); - for (Object ele : jsonArray.getList()) { - sb.append(ele.toString()) - .append("_"); - } - - return sb.toString(); - } - @Override public void close() throws Exception { super.close(); diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index d85a9990d..99cc3b2db 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -102,19 +102,11 @@ public void connMongoDb() throws Exception { } @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { CRow inputCopy = new CRow(input.row(), input.change()); BasicDBObject basicDbObject = new BasicDBObject(); - for (int i = 0; i < sideInfo.getEqualFieldList().size(); i++) { - Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = inputCopy.row().getField(conValIndex); - if (equalObj == null) { - dealMissKey(inputCopy, resultFuture); - return; - } - basicDbObject.put(sideInfo.getEqualFieldList().get(i), equalObj); - } try { + basicDbObject.putAll(inputParams); // 填充谓词 sideInfo.getSideTableInfo().getPredicateInfoes().stream().map(info -> { BasicDBObject filterCondition = MongoUtil.buildFilterObject(info); @@ -127,27 +119,8 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce LOG.info("add predicate infoes error ", e); } - String key = buildCacheKey(basicDbObject.values()); - if (openCache()) { - CacheObj val = getFromCache(key); - if (val != null) { - - if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(inputCopy, resultFuture); - return; - } else if (ECacheContentType.MultiLine == val.getType()) { - List rowList = Lists.newArrayList(); - for (Object jsonArray : (List) val.getContent()) { - Row row = fillData(inputCopy.row(), jsonArray); - rowList.add(new CRow(row, inputCopy.change())); - } - resultFuture.complete(rowList); - } else { - throw new RuntimeException("not support cache obj type " + val.getType()); - } - return; - } - } + String key = buildCacheKey(inputParams); + AtomicInteger atomicInteger = new AtomicInteger(0); MongoCollection dbCollection = db.getCollection(mongoSideTableInfo.getTableName(), Document.class); List cacheContent = Lists.newArrayList(); @@ -178,6 +151,17 @@ public void onResult(final Void result, final Throwable t) { dbCollection.find(basicDbObject).forEach(printDocumentBlock, callbackWhenFinished); } + @Override + public String buildCacheKey(Map inputParams) { + StringBuilder sb = new StringBuilder(); + for (Object ele : inputParams.values()) { + sb.append(ele.toString()) + .append("_"); + } + + return sb.toString(); + } + @Override public Row fillData(Row input, Object line) { Document doc = (Document) line; @@ -216,14 +200,4 @@ public void close() throws Exception { } } - public String buildCacheKey(Collection collection) { - StringBuilder sb = new StringBuilder(); - for (Object ele : collection) { - sb.append(ele.toString()) - .append("_"); - } - - return sb.toString(); - } - } From 118d0fd5feef1f97a26be1e271f55f9b6ee9c59a Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 18 Mar 2020 15:57:04 +0800 Subject: [PATCH 153/523] =?UTF-8?q?=E8=A1=A5=E5=85=85=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/operator/SideAsyncOperator.java | 4 +- .../sql/side/rdb/async/RdbAsyncReqRow.java | 164 +++++++++--------- .../sql/side/redis/RedisAsyncReqRow.java | 55 ++---- 3 files changed, 90 insertions(+), 133 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java index 3c2010907..b23de8102 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java @@ -66,10 +66,10 @@ public static DataStream getSideJoinDataStream(DataStream inputStream, String si //TODO How much should be set for the degree of parallelism? Timeout? capacity settings? if (ORDERED.equals(sideTableInfo.getCacheMode())){ - return AsyncDataStream.orderedWait(inputStream, asyncDbReq, sideTableInfo.getAsyncTimeout(), TimeUnit.MILLISECONDS, sideTableInfo.getAsyncCapacity()) + return AsyncDataStream.orderedWait(inputStream, asyncDbReq, -1, TimeUnit.MILLISECONDS, sideTableInfo.getAsyncCapacity()) .setParallelism(sideTableInfo.getParallelism()); }else { - return AsyncDataStream.unorderedWait(inputStream, asyncDbReq, sideTableInfo.getAsyncTimeout(), TimeUnit.MILLISECONDS, sideTableInfo.getAsyncCapacity()) + return AsyncDataStream.unorderedWait(inputStream, asyncDbReq, -1, TimeUnit.MILLISECONDS, sideTableInfo.getAsyncCapacity()) .setParallelism(sideTableInfo.getParallelism()); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index f78c30c30..5e5de7e38 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -25,10 +25,12 @@ import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; +import io.vertx.core.json.Json; import io.vertx.core.json.JsonArray; import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -37,8 +39,12 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; +import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; /** * Date: 2018/11/26 @@ -69,94 +75,45 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private transient SQLClient rdbSqlClient; + private Logger logger = LoggerFactory.getLogger(getClass()); + public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); } - @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { - CRow copyCrow = new CRow(input.row(), input.change()); - JsonArray inputParams = new JsonArray(); - for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = copyCrow.row().getField(conValIndex); - if (equalObj == null) { - dealMissKey(copyCrow, resultFuture); - return; - } - inputParams.add(equalObj); - } - String key = buildCacheKey(inputParams); - if (openCache()) { - CacheObj val = getFromCache(key); - if (val != null) { - if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(copyCrow, resultFuture); - return; - } else if (ECacheContentType.MultiLine == val.getType()) { - try { - List rowList = getRows(copyCrow, null, (List) val.getContent()); - resultFuture.complete(rowList); - } catch (Exception e) { - dealFillDataError(resultFuture, e, copyCrow); - } - } else { - resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); - } - return; - } - } + @Override + protected void preInvoke(CRow input, ResultFuture resultFuture){ - rdbSqlClient.getConnection(conn -> { - if (conn.failed()) { - //Treatment failures - resultFuture.completeExceptionally(conn.cause()); - return; - } + } - final SQLConnection connection = conn.result(); - String sqlCondition = sideInfo.getSqlCondition(); - connection.queryWithParams(sqlCondition, inputParams, rs -> { - if (rs.failed()) { - LOG.error("Cannot retrieve the data from the database", rs.cause()); - resultFuture.completeExceptionally(rs.cause()); - return; - } - List cacheContent = Lists.newArrayList(); - List results = rs.result().getResults(); - if (results.size() > 0) { - try { - List rowList = getRows(copyCrow, cacheContent, results); - dealCacheData(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); - resultFuture.complete(rowList); - } catch (Exception e){ - dealFillDataError(resultFuture, e, copyCrow); + @Override + public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { + AtomicBoolean connectFinish = new AtomicBoolean(false); + AtomicInteger counter = new AtomicInteger(0); + while(!connectFinish.get()){ + rdbSqlClient.getConnection(conn -> { + if(conn.failed()){ + if(counter.get() % 1000 == 0){ + logger.error("getConnection error", conn.cause()); } - } else { - dealMissKey(copyCrow, resultFuture); - dealCacheData(key, CacheMissVal.getMissKeyObj()); - } - - // and close the connection - connection.close(done -> { - if (done.failed()) { - throw new RuntimeException(done.cause()); + if(counter.incrementAndGet() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ + resultFuture.completeExceptionally(conn.cause()); + connectFinish.set(true); + return; } - }); + } + connectFinish.set(true); + ScheduledFuture timerFuture = registerTimer(input, resultFuture); + cancelTimerWhenComplete(resultFuture, timerFuture); + handleQuery(conn.result(), inputParams, input, resultFuture); }); - }); + } } - protected List getRows(CRow inputRow, List cacheContent, List results) { - List rowList = Lists.newArrayList(); - for (JsonArray line : results) { - Row row = fillData(inputRow.row(), line); - if (null != cacheContent && openCache()) { - cacheContent.add(line); - } - rowList.add(new CRow(row, inputRow.change())); - } - return rowList; + @Override + public String buildCacheKey(Map inputParam) { + return StringUtils.join(inputParam.values(),"_"); } @Override @@ -195,18 +152,53 @@ public void close() throws Exception { } - public String buildCacheKey(JsonArray jsonArray) { - StringBuilder sb = new StringBuilder(); - for (Object ele : jsonArray.getList()) { - sb.append(ele.toString()) - .append("_"); - } - - return sb.toString(); - } - public void setRdbSqlClient(SQLClient rdbSqlClient) { this.rdbSqlClient = rdbSqlClient; } + private void handleQuery(SQLConnection connection,Map inputParams, CRow input, ResultFuture resultFuture){ + String key = buildCacheKey(inputParams); + JsonArray params = new JsonArray(Lists.newArrayList(inputParams.values())); + connection.queryWithParams(sideInfo.getSqlCondition(), params, rs -> { + if (rs.failed()) { + LOG.error("Cannot retrieve the data from the database", rs.cause()); + resultFuture.completeExceptionally(rs.cause()); + return; + } + + List cacheContent = Lists.newArrayList(); + + int resultSize = rs.result().getResults().size(); + if (resultSize > 0) { + List rowList = Lists.newArrayList(); + + for (JsonArray line : rs.result().getResults()) { + Row row = fillData(input.row(), line); + if (openCache()) { + cacheContent.add(line); + } + rowList.add(new CRow(row, input.change())); + } + + if (openCache()) { + putCache(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); + } + + resultFuture.complete(rowList); + } else { + dealMissKey(input, resultFuture); + if (openCache()) { + putCache(key, CacheMissVal.getMissKeyObj()); + } + } + + // and close the connection + connection.close(done -> { + if (done.failed()) { + throw new RuntimeException(done.cause()); + } + }); + }); + } + } diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index d2a2aed56..b6b6c5a9d 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -126,49 +126,13 @@ public Row fillData(Row input, Object sideInput) { } @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { - CRow inputCopy = new CRow(input.row(),input.change()); - List keyData = Lists.newLinkedList(); - for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { - Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = inputCopy.row().getField(conValIndex); - if(equalObj == null){ - dealMissKey(inputCopy, resultFuture); - return; - } - String value = equalObj.toString(); - keyData.add(sideInfo.getEqualFieldList().get(i)); - keyData.add(value); - } - - String key = buildCacheKey(keyData); - - if(openCache()){ - CacheObj val = getFromCache(key); - if(val != null){ - if(ECacheContentType.MissVal == val.getType()){ - dealMissKey(inputCopy, resultFuture); - return; - }else if(ECacheContentType.MultiLine == val.getType()){ - try { - Row row = fillData(inputCopy.row(), val.getContent()); - resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); - } catch (Exception e) { - dealFillDataError(resultFuture, e, inputCopy); - } - }else{ - RuntimeException exception = new RuntimeException("not support cache obj type " + val.getType()); - resultFuture.completeExceptionally(exception); - } - return; - } - } - + public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { + String key = buildCacheKey(inputParams); Map keyValue = Maps.newHashMap(); List value = async.keys(key + ":*").get(); String[] values = value.toArray(new String[value.size()]); if (values.length == 0) { - dealMissKey(inputCopy, resultFuture); + dealMissKey(input, resultFuture); } else { RedisFuture>> future = ((RedisStringAsyncCommands) async).mget(values); future.thenAccept(new Consumer>>() { @@ -181,14 +145,14 @@ public void accept(List> keyValues) { keyValue.put(splitKeys[3], keyValues.get(i).getValue()); } try { - Row row = fillData(inputCopy.row(), keyValue); + Row row = fillData(input.row(), keyValue); dealCacheData(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, keyValue)); - resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); + resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, inputCopy); + dealFillDataError(resultFuture, e, input); } } else { - dealMissKey(inputCopy, resultFuture); + dealMissKey(input, resultFuture); dealCacheData(key, CacheMissVal.getMissKeyObj()); } } @@ -196,8 +160,9 @@ public void accept(List> keyValues) { } } - private String buildCacheKey(List keyData) { - String kv = String.join(":", keyData); + @Override + public String buildCacheKey(Map inputParams) { + String kv = StringUtils.join(inputParams.values(), ":"); String tableName = redisSideTableInfo.getTableName(); StringBuilder preKey = new StringBuilder(); preKey.append(tableName).append(":").append(kv); From 1cb21db06ca2dce2bdc9cb0e53a984647151a692 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 18 Mar 2020 16:35:32 +0800 Subject: [PATCH 154/523] kafka avro and csv ,retract sink --- README.md | 24 +- docs/kafkaSink.md | 223 ++++++++++++++++++ docs/kafkaSource.md | 184 +-------------- .../kafka/AbstractKafkaProducerFactory.java | 5 +- .../AvroCRowSerializationSchema.java | 44 +++- .../CsvCRowSerializationSchema.java | 39 ++- .../sql/sink/kafka/table/KafkaSinkParser.java | 3 + .../sink/kafka/table/KafkaSinkTableInfo.java | 28 +++ 8 files changed, 324 insertions(+), 226 deletions(-) create mode 100644 docs/kafkaSink.md diff --git a/README.md b/README.md index ccf94f1ff..c84d77d5a 100644 --- a/README.md +++ b/README.md @@ -7,34 +7,11 @@ > > * 支持原生FLinkSQL所有的语法 > > * 扩展了输入和输出的性能指标到promethus - ## 新特性: - * 1.kafka源表支持not null语法,支持字符串类型的时间转换。 - * 2.rdb维表与DB建立连接时，周期进行连接，防止连接断开。rdbsink写入时，对连接进行检查。 - * 3.异步维表支持非等值连接，比如：<>,<,>。 - * 4.增加kafka数组解析 - * 5.增加kafka1.0以上版本的支持 - * 6.增加postgresql、kudu、clickhouse维表、结果表的支持 - * 7.支持插件的依赖方式,参考pluginLoadMode参数 - * 8.支持cep处理 - * 9.支持udaf - * 10.支持谓词下移 - * 11.支持状态的ttl - - ## BUG修复： - * 1.修复不能解析sql中orderby,union语法。 - * 2.修复yarnPer模式提交失败的异常。 - * 3.一些bug的修复 - # 已支持 * 源表：kafka 0.9、0.10、0.11、1.x版本 * 维表：mysql, SQlServer,oracle, hbase, mongo, redis, cassandra, serversocket, kudu, postgresql, clickhouse, impala, db2, sqlserver * 结果表：mysql, SQlServer, oracle, hbase, elasticsearch5.x, mongo, redis, cassandra, console, kudu, postgresql, clickhouse, impala, db2, sqlserver -# 后续开发计划 - * 维表快照 - * kafka avro格式 - * topN - ## 1 快速起步 ### 1.1 运行模式 @@ -205,6 +182,7 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack * [impala 结果表插件](docs/impalaSink.md) * [db2 结果表插件](docs/db2Sink.md) * [sqlserver 结果表插件](docs/sqlserverSink.md) +* [kafka 结果表插件](docs/kafkaSink.md) ### 2.3 维表插件 * [hbase 维表插件](docs/hbaseSide.md) diff --git a/docs/kafkaSink.md b/docs/kafkaSink.md new file mode 100644 index 000000000..3c6eb1dc6 --- /dev/null +++ b/docs/kafkaSink.md @@ -0,0 +1,223 @@ +## 1.格式： +``` +CREATE TABLE tableName( + colName colType, + ... + function(colNameX) AS aliasName, + WATERMARK FOR colName AS withOffset( colName , delayTime ) + )WITH( + type ='kafka11', + bootstrapServers ='ip:port,ip:port...', + zookeeperQuorum ='ip:port,ip:port/zkparent', + offsetReset ='latest', + topic ='topicName', + groupId='test', + parallelism ='parllNum', + ); +``` + +## 2.支持的版本 + kafka09,kafka10,kafka11及以上版本 + **kafka读取和写入的版本必须一致，否则会有兼容性错误。** + +## 3.表结构定义 + +|参数名称|含义| +|----|---| +| tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称| +| colName | 列名称| +| colType | 列类型 [colType支持的类型](colType.md)| + +## 4.参数： + +|参数名称|含义|是否必填|默认值| +|----|---|---|---| +|type | kafka09 | 是|kafka09、kafka10、kafka11、kafka(对应kafka1.0及以上版本)| +|groupId | 需要读取的 groupId 名称|否|| +|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| +|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| +|topic | 需要读取的 topic 名称|是|| +|parallelism | 并行度设置|否|1| +|partitionKeys | 用来分区的字段|否|| +|updateMode | 回溯流数据下发模式，append,upsert.upsert模式下会将是否为回溯信息以字段形式进行下发。|否|append| +|sinkdatatype | 写入kafka数据格式，json,avro,csv|否|json| +|fieldDelimiter | csv数据分隔符|否| \ | + + +**kafka相关参数可以自定义，使用kafka.开头即可。** +``` +kafka.consumer.id +kafka.socket.timeout.ms +kafka.fetch.message.max.bytes +kafka.num.consumer.fetchers +kafka.auto.commit.enable +kafka.auto.commit.interval.ms +kafka.queued.max.message.chunks +kafka.rebalance.max.retries +kafka.fetch.min.bytes +kafka.fetch.wait.max.ms +kafka.rebalance.backoff.ms +kafka.refresh.leader.backoff.ms +kafka.consumer.timeout.ms +kafka.exclude.internal.topics +kafka.partition.assignment.strategy +kafka.client.id +kafka.zookeeper.session.timeout.ms +kafka.zookeeper.connection.timeout.ms +kafka.zookeeper.sync.time.ms +kafka.offsets.storage +kafka.offsets.channel.backoff.ms +kafka.offsets.channel.socket.timeout.ms +kafka.offsets.commit.max.retries +kafka.dual.commit.enabled +kafka.partition.assignment.strategy +kafka.socket.receive.buffer.bytes +kafka.fetch.min.bytes +``` + +## 5.样例： + +### json格式： +``` +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='2', + partitionKeys = 'channel,pv', + updateMode='upsert' + ); + +upsert模式下发的数据格式：{"channel":"zs","pv":"330",retract:true} +append模式下发的数据格式：{"channel":"zs","pv":"330"} + +``` + +### avro格式： + +如果updateMode='upsert',schemaInfo需要包含retract属性信息。 + +``` +CREATE TABLE MyTable( + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='1', + topicIsPattern ='false' + ); + +create table sideTable( + channel varchar, + xccount int, + PRIMARY KEY(channel), + PERIOD FOR SYSTEM_TIME + )WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', + userName='dtstack', + password='abc123', + tableName='sidetest', + cache = 'LRU', + cacheTTLMs='10000', + parallelism ='1' + + ); + + +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + --type='console' + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='1', + updateMode='upsert', + sinkdatatype = 'avro', + schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"} + ,{"name":"pv","type":"string"},{"name":"channel","type":"string"}, + {"name":"retract","type":"boolean"}]}' + + ); + + +insert +into + MyResult + select + a.channel as channel, + a.pv as pv + from + MyTable a +``` +### csv格式： + +``` +CREATE TABLE MyTable( + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='2', + topicIsPattern ='false' + ); + +create table sideTable( + channel varchar, + xccount int, + PRIMARY KEY(channel), + PERIOD FOR SYSTEM_TIME + )WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', + userName='dtstack', + password='abc123', + tableName='sidetest', + cache = 'LRU', + cacheTTLMs='10000', + parallelism ='1' + + ); + + +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='2', + updateMode='upsert', + sinkdatatype = 'csv', + fieldDelimiter='*' + + + + ); + + +insert +into + MyResult + select + a.channel as channel, + a.pv as pv + from + MyTable a +``` diff --git a/docs/kafkaSource.md b/docs/kafkaSource.md index 76096aba6..9b488863e 100644 --- a/docs/kafkaSource.md +++ b/docs/kafkaSource.md @@ -1,14 +1,12 @@ ## 1.格式： ``` -数据现在支持json格式{"xx":"bb","cc":"dd"} - CREATE TABLE tableName( colName colType, ... function(colNameX) AS aliasName, WATERMARK FOR colName AS withOffset( colName , delayTime ) )WITH( - type ='kafka09', + type ='kafka11', bootstrapServers ='ip:port,ip:port...', zookeeperQuorum ='ip:port,ip:port/zkparent', offsetReset ='latest', @@ -89,7 +87,7 @@ CREATE TABLE MyTable( xctime bigint, CHARACTER_LENGTH(channel) AS timeLeng )WITH( - type ='kafka09', + type ='kafka11', bootstrapServers ='172.16.8.198:9092', zookeeperQuorum ='172.16.8.198:2181/kafka', offsetReset ='latest', @@ -208,181 +206,3 @@ CREATE TABLE MyTable( lengthcheckpolicy = 'PAD' ); ``` -# 三、text格式数据源UDF自定义拆分 -Kafka源表数据解析流程：Kafka Source Table -> UDTF ->Realtime Compute -> SINK。从Kakfa读入的数据，都是VARBINARY（二进制）格式，对读入的每条数据，都需要用UDTF将其解析成格式化数据。 - 与其他格式不同，本格式定义DDL必须与以下SQL一摸一样，表中的五个字段顺序务必保持一致： - -## 1. 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 -``` -create table kafka_stream( - _topic STRING, - _messageKey STRING, - _message STRING, - _partition INT, - _offset BIGINT, -) with ( - type ='kafka09', - bootstrapServers ='172.16.8.198:9092', - zookeeperQuorum ='172.16.8.198:2181/kafka', - offsetReset ='latest', - topic ='nbTest1', - parallelism ='1', - sourcedatatype='text' - ） -``` -## 2.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|| -|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|topic | 需要读取的 topic 名称|是|| -|offsetReset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| -|parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型|否|text| -**kafka相关参数可以自定义，使用kafka.开头即可。** - -## 2.自定义： -从kafka读出的数据，需要进行窗口计算。 按照实时计算目前的设计，滚窗/滑窗等窗口操作，需要（且必须）在源表DDL上定义Watermark。Kafka源表比较特殊。如果要以kafka中message字段中的的Event Time进行窗口操作， -需要先从message字段，使用UDX解析出event time，才能定义watermark。 在kafka源表场景中，需要使用计算列。 假设，kafka中写入的数据如下： -2018-11-11 00:00:00|1|Anna|female整个计算流程为：Kafka SOURCE->UDTF->Realtime Compute->RDS SINK（单一分隔符可直接使用类csv格式模板，自定义适用于更复杂的数据类型，本说明只做参考） - -**SQL** -``` --- 定义解析Kakfa message的UDTF - CREATE FUNCTION kafkapaser AS 'com.XXXX.kafkaUDTF'; - CREATE FUNCTION kafkaUDF AS 'com.XXXX.kafkaUDF'; - -- 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 - create table kafka_src ( - _topic STRING, - _messageKey STRING, - _message STRING, - _partition INT, - _offset BIGINT, - ctime AS TO_TIMESTAMP(kafkaUDF(_message)), -- 定义计算列，计算列可理解为占位符，源表中并没有这一列，其中的数据可经过下游计算得出。注意计算里的类型必须为timestamp才能在做watermark。 - watermark for ctime as withoffset(ctime,0) -- 在计算列上定义watermark - ) WITH ( - type = 'kafka010', -- Kafka Source类型，与Kafka版本强相关，目前支持的Kafka版本请参考本文档 - topic = 'test_kafka_topic', - ... - ); - create table rds_sink ( - name VARCHAR, - age INT, - grade VARCHAR, - updateTime TIMESTAMP - ) WITH( - type='mysql', - url='jdbc:mysql://localhost:3306/test', - tableName='test4', - userName='test', - password='XXXXXX' - ); - -- 使用UDTF，将二进制数据解析成格式化数据 - CREATE VIEW input_view ( - name, - age, - grade, - updateTime - ) AS - SELECT - COUNT(*) as cnt, - T.ctime, - T.order, - T.name, - T.sex - from - kafka_src as S, - LATERAL TABLE (kafkapaser _message)) as T ( - ctime, - order, - name, - sex - ) - Group BY T.sex, - TUMBLE(ctime, INTERVAL '1' MINUTE); - -- 对input_view中输出的数据做计算 - CREATE VIEW view2 ( - cnt, - sex - ) AS - SELECT - COUNT(*) as cnt, - T.sex - from - input_view - Group BY sex, TUMBLE(ctime, INTERVAL '1' MINUTE); - -- 使用解析出的格式化数据进行计算，并将结果输出到RDS中 - insert into rds_sink - SELECT - cnt,sex - from view2; - ``` -**UDF&UDTF** -``` -package com.XXXX; - import com.XXXX.fastjson.JSONObject; - import org.apache.flink.table.functions.TableFunction; - import org.apache.flink.table.types.DataType; - import org.apache.flink.table.types.DataTypes; - import org.apache.flink.types.Row; - import java.io.UnsupportedEncodingException; - /** - 以下例子解析输入Kafka中的JSON字符串，并将其格式化输出 - **/ - public class kafkaUDTF extends TableFunction { - public void eval(byte[] message) { - try { - // 读入一个二进制数据，并将其转换为String格式 - String msg = new String(message, "UTF-8"); - // 提取JSON Object中各字段 - String ctime = Timestamp.valueOf(data.split('\\|')[0]); - String order = data.split('\\|')[1]; - String name = data.split('\\|')[2]; - String sex = data.split('\\|')[3]; - // 将解析出的字段放到要输出的Row()对象 - Row row = new Row(4); - row.setField(0, ctime); - row.setField(1, age); - row.setField(2, grade); - row.setField(3, updateTime); - System.out.println("Kafka message str ==>" + row.toString()); - // 输出一行 - collect(row); - } catch (ClassCastException e) { - System.out.println("Input data format error. Input data " + msg + "is not json string"); - } - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - } - @Override - // 如果返回值是Row，就必须重载实现这个方法，显式地告诉系统返回的字段类型 - // 定义输出Row()对象的字段类型 - public DataType getResultType(Object[] arguments, Class[] argTypes) { - return DataTypes.createRowType(DataTypes.TIMESTAMP,DataTypes.STRING, DataTypes.Integer, DataTypes.STRING,DataTypes.STRING); - } - } - - package com.dp58; - package com.dp58.sql.udx; - import org.apache.flink.table.functions.FunctionContext; - import org.apache.flink.table.functions.ScalarFunction; - public class KafkaUDF extends ScalarFunction { - // 可选，open方法可以不写 - // 需要import org.apache.flink.table.functions.FunctionContext; - public String eval(byte[] message) { - // 读入一个二进制数据，并将其转换为String格式 - String msg = new String(message, "UTF-8"); - return msg.split('\\|')[0]; - } - public long eval(String b, String c) { - return eval(b) + eval(c); - } - //可选，close方法可以不写 - @Override - public void close() { - } - } - ``` diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java index ebd313b29..9dcaf222b 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java @@ -75,15 +75,16 @@ private SerializationSchema createSerializationSchema(KafkaSinkTableInfo k if (StringUtils.isBlank(kafkaSinkTableInfo.getFieldDelimiter())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.CSV.name() + " must set fieldDelimiter"); } - final CsvCRowSerializationSchema.Builder serSchemaBuilder = new CsvCRowSerializationSchema.Builder(typeInformation); serSchemaBuilder.setFieldDelimiter(kafkaSinkTableInfo.getFieldDelimiter().toCharArray()[0]); + serSchemaBuilder.setUpdateMode(kafkaSinkTableInfo.getUpdateMode()); + serializationSchema = serSchemaBuilder.build(); } else if (FormatType.AVRO.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { if (StringUtils.isBlank(kafkaSinkTableInfo.getSchemaString())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.AVRO.name() + " must set schemaString"); } - serializationSchema = new AvroCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(),kafkaSinkTableInfo.getUpdateMode()); + serializationSchema = new AvroCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(), kafkaSinkTableInfo.getUpdateMode()); } if (null == serializationSchema) { diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java index 34fa22c99..692e208b5 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java @@ -36,8 +36,6 @@ import org.apache.avro.util.Utf8; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.formats.avro.AvroRowDeserializationSchema; -import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; @@ -55,17 +53,16 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.TimeZone; +import java.util.stream.Collectors; /** * Serialization schema that serializes CROW into Avro bytes. * *

Serializes objects that are represented in (nested) Flink rows. It support types that * are compatible with Flink's Table & SQL API. - * - *

Note: Changes in this class need to be kept in sync with the corresponding runtime - * class {@link AvroRowDeserializationSchema} and schema converter {@link AvroSchemaConverter}. - * + ** * @author maqi */ public class AvroCRowSerializationSchema implements SerializationSchema { @@ -107,14 +104,14 @@ public class AvroCRowSerializationSchema implements SerializationSchema { private String updateMode; - private final String retractKey = "retract"; + private String retractKey = "retract"; /** * Creates an Avro serialization schema for the given specific record class. * * @param recordClazz Avro record class used to serialize Flink's row to Avro's record */ - public AvroCRowSerializationSchema(Class recordClazz) { + public AvroCRowSerializationSchema(Class recordClazz, String updateMode) { Preconditions.checkNotNull(recordClazz, "Avro record class must not be null."); this.recordClazz = recordClazz; this.schema = SpecificData.get().getSchema(recordClazz); @@ -122,6 +119,7 @@ public AvroCRowSerializationSchema(Class recordClazz) this.datumWriter = new SpecificDatumWriter<>(schema); this.arrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); + this.updateMode = updateMode; } /** @@ -152,10 +150,10 @@ public byte[] serialize(CRow crow) { // convert to record final GenericRecord record = convertRowToAvroRecord(schema, row); + + dealRetractField(change, record); + arrayOutputStream.reset(); - if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { - record.put(retractKey, change); - } datumWriter.write(record, encoder); encoder.flush(); return arrayOutputStream.toByteArray(); @@ -164,6 +162,18 @@ public byte[] serialize(CRow crow) { } } + protected void dealRetractField(boolean change, GenericRecord record) { + schema.getFields() + .stream() + .filter(field -> StringUtils.equalsIgnoreCase(field.name(), retractKey)) + .findFirst() + .ifPresent(field -> { + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + record.put(retractKey, convertFlinkType(field.schema(), change)); + } + }); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -184,7 +194,12 @@ public int hashCode() { // -------------------------------------------------------------------------------------------- private GenericRecord convertRowToAvroRecord(Schema schema, Row row) { - final List fields = schema.getFields(); + + final List fields = schema.getFields() + .stream() + .filter(field -> !StringUtils.equalsIgnoreCase(field.name(), retractKey)) + .collect(Collectors.toList()); + final int length = fields.size(); final GenericRecord record = new GenericData.Record(schema); for (int i = 0; i < length; i++) { @@ -328,6 +343,8 @@ private long convertFromTimestamp(Schema schema, Timestamp date) { private void writeObject(ObjectOutputStream outputStream) throws IOException { outputStream.writeObject(recordClazz); outputStream.writeObject(schemaString); // support for null + outputStream.writeObject(retractKey); + outputStream.writeObject(updateMode); } @SuppressWarnings("unchecked") @@ -339,6 +356,9 @@ private void readObject(ObjectInputStream inputStream) throws ClassNotFoundExcep } else { schema = new Schema.Parser().parse(schemaString); } + retractKey = (String) inputStream.readObject(); + updateMode = (String) inputStream.readObject(); + datumWriter = new SpecificDatumWriter<>(schema); arrayOutputStream = new ByteArrayOutputStream(); encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java index 903395f9d..4e57b6f2a 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java @@ -47,7 +47,10 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.util.Arrays; +import java.util.Iterator; import java.util.Objects; +import java.util.stream.IntStream; +import java.util.stream.Stream; /** * Serialization schema that serializes an object of Flink types into a CSV bytes. @@ -72,25 +75,27 @@ public final class CsvCRowSerializationSchema implements SerializationSchema kafkaParam = new HashMap(); @@ -119,9 +129,27 @@ public boolean check() { Preconditions.checkNotNull(getType(), "kafka of type is required"); Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); Preconditions.checkNotNull(topic, "kafka of topic is required"); + + if (StringUtils.equalsIgnoreCase(getSinkDataType(), FormatType.AVRO.name())) { + avroParamCheck(); + } + return false; } + public void avroParamCheck() { + Preconditions.checkNotNull(schemaString, "avro type schemaInfo is required"); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + Schema schema = new Schema.Parser().parse(schemaString); + schema.getFields() + .stream() + .filter(field -> StringUtils.equalsIgnoreCase(field.name(), RETRACT_FIELD_KEY)) + .findFirst() + .orElseThrow(() -> + new NullPointerException(String.valueOf("arvo upsert mode the retract attribute must be contained in schemaInfo field "))); + } + } + public String getEnableKeyPartition() { return enableKeyPartition; } From 7c3f0a268d1fad776168ebb3b721653f7043ca97 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 18 Mar 2020 18:21:06 +0800 Subject: [PATCH 155/523] field length check --- .../sql/side/oracle/OracleAsyncSideInfo.java | 20 ++++++------------- .../sql/side/rdb/table/RdbSideTableInfo.java | 2 ++ .../sql/sink/rdb/table/RdbTableInfo.java | 2 ++ 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java index 5ff27c542..8e801970f 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java @@ -25,24 +25,13 @@ import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.ParseUtils; -import com.mchange.lang.CharUtils; -import org.apache.calcite.sql.SqlNode; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; - -import java.util.Arrays; import java.util.List; public class OracleAsyncSideInfo extends RdbAsyncSideInfo { - private final String SQL_DEFAULT_PLACEHOLDER = " ? "; - private final String DEAL_CHAR_KEY = "char"; - private String RPAD_FORMAT = "rpad(?, %d, ' ')"; - - public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -62,13 +51,16 @@ public String wrapperPlaceholder(String fieldName) { int pos = sideTableInfo.getFieldList().indexOf(fieldName); String type = sideTableInfo.getFieldTypeList().get(pos); - if (StringUtils.contains(type.toLowerCase(), DEAL_CHAR_KEY)) { + String sqlDefaultPlaceholder = " ? "; + String rpadFormat = "rpad(?, %d, ' ')"; + + if (StringUtils.contains(type.toLowerCase(), "char")) { TableInfo.FieldExtraInfo fieldExtraInfo = sideTableInfo.getFieldExtraInfoList().get(pos); int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); if (charLength > 0) { - return String.format(RPAD_FORMAT, charLength); + return String.format(rpadFormat, charLength); } } - return SQL_DEFAULT_PLACEHOLDER; + return sqlDefaultPlaceholder; } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java index 0dfbef325..2c8c18fcd 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java @@ -47,6 +47,8 @@ public boolean check() { Preconditions.checkNotNull(tableName, "rdb of tableName is required"); Preconditions.checkNotNull(userName, "rdb of userName is required"); Preconditions.checkNotNull(password, "rdb of password is required"); + Preconditions.checkArgument(getFieldList().size() == getFieldExtraInfoList().size(), + "fields and fieldExtraInfoList attributes must be the same length"); return true; } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java index d43104fcb..c9dd3f5d5 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java @@ -144,6 +144,8 @@ public boolean check() { Preconditions.checkNotNull(tableName, "rdb field of tableName is required"); Preconditions.checkNotNull(userName, "rdb field of userName is required"); Preconditions.checkNotNull(password, "rdb field of password is required"); + Preconditions.checkArgument(getFieldList().size() == getFieldExtraInfoList().size(), + "fields and fieldExtraInfoList attributes must be the same length"); return true; } From d124305ffe07b66f8eee430b3c0ac1072281dc14 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 19 Mar 2020 18:37:48 +0800 Subject: [PATCH 156/523] add jacoco --- pom.xml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pom.xml b/pom.xml index 5493645e9..de963240c 100644 --- a/pom.xml +++ b/pom.xml @@ -54,6 +54,20 @@ + + org.jacoco + jacoco-maven-plugin + 0.7.8 + + + + prepare-agent + report + + + + + pl.project13.maven git-commit-id-plugin From 25316e3d705a5626b0a01a72e9e57e1693ace60d Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 19 Mar 2020 20:07:41 +0800 Subject: [PATCH 157/523] add kafka charsetname --- .../dtnest/DtNestRowDeserializationSchema.java | 9 +++++++-- .../source/kafka/AbstractKafkaConsumerFactory.java | 4 ++-- .../sql/source/kafka/table/KafkaSourceParser.java | 1 + .../sql/source/kafka/table/KafkaSourceTableInfo.java | 12 ++++++++++++ 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 78b579305..b6bf377ed 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -58,17 +58,22 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final String[] fieldNames; private final TypeInformation[] fieldTypes; private List fieldExtraInfos; + private String charsetName; - public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { + public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, + List fieldExtraInfos, + String charsetName) { this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); this.rowAndFieldMapping = rowAndFieldMapping; this.fieldExtraInfos = fieldExtraInfos; + this.charsetName = charsetName; } @Override public Row deserialize(byte[] message) throws IOException { - JsonNode root = objectMapper.readTree(message); + String decoderStr = new String(message, charsetName); + JsonNode root = objectMapper.readTree(decoderStr); this.parseTree(root, null); Row row = new Row(fieldNames.length); diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java index 2cae221cf..49680ea63 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java @@ -56,8 +56,8 @@ protected DeserializationMetricWrapper createDeserializationMetricWrapper(KafkaS private DeserializationSchema createDeserializationSchema(KafkaSourceTableInfo kafkaSourceTableInfo, TypeInformation typeInformation) { DeserializationSchema deserializationSchema = null; if (FormatType.DT_NEST.name().equalsIgnoreCase(kafkaSourceTableInfo.getSourceDataType())) { - deserializationSchema = new DtNestRowDeserializationSchema(typeInformation, kafkaSourceTableInfo.getPhysicalFields(), kafkaSourceTableInfo.getFieldExtraInfoList()); - + deserializationSchema = new DtNestRowDeserializationSchema(typeInformation, kafkaSourceTableInfo.getPhysicalFields(), + kafkaSourceTableInfo.getFieldExtraInfoList(),kafkaSourceTableInfo.getCharsetName()); } else if (FormatType.JSON.name().equalsIgnoreCase(kafkaSourceTableInfo.getSourceDataType())) { if (StringUtils.isNotBlank(kafkaSourceTableInfo.getSchemaString())) { diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 005bfb4c2..6d84a057a 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -52,6 +52,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Fri, 20 Mar 2020 09:37:01 +0800 Subject: [PATCH 158/523] socket decoder --- .../serversocket/CustomerSocketTextStreamFunction.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java index e46e0a76f..bdcea4672 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java @@ -64,12 +64,16 @@ public class CustomerSocketTextStreamFunction implements SourceFunction { private transient Socket currentSocket; + private String CHARSET_NAME = "UTF-8"; + ServersocketSourceTableInfo tableInfo; + + public CustomerSocketTextStreamFunction(ServersocketSourceTableInfo tableInfo, TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { this.tableInfo = tableInfo; - this.deserializationSchema = new DtNestRowDeserializationSchema(typeInfo, rowAndFieldMapping, fieldExtraInfos); + this.deserializationSchema = new DtNestRowDeserializationSchema(typeInfo, rowAndFieldMapping, fieldExtraInfos, CHARSET_NAME); this.deserializationMetricWrapper = new DeserializationMetricWrapper(typeInfo, deserializationSchema); } From 615ea8363bf60c1ee57c586a0fc64720a6fc8535 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 20 Mar 2020 15:55:18 +0800 Subject: [PATCH 159/523] =?UTF-8?q?=E5=BC=82=E5=B8=B8=E9=87=8D=E8=AF=95?= =?UTF-8?q?=E9=80=BB=E8=BE=91=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/BaseAsyncReqRow.java | 10 ++++++-- .../sql/side/rdb/async/RdbAsyncReqRow.java | 23 +++++++++++-------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index ae987e142..221bf2c85 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -30,6 +30,7 @@ import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.MapUtils; +import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -62,6 +63,7 @@ public abstract class BaseAsyncReqRow extends RichAsyncFunction implements ISideReqRow { private static final Logger LOG = LoggerFactory.getLogger(BaseAsyncReqRow.class); private static final long serialVersionUID = 2098635244857937717L; + private RuntimeContext runtimeContext; private static int TIMEOUT_LOG_FLUSH_NUM = 10; private int timeOutNum = 0; @@ -72,7 +74,11 @@ public abstract class BaseAsyncReqRow extends RichAsyncFunction impl public BaseAsyncReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; } - + @Override + public void setRuntimeContext(RuntimeContext runtimeContext) { + super.setRuntimeContext(runtimeContext); + this.runtimeContext = runtimeContext; + } @Override public void open(Configuration parameters) throws Exception { super.open(parameters); @@ -221,7 +227,7 @@ private void invokeWithCache(Map inputParams, CRow input, Result public abstract String buildCacheKey(Map inputParams); private ProcessingTimeService getProcessingTimeService(){ - return ((StreamingRuntimeContext)this.getRuntimeContext()).getProcessingTimeService(); + return ((StreamingRuntimeContext)this.runtimeContext).getProcessingTimeService(); } protected ScheduledFuture registerTimer(CRow input, ResultFuture resultFuture){ diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 5e5de7e38..6b4298f24 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -89,25 +89,28 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { - AtomicBoolean connectFinish = new AtomicBoolean(false); - AtomicInteger counter = new AtomicInteger(0); - while(!connectFinish.get()){ + AtomicInteger failCounter = new AtomicInteger(0); + while(true){ + AtomicBoolean connectFinish = new AtomicBoolean(false); rdbSqlClient.getConnection(conn -> { + connectFinish.set(true); if(conn.failed()){ - if(counter.get() % 1000 == 0){ + if(failCounter.getAndIncrement() % 1000 == 0){ logger.error("getConnection error", conn.cause()); } - if(counter.incrementAndGet() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ - resultFuture.completeExceptionally(conn.cause()); - connectFinish.set(true); - return; - } + conn.result().close(); } - connectFinish.set(true); ScheduledFuture timerFuture = registerTimer(input, resultFuture); cancelTimerWhenComplete(resultFuture, timerFuture); handleQuery(conn.result(), inputParams, input, resultFuture); }); + while(!connectFinish.get()){ + Thread.sleep(50); + } + if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ + resultFuture.completeExceptionally(new RuntimeException("connection fail")); + return; + } } } From baabfd45555949eb8ead829da31e6d1c492dfbbd Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 20 Mar 2020 16:41:31 +0800 Subject: [PATCH 160/523] extract abstractKafkaSource --- .../dtstack/flink/sql/util/DtStringUtil.java | 8 +- .../sql/source/kafka/AbstractKafkaSource.java | 117 ++++++++++++ .../sql/source/kafka/enums/EKafkaOffset.java | 31 ++++ .../source/kafka/table/KafkaSourceParser.java | 29 +-- .../kafka/table/KafkaSourceTableInfo.java | 175 ++++++++---------- .../flink/sql/source/kafka/KafkaSource.java | 74 +------- .../flink/sql/source/kafka/KafkaSource.java | 72 +------ .../flink/sql/source/kafka/KafkaSource.java | 87 ++------- .../flink/sql/source/kafka/KafkaSource.java | 90 ++------- 9 files changed, 292 insertions(+), 391 deletions(-) create mode 100644 kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java create mode 100644 kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index b2486b1ab..10a6b4f63 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -219,14 +219,14 @@ public static String addJdbcParam(String dbUrl, Map addParams, b return preStr + "?" + sb.toString(); } - public static boolean isJosn(String str){ + public static boolean isJson(String str) { boolean flag = false; - if(StringUtils.isNotBlank(str)){ + if (StringUtils.isNotBlank(str)) { try { - objectMapper.readValue(str,Map.class); + objectMapper.readValue(str, Map.class); flag = true; } catch (Throwable e) { - flag=false; + flag = false; } } return flag; diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java new file mode 100644 index 000000000..852a381e2 --- /dev/null +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.source.kafka; + +import com.dtstack.flink.sql.source.IStreamSourceGener; +import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; +import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.util.PluginUtil; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; +import org.apache.flink.table.api.Table; +import org.apache.flink.types.Row; +import org.apache.kafka.clients.consumer.ConsumerConfig; + +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * Date: 2020/3/20 + * Company: www.dtstack.com + * @author maqi + */ +public abstract class AbstractKafkaSource implements IStreamSourceGener

{ + + private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; + + protected Properties getKafkaProperties(KafkaSourceTableInfo kafkaSourceTableInfo) { + Properties props = new Properties(); + props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaSourceTableInfo.getBootstrapServers()); + + if (DtStringUtil.isJson(kafkaSourceTableInfo.getOffsetReset())) { + props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, EKafkaOffset.NONE.name().toLowerCase()); + } else { + props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, kafkaSourceTableInfo.getOffsetReset()); + } + + if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())) { + props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, kafkaSourceTableInfo.getGroupId()); + } + + for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { + props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); + } + return props; + } + + protected String generateOperatorName(String tabName, String topicName) { + return SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", tabName); + } + + protected TypeInformation getRowTypeInformation(KafkaSourceTableInfo kafkaSourceTableInfo) { + Class[] fieldClasses = kafkaSourceTableInfo.getFieldClasses(); + TypeInformation[] types = IntStream.range(0, fieldClasses.length) + .mapToObj(i -> TypeInformation.of(fieldClasses[i])) + .toArray(TypeInformation[]::new); + + return new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + } + + protected void setStartPosition(String offset, String topicName, FlinkKafkaConsumerBase kafkaSrc) { + if (StringUtils.equalsIgnoreCase(offset, EKafkaOffset.EARLIEST.name())) { + kafkaSrc.setStartFromEarliest(); + } else if (DtStringUtil.isJson(offset)) { + Map specificStartupOffsets = buildOffsetMap(offset, topicName); + kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); + } else { + kafkaSrc.setStartFromLatest(); + } + } + + /** + * kafka offset,eg.. {"0":12312,"1":12321,"2":12312} + * @param offsetJson + * @param topicName + * @return + */ + protected Map buildOffsetMap(String offsetJson, String topicName) { + try { + Properties properties = PluginUtil.jsonStrToObject(offsetJson, Properties.class); + Map offsetMap = PluginUtil.objectToMap(properties); + Map specificStartupOffsets = offsetMap + .entrySet() + .stream() + .collect(Collectors.toMap( + (Map.Entry entry) -> new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), + (Map.Entry entry) -> Long.valueOf(entry.getValue().toString())) + ); + + return specificStartupOffsets; + } catch (Exception e) { + throw new RuntimeException("not support offsetReset type:" + offsetJson); + } + } + +} diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java new file mode 100644 index 000000000..476ccc8b7 --- /dev/null +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.source.kafka.enums; + +/** + * Date: 2020/3/20 + * Company: www.dtstack.com + * @author maqi + */ +public enum EKafkaOffset { + + LATEST, + EARLIEST, + NONE +} diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 8b0b9753c..867f48d6a 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -19,11 +19,13 @@ package com.dtstack.flink.sql.source.kafka.table; +import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; import com.dtstack.flink.sql.table.AbstractSourceParser; import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; +import java.util.stream.Collectors; /** * Reason: @@ -37,28 +39,27 @@ public class KafkaSourceParser extends AbstractSourceParser { public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { KafkaSourceTableInfo kafkaSourceTableInfo = new KafkaSourceTableInfo(); - kafkaSourceTableInfo.setName(tableName); - kafkaSourceTableInfo.setType(MathUtil.getString(props.get(KafkaSourceTableInfo.TYPE_KEY.toLowerCase()))); parseFieldsInfo(fieldsInfo, kafkaSourceTableInfo); + kafkaSourceTableInfo.setName(tableName); + kafkaSourceTableInfo.setType(MathUtil.getString(props.get(KafkaSourceTableInfo.TYPE_KEY.toLowerCase()))); kafkaSourceTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(KafkaSourceTableInfo.PARALLELISM_KEY.toLowerCase()))); - String bootstrapServer = MathUtil.getString(props.get(KafkaSourceTableInfo.BOOTSTRAPSERVERS_KEY.toLowerCase())); - if (bootstrapServer == null || "".equals(bootstrapServer.trim())) { - throw new Exception("BootstrapServers can not be empty!"); - } else { - kafkaSourceTableInfo.setBootstrapServers(bootstrapServer); - } + kafkaSourceTableInfo.setBootstrapServers(MathUtil.getString(props.get(KafkaSourceTableInfo.BOOTSTRAPSERVERS_KEY.toLowerCase()))); kafkaSourceTableInfo.setGroupId(MathUtil.getString(props.get(KafkaSourceTableInfo.GROUPID_KEY.toLowerCase()))); kafkaSourceTableInfo.setTopic(MathUtil.getString(props.get(KafkaSourceTableInfo.TOPIC_KEY.toLowerCase()))); - kafkaSourceTableInfo.setOffsetReset(MathUtil.getString(props.get(KafkaSourceTableInfo.OFFSETRESET_KEY.toLowerCase()))); + kafkaSourceTableInfo.setOffsetReset(MathUtil.getString(props.getOrDefault(KafkaSourceTableInfo.OFFSETRESET_KEY.toLowerCase(), EKafkaOffset.LATEST.name().toLowerCase()))); kafkaSourceTableInfo.setTopicIsPattern(MathUtil.getBoolean(props.get(KafkaSourceTableInfo.TOPICISPATTERN_KEY.toLowerCase()))); kafkaSourceTableInfo.setTimeZone(MathUtil.getString(props.get(KafkaSourceTableInfo.TIME_ZONE_KEY.toLowerCase()))); - for (String key : props.keySet()) { - if (!key.isEmpty() && key.startsWith("kafka.")) { - kafkaSourceTableInfo.addKafkaParam(key.substring(6), props.get(key).toString()); - } - } + + Map kafkaParams = props.keySet().stream() + .filter(key -> !key.isEmpty() && key.startsWith("kafka.")) + .collect(Collectors.toMap( + key -> key.substring(6), key -> props.get(key).toString()) + ); + + kafkaSourceTableInfo.addKafkaParam(kafkaParams); kafkaSourceTableInfo.check(); + return kafkaSourceTableInfo; } } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index 364cbff49..e1c014a69 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -17,7 +17,6 @@ */ - package com.dtstack.flink.sql.source.kafka.table; import com.dtstack.flink.sql.format.FormatType; @@ -37,134 +36,118 @@ public class KafkaSourceTableInfo extends AbstractSourceTableInfo { - public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; - - public static final String TOPIC_KEY = "topic"; - - public static final String TYPE_KEY = "type"; + public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; - public static final String GROUPID_KEY = "groupId"; + public static final String TOPIC_KEY = "topic"; - public static final String OFFSETRESET_KEY = "offsetReset"; + public static final String TYPE_KEY = "type"; - public static final String TOPICISPATTERN_KEY = "topicIsPattern"; + public static final String GROUPID_KEY = "groupId"; - private String bootstrapServers; + public static final String OFFSETRESET_KEY = "offsetReset"; - private String topic; + public static final String TOPICISPATTERN_KEY = "topicIsPattern"; - private String groupId; + private String bootstrapServers; - //latest, earliest - private String offsetReset = "latest"; + private String topic; - private String offset; + private String groupId; - private Boolean topicIsPattern = false; + private String offsetReset; - private String sourceDataType = FormatType.DT_NEST.name(); + private Boolean topicIsPattern = false; - private String schemaString; + private String sourceDataType = FormatType.DT_NEST.name(); - private String fieldDelimiter; + private String schemaString; - public String getBootstrapServers() { - return bootstrapServers; - } + private String fieldDelimiter; - public void setBootstrapServers(String bootstrapServers) { - this.bootstrapServers = bootstrapServers; - } + public Map kafkaParam = new HashMap<>(); - public String getTopic() { - return topic; - } - public void setTopic(String topic) { - this.topic = topic; - } + public String getBootstrapServers() { + return bootstrapServers; + } - public String getGroupId() { - return groupId; - } + public void setBootstrapServers(String bootstrapServers) { + this.bootstrapServers = bootstrapServers; + } - public void setGroupId(String groupId) { - this.groupId = groupId; - } + public String getTopic() { + return topic; + } - public String getOffsetReset() { - return offsetReset; - } + public void setTopic(String topic) { + this.topic = topic; + } - public void setOffsetReset(String offsetReset) { - if(offsetReset == null){ - return; - } - this.offsetReset = offsetReset; - } + public String getGroupId() { + return groupId; + } - public String getOffset() { - return offset; - } + public void setGroupId(String groupId) { + this.groupId = groupId; + } - public void setOffset(String offset) { - if (offsetReset == null) { - return; - } - this.offset = offset; - } + public String getOffsetReset() { + return offsetReset; + } - public Boolean getTopicIsPattern() { - return topicIsPattern; - } + public void setOffsetReset(String offsetReset) { + this.offsetReset = offsetReset; + } - public void setTopicIsPattern(Boolean topicIsPattern) { - this.topicIsPattern = topicIsPattern; - } + public Boolean getTopicIsPattern() { + return topicIsPattern; + } - public Map kafkaParam = new HashMap<>(); + public void setTopicIsPattern(Boolean topicIsPattern) { + this.topicIsPattern = topicIsPattern; + } - public void addKafkaParam(String key, String value) { - kafkaParam.put(key, value); - } + public void addKafkaParam(Map kafkaParam) { + kafkaParam.putAll(kafkaParam); + } - public String getKafkaParam(String key) { - return kafkaParam.get(key); - } + public String getKafkaParam(String key) { + return kafkaParam.get(key); + } - public Set getKafkaParamKeys() { - return kafkaParam.keySet(); - } + public Set getKafkaParamKeys() { + return kafkaParam.keySet(); + } - public String getSourceDataType() { - return sourceDataType; - } + public String getSourceDataType() { + return sourceDataType; + } - public void setSourceDataType(String sourceDataType) { - this.sourceDataType = sourceDataType; - } + public void setSourceDataType(String sourceDataType) { + this.sourceDataType = sourceDataType; + } - public String getSchemaString() { - return schemaString; - } + public String getSchemaString() { + return schemaString; + } - public void setSchemaString(String schemaString) { - this.schemaString = schemaString; - } + public void setSchemaString(String schemaString) { + this.schemaString = schemaString; + } - public String getFieldDelimiter() { - return fieldDelimiter; - } + public String getFieldDelimiter() { + return fieldDelimiter; + } - public void setFieldDelimiter(String fieldDelimiter) { - this.fieldDelimiter = fieldDelimiter; - } + public void setFieldDelimiter(String fieldDelimiter) { + this.fieldDelimiter = fieldDelimiter; + } - @Override - public boolean check() { - Preconditions.checkNotNull(getType(), "kafka of type is required"); - Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); - Preconditions.checkNotNull(topic, "kafka of topic is required"); - return false; - } + @Override + public boolean check() { + Preconditions.checkNotNull(getType(), "kafka of type is required"); + Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); + Preconditions.checkNotNull(topic, "kafka of topic is required"); + return false; + } } diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 7e2d77c27..394ea86ee 100644 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -18,24 +18,17 @@ package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; import com.dtstack.flink.sql.table.AbstractSourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; /** @@ -43,75 +36,24 @@ * @create: 2019-11-05 10:55 * @description: **/ -public class KafkaSource implements IStreamSourceGener
{ +public class KafkaSource extends AbstractKafkaSource { - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") @Override public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) { - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())) { - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } - - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } - - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer kafkaSrc = (FlinkKafkaConsumer) new KafkaConsumerFactory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - FlinkKafkaConsumer kafkaSrc = (FlinkKafkaConsumer) new KafkaConsumerFactory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); - - - //earliest,latest - if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.objectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for (Map.Entry entry : offsetMap.entrySet()) { - specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - } else { - kafkaSrc.setStartFromLatest(); - } + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } return tableEnv.fromDataStream(kafkaSource, fields); } } diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index ffb466a4c..9f8917761 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -17,17 +17,13 @@ */ - package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer09; @@ -36,7 +32,6 @@ import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; -import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -47,74 +42,23 @@ * @author xuchao */ -public class KafkaSource implements IStreamSourceGener
{ - - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") +public class KafkaSource extends AbstractKafkaSource { @Override public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } - // only required for Kafka 0.8 - //TODO props.setProperty("zookeeper.connect", kafkaSourceTableInfo.) + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer09 kafkaSrc = (FlinkKafkaConsumer09) new KafkaConsumer09Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for(int i = 0; i< kafkaSourceTableInfo.getFieldClasses().length; i++){ - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } - - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); - FlinkKafkaConsumer09 kafkaSrc = (FlinkKafkaConsumer09) new KafkaConsumer09Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); - - //earliest,latest - if("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())){ - kafkaSrc.setStartFromEarliest(); - }else if(DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.objectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for(Map.Entry entry:offsetMap.entrySet()){ - specificStartupOffsets.put(new KafkaTopicPartition(topicName,Integer.valueOf(entry.getKey())),Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - }else { - kafkaSrc.setStartFromLatest(); - } + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } return tableEnv.fromDataStream(kafkaSource, fields); } } \ No newline at end of file diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index ff4aed89d..23989ab7e 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -19,24 +19,17 @@ package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; import com.dtstack.flink.sql.table.AbstractSourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; /** @@ -47,74 +40,24 @@ * @author sishu.yss */ -public class KafkaSource implements IStreamSourceGener
{ +public class KafkaSource extends AbstractKafkaSource { - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; + @Override + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; + String topicName = kafkaSourceTableInfo.getTopic(); - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") - @Override - public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer010 kafkaSrc = (FlinkKafkaConsumer010) new KafkaConsumer010Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; - String topicName = kafkaSourceTableInfo.getTopic(); + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); + String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } - - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); - - FlinkKafkaConsumer010 kafkaSrc = (FlinkKafkaConsumer010) new KafkaConsumer010Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); - - //earliest,latest - if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.objectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for (Map.Entry entry : offsetMap.entrySet()) { - specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - } else { - kafkaSrc.setStartFromLatest(); - } - - String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } - return tableEnv.fromDataStream(kafkaSource, fields); - } + return tableEnv.fromDataStream(kafkaSource, fields); + } } \ No newline at end of file diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 11be1898a..f58d59d05 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -19,24 +19,16 @@ package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; import com.dtstack.flink.sql.table.AbstractSourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; - -import java.util.HashMap; -import java.util.Map; import java.util.Properties; /** @@ -47,76 +39,24 @@ * @author sishu.yss */ -public class KafkaSource implements IStreamSourceGener
{ - - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") - @Override - public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; - String topicName = kafkaSourceTableInfo.getTopic(); - - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } - // only required for Kafka 0.8 - //TODO props.setProperty("zookeeper.connect", kafka09SourceTableInfo.) - - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } +public class KafkaSource extends AbstractKafkaSource { - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + @Override + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; + String topicName = kafkaSourceTableInfo.getTopic(); - FlinkKafkaConsumer011 kafkaSrc = (FlinkKafkaConsumer011) new KafkaConsumer011Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer011 kafkaSrc = (FlinkKafkaConsumer011) new KafkaConsumer011Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - //earliest,latest - if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.objectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for (Map.Entry entry : offsetMap.entrySet()) { - specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - } else { - kafkaSrc.setStartFromLatest(); - } + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); - String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); + String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } - return tableEnv.fromDataStream(kafkaSource, fields); - } + return tableEnv.fromDataStream(kafkaSource, fields); + } } From 0b22f9c7d5c776ad7356b731c95a8704e4dd5530 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 20 Mar 2020 17:20:46 +0800 Subject: [PATCH 161/523] add avro params --- docs/kafkaSource.md | 224 ++---------------- .../source/kafka/table/KafkaSourceParser.java | 5 + .../kafka/table/KafkaSourceTableInfo.java | 8 +- 3 files changed, 38 insertions(+), 199 deletions(-) diff --git a/docs/kafkaSource.md b/docs/kafkaSource.md index 76096aba6..00c252c99 100644 --- a/docs/kafkaSource.md +++ b/docs/kafkaSource.md @@ -1,6 +1,5 @@ ## 1.格式： ``` -数据现在支持json格式{"xx":"bb","cc":"dd"} CREATE TABLE tableName( colName colType, @@ -15,9 +14,8 @@ CREATE TABLE tableName( topic ='topicName', groupId='test', parallelism ='parllNum', - --timezone='America/Los_Angeles', timezone='Asia/Shanghai', - sourcedatatype ='json' #可不设置 + sourcedatatype ='dt_nest' #可不设置 ); ``` @@ -47,7 +45,9 @@ CREATE TABLE tableName( |topicIsPattern | topic是否是正则表达式格式(true|false) |否| false |offsetReset | 读取的topic 的offset初始位置[latest|earliest|指定offset值({"0":12312,"1":12321,"2":12312},{"partition_no":offset_value})]|否|latest| |parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型|否|json| +|sourcedatatype | 数据类型,avro,csv,json,dt_nest。dt_nest为默认JSON解析器，能够解析嵌套JSON数据类型，其他仅支持非嵌套格式|否|dt_nest| +|schemaInfo | avro类型使用的schema信息|否|| +|fieldDelimiter |csv类型使用的数据分隔符|否| | | |timezone|时区设置[timezone支持的参数](timeZone.md)|否|'Asia/Shanghai' **kafka相关参数可以自定义，使用kafka.开头即可。** ``` @@ -169,24 +169,10 @@ CREATE TABLE MyTable( parallelism ='1' ); ``` -# 二、csv格式数据源 -根据字段分隔符进行数据分隔，按顺序匹配sql中配置的列。如数据分隔列数和sql中配置的列数相等直接匹配；如不同参照lengthcheckpolicy策略处理。 -## 1.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|| -|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|topic | 需要读取的 topic 名称|是|| -|offsetReset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| -|parallelism | 并行度设置 |否|1| -|sourcedatatype | 数据类型|是 |csv| -|fielddelimiter | 字段分隔符|是 || -|lengthcheckpolicy | 单行字段条数检查策略 |否|可选，默认为SKIP,其它可选值为EXCEPTION、PAD。SKIP：字段数目不符合时跳过 。EXCEPTION:字段数目不符合时抛出异常。PAD:按顺序填充，不存在的置为null。| -**kafka相关参数可以自定义，使用kafka.开头即可。** -## 2.样例： +## 7.csv格式数据源 + + ``` CREATE TABLE MyTable( name varchar, @@ -203,186 +189,28 @@ CREATE TABLE MyTable( --topic ='mqTest.*', --topicIsPattern='true' parallelism ='1', - sourcedatatype ='csv', - fielddelimiter ='\|', - lengthcheckpolicy = 'PAD' + sourceDatatype ='csv' ); ``` -# 三、text格式数据源UDF自定义拆分 -Kafka源表数据解析流程：Kafka Source Table -> UDTF ->Realtime Compute -> SINK。从Kakfa读入的数据，都是VARBINARY（二进制）格式，对读入的每条数据，都需要用UDTF将其解析成格式化数据。 - 与其他格式不同，本格式定义DDL必须与以下SQL一摸一样，表中的五个字段顺序务必保持一致： - -## 1. 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 -``` -create table kafka_stream( - _topic STRING, - _messageKey STRING, - _message STRING, - _partition INT, - _offset BIGINT, -) with ( - type ='kafka09', - bootstrapServers ='172.16.8.198:9092', - zookeeperQuorum ='172.16.8.198:2181/kafka', - offsetReset ='latest', - topic ='nbTest1', - parallelism ='1', - sourcedatatype='text' - ） -``` -## 2.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|| -|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|topic | 需要读取的 topic 名称|是|| -|offsetReset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| -|parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型|否|text| -**kafka相关参数可以自定义，使用kafka.开头即可。** +## 8.avro格式数据源 -## 2.自定义： -从kafka读出的数据，需要进行窗口计算。 按照实时计算目前的设计，滚窗/滑窗等窗口操作，需要（且必须）在源表DDL上定义Watermark。Kafka源表比较特殊。如果要以kafka中message字段中的的Event Time进行窗口操作， -需要先从message字段，使用UDX解析出event time，才能定义watermark。 在kafka源表场景中，需要使用计算列。 假设，kafka中写入的数据如下： -2018-11-11 00:00:00|1|Anna|female整个计算流程为：Kafka SOURCE->UDTF->Realtime Compute->RDS SINK（单一分隔符可直接使用类csv格式模板，自定义适用于更复杂的数据类型，本说明只做参考） - -**SQL** ``` --- 定义解析Kakfa message的UDTF - CREATE FUNCTION kafkapaser AS 'com.XXXX.kafkaUDTF'; - CREATE FUNCTION kafkaUDF AS 'com.XXXX.kafkaUDF'; - -- 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 - create table kafka_src ( - _topic STRING, - _messageKey STRING, - _message STRING, - _partition INT, - _offset BIGINT, - ctime AS TO_TIMESTAMP(kafkaUDF(_message)), -- 定义计算列，计算列可理解为占位符，源表中并没有这一列，其中的数据可经过下游计算得出。注意计算里的类型必须为timestamp才能在做watermark。 - watermark for ctime as withoffset(ctime,0) -- 在计算列上定义watermark - ) WITH ( - type = 'kafka010', -- Kafka Source类型，与Kafka版本强相关，目前支持的Kafka版本请参考本文档 - topic = 'test_kafka_topic', - ... - ); - create table rds_sink ( - name VARCHAR, - age INT, - grade VARCHAR, - updateTime TIMESTAMP - ) WITH( - type='mysql', - url='jdbc:mysql://localhost:3306/test', - tableName='test4', - userName='test', - password='XXXXXX' +CREATE TABLE MyTable( + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='1', + topicIsPattern ='false', + kafka.group.id='mqTest', + sourceDataType ='avro', + schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"},{"name":"pv","type":"string"}]}' ); - -- 使用UDTF，将二进制数据解析成格式化数据 - CREATE VIEW input_view ( - name, - age, - grade, - updateTime - ) AS - SELECT - COUNT(*) as cnt, - T.ctime, - T.order, - T.name, - T.sex - from - kafka_src as S, - LATERAL TABLE (kafkapaser _message)) as T ( - ctime, - order, - name, - sex - ) - Group BY T.sex, - TUMBLE(ctime, INTERVAL '1' MINUTE); - -- 对input_view中输出的数据做计算 - CREATE VIEW view2 ( - cnt, - sex - ) AS - SELECT - COUNT(*) as cnt, - T.sex - from - input_view - Group BY sex, TUMBLE(ctime, INTERVAL '1' MINUTE); - -- 使用解析出的格式化数据进行计算，并将结果输出到RDS中 - insert into rds_sink - SELECT - cnt,sex - from view2; - ``` -**UDF&UDTF** + ``` -package com.XXXX; - import com.XXXX.fastjson.JSONObject; - import org.apache.flink.table.functions.TableFunction; - import org.apache.flink.table.types.DataType; - import org.apache.flink.table.types.DataTypes; - import org.apache.flink.types.Row; - import java.io.UnsupportedEncodingException; - /** - 以下例子解析输入Kafka中的JSON字符串，并将其格式化输出 - **/ - public class kafkaUDTF extends TableFunction { - public void eval(byte[] message) { - try { - // 读入一个二进制数据，并将其转换为String格式 - String msg = new String(message, "UTF-8"); - // 提取JSON Object中各字段 - String ctime = Timestamp.valueOf(data.split('\\|')[0]); - String order = data.split('\\|')[1]; - String name = data.split('\\|')[2]; - String sex = data.split('\\|')[3]; - // 将解析出的字段放到要输出的Row()对象 - Row row = new Row(4); - row.setField(0, ctime); - row.setField(1, age); - row.setField(2, grade); - row.setField(3, updateTime); - System.out.println("Kafka message str ==>" + row.toString()); - // 输出一行 - collect(row); - } catch (ClassCastException e) { - System.out.println("Input data format error. Input data " + msg + "is not json string"); - } - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - } - @Override - // 如果返回值是Row，就必须重载实现这个方法，显式地告诉系统返回的字段类型 - // 定义输出Row()对象的字段类型 - public DataType getResultType(Object[] arguments, Class[] argTypes) { - return DataTypes.createRowType(DataTypes.TIMESTAMP,DataTypes.STRING, DataTypes.Integer, DataTypes.STRING,DataTypes.STRING); - } - } - - package com.dp58; - package com.dp58.sql.udx; - import org.apache.flink.table.functions.FunctionContext; - import org.apache.flink.table.functions.ScalarFunction; - public class KafkaUDF extends ScalarFunction { - // 可选，open方法可以不写 - // 需要import org.apache.flink.table.functions.FunctionContext; - public String eval(byte[] message) { - // 读入一个二进制数据，并将其转换为String格式 - String msg = new String(message, "UTF-8"); - return msg.split('\\|')[0]; - } - public long eval(String b, String c) { - return eval(b) + eval(c); - } - //可选，close方法可以不写 - @Override - public void close() { - } - } - ``` + diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 867f48d6a..96ccd1783 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.source.kafka.table; +import com.dtstack.flink.sql.format.FormatType; import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; import com.dtstack.flink.sql.table.AbstractSourceParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -51,6 +52,10 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map kafkaParams = props.keySet().stream() .filter(key -> !key.isEmpty() && key.startsWith("kafka.")) .collect(Collectors.toMap( diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index e1c014a69..c27eee376 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -48,6 +48,12 @@ public class KafkaSourceTableInfo extends AbstractSourceTableInfo { public static final String TOPICISPATTERN_KEY = "topicIsPattern"; + public static final String SCHEMA_STRING_KEY = "schemaInfo"; + + public static final String CSV_FIELD_DELIMITER_KEY = "fieldDelimiter"; + + public static final String SOURCE_DATA_TYPE_KEY = "sourceDataType"; + private String bootstrapServers; private String topic; @@ -58,7 +64,7 @@ public class KafkaSourceTableInfo extends AbstractSourceTableInfo { private Boolean topicIsPattern = false; - private String sourceDataType = FormatType.DT_NEST.name(); + private String sourceDataType; private String schemaString; From 728490ac4e29f92119899b23c0dfd6a522420b6e Mon Sep 17 00:00:00 2001 From: xuchao Date: Sun, 22 Mar 2020 18:49:08 +0800 Subject: [PATCH 162/523] =?UTF-8?q?=E5=88=9D=E6=AD=A5=E9=87=8D=E6=9E=84=20?= =?UTF-8?q?sql=E8=AF=AD=E5=8F=A5=E8=A7=A3=E6=9E=90=E9=80=BB=E8=BE=91?= =?UTF-8?q?=EF=BC=9B=20=E5=AD=97=E6=AE=B5=E6=9B=BF=E6=8D=A2=E5=92=8C?= =?UTF-8?q?=E9=87=8D=E5=91=BD=E5=90=8D=E7=9B=B8=E5=85=B3=E7=9A=84=E9=80=BB?= =?UTF-8?q?=E8=BE=91=E5=9C=A8=E8=A7=A3=E6=9E=90=E9=98=B6=E6=AE=B5=E5=AE=8C?= =?UTF-8?q?=E6=88=90=EF=BC=9B=20=E7=9B=AE=E6=A0=87=EF=BC=9A=E7=BB=8F?= =?UTF-8?q?=E8=BF=87=E8=A7=A3=E6=9E=90sql=E4=B9=8B=E5=90=8E=E7=9A=84?= =?UTF-8?q?=E6=89=80=E6=9C=89sql=E6=98=AF=E7=A1=AE=E5=AE=9A=E7=9A=84?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/FieldReplaceInfo.java | 27 +- .../com/dtstack/flink/sql/side/JoinInfo.java | 52 +++ .../flink/sql/side/JoinNodeDealer.java | 398 ++++++++++++++---- .../dtstack/flink/sql/side/SideSqlExec.java | 351 +-------------- .../dtstack/flink/sql/util/ParseUtils.java | 41 +- .../dtstack/flink/sql/util/TableUtils.java | 130 ++++-- 6 files changed, 554 insertions(+), 445 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/FieldReplaceInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/FieldReplaceInfo.java index 703721ef2..696a10af2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/FieldReplaceInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/FieldReplaceInfo.java @@ -20,9 +20,14 @@ package com.dtstack.flink.sql.side; +import com.dtstack.flink.sql.util.ReflectionUtils; import com.google.common.collect.HashBasedTable; import org.apache.commons.lang3.StringUtils; +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + /** * 用于记录转换之后的表和原来表直接字段的关联关系 * Date: 2018/8/30 @@ -78,12 +83,32 @@ public void setTargetTableAlias(String targetTableAlias) { * @param fieldName * @return */ - public String getTargetFieldName(String tableName, String fieldName){ + public String getTargetFieldName(String tableName, String fieldName) { String targetFieldName = mappingTable.get(tableName, fieldName); if(StringUtils.isNotBlank(targetFieldName)){ return targetFieldName; } + + /* Field field = ReflectionUtils.getDeclaredField(mappingTable, "backingMap"); + field.setAccessible(true); + HashMap> map = null; + try { + map = (HashMap) field.get(mappingTable); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + + if(map.size() == 1 && tableName.equalsIgnoreCase(targetTableAlias)){ + for(Map tmp : map.values()){ + targetFieldName = tmp.get(fieldName); + if(targetFieldName != null){ + return targetFieldName; + } + } + }*/ + + if(preNode == null){ return null; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java index 8a8fe21f6..c3a8656ff 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java @@ -20,6 +20,9 @@ package com.dtstack.flink.sql.side; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.HashBiMap; +import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlNode; import com.google.common.base.Strings; @@ -67,6 +70,18 @@ public class JoinInfo implements Serializable { private JoinType joinType; + private HashBiMap fieldRefInfo = HashBiMap.create(); + + /** + * 左表需要查询的字段信息和output的时候对应的列名称 + */ + private Map leftSelectFieldInfo = Maps.newHashMap(); + + /** + * 右表需要查询的字段信息和output的时候对应的列名称 + */ + private Map rightSelectFieldInfo = Maps.newHashMap(); + public String getSideTableName(){ if(leftIsSideTable){ return leftTableAlias; @@ -203,6 +218,43 @@ public void setLeftIsTmpTable(boolean leftIsTmpTable) { this.leftIsTmpTable = leftIsTmpTable; } + public HashBiMap getFieldRefInfo() { + return fieldRefInfo; + } + + public void setFieldRefInfo(HashBiMap fieldRefInfo) { + this.fieldRefInfo = fieldRefInfo; + } + + public Map getLeftSelectFieldInfo() { + return leftSelectFieldInfo; + } + + public void setLeftSelectFieldInfo(Map leftSelectFieldInfo) { + this.leftSelectFieldInfo = leftSelectFieldInfo; + } + + public Map getRightSelectFieldInfo() { + return rightSelectFieldInfo; + } + + public void setRightSelectFieldInfo(Map rightSelectFieldInfo) { + this.rightSelectFieldInfo = rightSelectFieldInfo; + } + + public HashBasedTable getTableFieldRef(){ + HashBasedTable mappingTable = HashBasedTable.create(); + getLeftSelectFieldInfo().forEach((key, value) -> { + mappingTable.put(getLeftTableAlias(), key, value); + }); + + getRightSelectFieldInfo().forEach((key, value) -> { + mappingTable.put(getRightTableAlias(), key, value); + }); + + return mappingTable; + } + @Override public String toString() { return "JoinInfo{" + diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index 40ebc22c4..1d2226fb1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -20,10 +20,10 @@ package com.dtstack.flink.sql.side; import com.dtstack.flink.sql.config.CalciteConfig; +import com.dtstack.flink.sql.util.ParseUtils; import com.dtstack.flink.sql.util.TableUtils; import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; +import com.google.common.collect.*; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.SqlBasicCall; @@ -83,8 +83,10 @@ public JoinNodeDealer(SideSQLParser sideSQLParser){ */ public JoinInfo dealJoinNode(SqlJoin joinNode, Set sideTableSet, Queue queueInfo, SqlNode parentWhere, - SqlNodeList parentSelectList, Set> joinFieldSet, - Map tableRef) { + SqlNodeList parentSelectList, + Set> joinFieldSet, + Map tableRef, + Map fieldRef) { SqlNode leftNode = joinNode.getLeft(); SqlNode rightNode = joinNode.getRight(); JoinType joinType = joinNode.getJoinType(); @@ -94,59 +96,27 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, Set sideTableSet, String rightTableName = ""; String rightTableAlias = ""; boolean leftTbisTmp = false; + //TODO 这个是啥东西 + HashBiMap fieldReplaceRef = HashBiMap.create(); //如果是连续join 判断是否已经处理过添加到执行队列 - Boolean needBuildTemp = false; extractJoinField(joinNode.getCondition(), joinFieldSet); if(leftNode.getKind() == IDENTIFIER){ leftTbName = leftNode.toString(); } else if (leftNode.getKind() == JOIN) { //处理连续join - Tuple2 nestJoinResult = dealNestJoin((SqlJoin) leftNode, sideTableSet, - queueInfo, parentWhere, parentSelectList, joinFieldSet, tableRef); - needBuildTemp = nestJoinResult.f0; - SqlBasicCall buildAs = TableUtils.buildAsNodeByJoinInfo(nestJoinResult.f1, null, null); - - if(needBuildTemp){ - //记录表之间的关联关系 - String newLeftTableName = buildAs.getOperands()[1].toString(); - Set fromTableNameSet = Sets.newHashSet(); - TableUtils.getFromTableInfo(joinNode.getLeft(), fromTableNameSet); - for(String tbTmp : fromTableNameSet){ - tableRef.put(tbTmp, newLeftTableName); - } - - //替换leftNode 为新的查询 - joinNode.setLeft(buildAs); - leftNode = buildAs; - - //替换select field 中的对应字段 - for(SqlNode sqlNode : parentSelectList.getList()){ - for(String tbTmp : fromTableNameSet) { - TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName); - } - } - - //替换where 中的条件相关 - for(String tbTmp : fromTableNameSet){ - TableUtils.replaceWhereCondition(parentWhere, tbTmp, newLeftTableName); - } - - leftTbisTmp = true; - - } + dealNestJoin(joinNode, sideTableSet, + queueInfo, parentWhere, parentSelectList, joinFieldSet, tableRef, fieldRef, parentSelectList, fieldReplaceRef); + leftNode = joinNode.getLeft(); - leftTbName = buildAs.getOperands()[0].toString(); - leftTbAlias = buildAs.getOperands()[1].toString(); + } - } else if (leftNode.getKind() == AS) { + if (leftNode.getKind() == AS) { AliasInfo aliasInfo = (AliasInfo) sideSQLParser.parseSql(leftNode, sideTableSet, queueInfo, parentWhere, parentSelectList); leftTbName = aliasInfo.getName(); leftTbAlias = aliasInfo.getAlias(); - } else { - throw new RuntimeException(String.format("---not deal node with type %s", leftNode.getKind().toString())); } boolean leftIsSide = checkIsSideTable(leftTbName, sideTableSet); @@ -161,10 +131,6 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, Set sideTableSet, throw new RuntimeException("side join not support join type of right[current support inner join and left join]"); } - if(leftNode.getKind() == JOIN && rightIsSide){ - needBuildTemp = true; - } - JoinInfo tableInfo = new JoinInfo(); tableInfo.setLeftTableName(leftTbName); tableInfo.setRightTableName(rightTableName); @@ -180,7 +146,6 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, Set sideTableSet, tableInfo.setRightTableAlias(rightTableAlias); } - TableUtils.replaceJoinFieldRefTableName(joinNode.getCondition(), tableRef); tableInfo.setLeftIsTmpTable(leftTbisTmp); tableInfo.setLeftIsSideTable(leftIsSide); @@ -188,51 +153,209 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, Set sideTableSet, tableInfo.setLeftNode(leftNode); tableInfo.setRightNode(rightNode); tableInfo.setJoinType(joinType); + tableInfo.setFieldRefInfo(fieldReplaceRef); + tableInfo.setCondition(joinNode.getCondition()); + TableUtils.replaceJoinFieldRefTableName(joinNode.getCondition(), fieldRef); - if(tableInfo.getLeftNode().getKind() != AS && needBuildTemp){ - extractTemporaryQuery(tableInfo.getLeftNode(), tableInfo.getLeftTableAlias(), (SqlBasicCall) parentWhere, - parentSelectList, queueInfo, joinFieldSet, tableRef); - }else { - SqlKind asNodeFirstKind = ((SqlBasicCall)tableInfo.getLeftNode()).operands[0].getKind(); - if(asNodeFirstKind == SELECT){ - queueInfo.offer(tableInfo.getLeftNode()); - tableInfo.setLeftNode(((SqlBasicCall)tableInfo.getLeftNode()).operands[1]); + //extract 需要查询的字段信息 + //TODO 抽取=========================== + if(rightIsSide){ + Set fromTableNameSet = Sets.newHashSet(); + TableUtils.getFromTableInfo(leftNode, fromTableNameSet); + Set extractCondition = Sets.newHashSet(); + extractWhereCondition(fromTableNameSet, (SqlBasicCall) parentWhere, extractCondition); + Set extractSelectField = extractSelectFields(parentSelectList, fromTableNameSet, tableRef); + Set fieldFromJoinCondition = extractSelectFieldFromJoinCondition(joinFieldSet, fromTableNameSet, tableRef); + + extractSelectField.addAll(extractCondition); + extractSelectField.addAll(fieldFromJoinCondition); + + Set rightFromTableNameSet = Sets.newHashSet(); + TableUtils.getFromTableInfo(rightNode, rightFromTableNameSet); + Set extractRightCondition = Sets.newHashSet(); + extractWhereCondition(rightFromTableNameSet, (SqlBasicCall) parentWhere, extractRightCondition); + Set rightExtractSelectField = extractSelectFields(parentSelectList, rightFromTableNameSet, tableRef); + Set rightFieldFromJoinCondition = extractSelectFieldFromJoinCondition(joinFieldSet, rightFromTableNameSet, tableRef); + rightExtractSelectField.addAll(extractRightCondition); + rightExtractSelectField.addAll(rightFieldFromJoinCondition); + + //重命名right 中和 left 重名的 + Map leftTbSelectField = Maps.newHashMap(); + Map rightTbSelectField = Maps.newHashMap(); + String newTableName = tableInfo.getNewTableAlias(); + + for(String tmpField : extractSelectField){ + String[] tmpFieldSplit = StringUtils.split(tmpField, '.'); + leftTbSelectField.put(tmpFieldSplit[1], tmpFieldSplit[1]); + fieldRef.put(tmpField, TableUtils.buildTableField(newTableName, tmpFieldSplit[1])); + } + + for(String tmpField : rightExtractSelectField){ + String[] tmpFieldSplit = StringUtils.split(tmpField, '.'); + String originalFieldName = tmpFieldSplit[1]; + String targetFieldName = originalFieldName; + if(leftTbSelectField.containsKey(originalFieldName)){ + targetFieldName = ParseUtils.dealDuplicateFieldName(leftTbSelectField, originalFieldName); + } + + rightTbSelectField.put(originalFieldName, targetFieldName); + fieldRef.put(tmpField, TableUtils.buildTableField(newTableName, targetFieldName)); } + + tableInfo.setLeftSelectFieldInfo(leftTbSelectField); + tableInfo.setRightSelectFieldInfo(rightTbSelectField); + } + + //========================================= + if(tableInfo.getLeftNode().getKind() != AS){ + return tableInfo; + } + + SqlKind asNodeFirstKind = ((SqlBasicCall)tableInfo.getLeftNode()).operands[0].getKind(); + if(asNodeFirstKind == SELECT){ + queueInfo.offer(tableInfo.getLeftNode()); + tableInfo.setLeftNode(((SqlBasicCall)tableInfo.getLeftNode()).operands[1]); + } + return tableInfo; } - //处理多层join - private Tuple2 dealNestJoin(SqlJoin joinNode, Set sideTableSet, + /** + * 处理多层join + * 判断左节点是否需要创建零时查询 + * （1）右节点是维表 + * （2）左节点不是 as 节点 + */ + private JoinInfo dealNestJoin(SqlJoin joinNode, Set sideTableSet, Queue queueInfo, SqlNode parentWhere, SqlNodeList selectList, Set> joinFieldSet, - Map tableRef){ - SqlNode rightNode = joinNode.getRight(); + Map tableRef, Map fieldRef, + SqlNodeList parentSelectList, + HashBiMap fieldReplaceRef){ + + SqlJoin leftJoinNode = (SqlJoin) joinNode.getLeft(); + SqlNode parentRightJoinNode = joinNode.getRight(); + SqlNode rightNode = leftJoinNode.getRight(); Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, parentWhere, selectList); - JoinInfo joinInfo = dealJoinNode(joinNode, sideTableSet, queueInfo, parentWhere, selectList, joinFieldSet, tableRef); + Tuple2 parentRightJoinInfo = parseRightNode(parentRightJoinNode, sideTableSet, queueInfo, parentWhere, selectList); + boolean parentRightIsSide = checkIsSideTable(parentRightJoinInfo.f0, sideTableSet); + + JoinInfo joinInfo = dealJoinNode(leftJoinNode, sideTableSet, queueInfo, parentWhere, selectList, joinFieldSet, tableRef, fieldRef); String rightTableName = rightTableNameAndAlias.f0; boolean rightIsSide = checkIsSideTable(rightTableName, sideTableSet); - boolean needBuildTemp = false; + SqlBasicCall buildAs = TableUtils.buildAsNodeByJoinInfo(joinInfo, null, null); - if(!rightIsSide){ - //右表不是维表的情况 - }else{ - //右边表是维表需要重新构建左表的临时查询 - queueInfo.offer(joinInfo); - needBuildTemp = true; + //TODO 抽取相同代码 + if(rightIsSide){ + addSideInfoToExeQueue(queueInfo, joinInfo, joinNode, parentSelectList, parentWhere, fieldReplaceRef, tableRef); + } + + SqlNode newLeftNode = joinNode.getLeft(); + + if(newLeftNode.getKind() != AS && parentRightIsSide){ + + String leftTbAlias = buildAs.getOperands()[1].toString(); + extractTemporaryQuery(newLeftNode, leftTbAlias, (SqlBasicCall) parentWhere, + parentSelectList, queueInfo, joinFieldSet, tableRef, fieldRef); + + //记录表之间的关联关系 + String newLeftTableName = buildAs.getOperands()[1].toString(); + Set fromTableNameSet = Sets.newHashSet(); + TableUtils.getFromTableInfo(newLeftNode, fromTableNameSet); + for(String tbTmp : fromTableNameSet){ + tableRef.put(tbTmp, newLeftTableName); + } + + //替换leftNode 为新的查询 + joinNode.setLeft(buildAs); + + //替换select field 中的对应字段 + for(SqlNode sqlNode : parentSelectList.getList()){ + for(String tbTmp : fromTableNameSet) { + //TODO + TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldReplaceRef, null); + } + } + + //替换where 中的条件相关 + for(String tbTmp : fromTableNameSet){ + TableUtils.replaceWhereCondition(parentWhere, tbTmp, newLeftTableName); + } + + } + + return joinInfo; + } + + /** + * 右边表是维表需要重新构建左表的临时查询 + * @param queueInfo + * @param joinInfo + * @param joinNode + * @param parentSelectList + * @param parentWhere + * @param fieldReplaceRef + * @param tableRef + */ + public void addSideInfoToExeQueue(Queue queueInfo, + JoinInfo joinInfo, + SqlJoin joinNode, + SqlNodeList parentSelectList, + SqlNode parentWhere, + HashBiMap fieldReplaceRef, + Map tableRef){ + //只处理维表 + if(!joinInfo.isRightIsSideTable()){ + return; + } + + SqlBasicCall buildAs = TableUtils.buildAsNodeByJoinInfo(joinInfo, null, null); + SqlNode leftJoinNode = joinNode.getLeft(); + queueInfo.offer(joinInfo); + //替换左表为新的表名称 + joinNode.setLeft(buildAs); + + String newLeftTableName = buildAs.getOperands()[1].toString(); + Set fromTableNameSet = Sets.newHashSet(); + TableUtils.getFromTableInfo(leftJoinNode, fromTableNameSet); + for(String tbTmp : fromTableNameSet){ + tableRef.put(tbTmp, newLeftTableName); + } + + //替换select field 中的对应字段 + //TODO + for(SqlNode sqlNode : parentSelectList.getList()){ + for(String tbTmp : fromTableNameSet) { + TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldReplaceRef, null); + } } - //return Tuple2.of(needBuildTemp, TableUtils.buildAsNodeByJoinInfo(joinInfo, null, null)); - return Tuple2.of(needBuildTemp, joinInfo); + //替换where 中的条件相关 + for(String tbTmp : fromTableNameSet){ + TableUtils.replaceWhereCondition(parentWhere, tbTmp, newLeftTableName); + } } + /** + * 抽取出中间查询表 + * @param node + * @param tableAlias + * @param parentWhere + * @param parentSelectList + * @param queueInfo + * @param joinFieldSet + * @param tableRef + * @return 源自段和新生成字段之间的映射关系 + */ private void extractTemporaryQuery(SqlNode node, String tableAlias, SqlBasicCall parentWhere, SqlNodeList parentSelectList, Queue queueInfo, Set> joinFieldSet, - Map tableRef){ + Map tableRef, + Map fieldRef){ try{ //父一级的where 条件中如果只和临时查询相关的条件都截取进来 Set fromTableNameSet = Sets.newHashSet(); @@ -246,8 +369,13 @@ private void extractTemporaryQuery(SqlNode node, String tableAlias, SqlBasicCall } Set extractSelectField = extractSelectFields(parentSelectList, fromTableNameSet, tableRef); - Set fieldFromJoinCondition = extractSelectFieldFromJoinCondition(joinFieldSet, fromTableNameSet); - String extractSelectFieldStr = buildSelectNode(extractSelectField, fieldFromJoinCondition); + Set fieldFromJoinCondition = extractSelectFieldFromJoinCondition(joinFieldSet, fromTableNameSet, tableRef); + Set newFields = buildSelectNode(extractSelectField, fieldFromJoinCondition); + String extractSelectFieldStr = StringUtils.join(newFields, ','); + + Map oldRefNewField = buildTmpTableFieldRefOriField(newFields, tableAlias); + fieldRef.putAll(oldRefNewField); + String extractConditionStr = buildCondition(extractCondition); String tmpSelectSql = String.format(SELECT_TEMP_SQL, @@ -260,9 +388,28 @@ private void extractTemporaryQuery(SqlNode node, String tableAlias, SqlBasicCall SqlBasicCall sqlBasicCall = buildAsSqlNode(tableAlias, sqlNode); queueInfo.offer(sqlBasicCall); + //替换select中的表结构 + //TODO + HashBiMap fieldReplaceRef = HashBiMap.create(); + for(SqlNode tmpSelect : parentSelectList.getList()){ + for(String tbTmp : fromTableNameSet) { + TableUtils.replaceSelectFieldTable(tmpSelect, tbTmp, tableAlias, fieldReplaceRef, null); + } + } + + //替换where 中的条件相关 + for(String tbTmp : fromTableNameSet){ + TableUtils.replaceWhereCondition(parentWhere, tbTmp, tableAlias); + } + + for(String tbTmp : fromTableNameSet){ + tableRef.put(tbTmp, tableAlias); + } + System.out.println("-------build temporary query-----------"); System.out.println(tmpSelectSql); System.out.println("---------------------------------------"); + }catch (Exception e){ e.printStackTrace(); throw new RuntimeException(e); @@ -287,12 +434,21 @@ private Set extractSelectFields(SqlNodeList parentSelectList, return extractFieldList; } - private Set extractSelectFieldFromJoinCondition(Set> joinFieldSet, Set fromTableNameSet){ + private Set extractSelectFieldFromJoinCondition(Set> joinFieldSet, + Set fromTableNameSet, + Map tableRef){ Set extractFieldList = Sets.newHashSet(); for(Tuple2 field : joinFieldSet){ if(fromTableNameSet.contains(field.f0)){ extractFieldList.add(field.f0 + "." + field.f1); } + + //TODO + if(tableRef.containsKey(field.f0)){ + if(fromTableNameSet.contains(tableRef.get(field.f0))){ + extractFieldList.add(tableRef.get(field.f0) + "." + field.f1); + } + } } return extractFieldList; @@ -447,14 +603,39 @@ public String buildCondition(List conditionList){ return " where " + StringUtils.join(conditionList, " AND "); } - public String buildSelectNode(Set extractSelectField, Set joinFieldSet){ + /** + * 构建抽取表的查询字段信息 + * 包括去除重复字段，名称相同的取别名 + * @param extractSelectField + * @param joinFieldSet + * @return + */ + public Set buildSelectNode(Set extractSelectField, Set joinFieldSet){ if(CollectionUtils.isEmpty(extractSelectField)){ throw new RuntimeException("no field is used"); } - Sets.SetView view = Sets.union(extractSelectField, joinFieldSet); + Sets.SetView view = Sets.union(extractSelectField, joinFieldSet); + Set newFieldSet = Sets.newHashSet(); + //为相同的列取别名 + HashBiMap refFieldMap = HashBiMap.create(); + for(String field : view){ + String[] fieldInfo = StringUtils.split(field, '.'); + String aliasName = fieldInfo[1]; + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(field); + if(refFieldMap.inverse().get(aliasName) != null){ + aliasName = ParseUtils.dealDuplicateFieldName(refFieldMap, aliasName); + stringBuilder.append(" as ") + .append(aliasName); + } + + refFieldMap.put(field, aliasName); - return StringUtils.join(view, ","); + newFieldSet.add(stringBuilder.toString()); + } + + return newFieldSet; } private boolean checkIsSideTable(String tableName, Set sideTableList){ @@ -474,6 +655,38 @@ private SqlBasicCall buildAsSqlNode(String internalTableName, SqlNode newSource) return new SqlBasicCall(operator, sqlNodes, sqlParserPos); } + /** + * 获取where中和指定表有关联的字段 + * @param fromTableNameSet + * @param parentWhere + * @param extractCondition + */ + private void extractWhereCondition(Set fromTableNameSet, SqlBasicCall parentWhere, Set extractCondition){ + + if(parentWhere == null){ + return; + } + + SqlKind kind = parentWhere.getKind(); + if(kind == AND){ + extractWhereCondition(fromTableNameSet, (SqlBasicCall) parentWhere.getOperands()[0], extractCondition); + extractWhereCondition(fromTableNameSet, (SqlBasicCall) parentWhere.getOperands()[1], extractCondition); + } else { + + Set fieldInfos = Sets.newHashSet(); + TableUtils.getConditionRefTable(parentWhere, fieldInfos); + fieldInfos.forEach(fieldInfo -> { + String[] splitInfo = StringUtils.split(fieldInfo, "."); + if(splitInfo.length == 2 && fromTableNameSet.contains(splitInfo[0])){ + extractCondition.add(fieldInfo); + } + }); + + } + + + } + /** * 检查关联的where 条件中的判断是否可以下移到新构建的子查询 @@ -504,8 +717,18 @@ private boolean checkAndRemoveWhereCondition(Set fromTableNameSet, return false; } else { + //条件表达式，如果该条件关联的表都是指定的表则移除 + Set fieldInfos = Sets.newHashSet(); + TableUtils.getConditionRefTable(parentWhere, fieldInfos); Set conditionRefTableNameSet = Sets.newHashSet(); - TableUtils.getConditionRefTable(parentWhere, conditionRefTableNameSet); + + fieldInfos.forEach(fieldInfo -> { + String[] splitInfo = StringUtils.split(fieldInfo, ","); + if(splitInfo.length == 2){ + conditionRefTableNameSet.add(splitInfo[1]); + } + }); + if(fromTableNameSet.containsAll(conditionRefTableNameSet)){ return true; @@ -574,6 +797,25 @@ private SqlIdentifier checkAndReplaceJoinCondition(SqlNode node, Map buildTmpTableFieldRefOriField(Set fieldSet, String newTableAliasName){ + Map refInfo = Maps.newConcurrentMap(); + for(String field : fieldSet){ + String[] fields = StringUtils.splitByWholeSeparator(field, "as"); + String oldKey = field; + String[] oldFieldInfo = StringUtils.splitByWholeSeparator(fields[0], "."); + String oldFieldName = oldFieldInfo.length == 2 ? oldFieldInfo[1] : oldFieldInfo[0]; + String newKey = fields.length == 2 ? newTableAliasName + "." + fields[1] : + newTableAliasName + "." + oldFieldName; + refInfo.put(oldKey, newKey); + } + + return refInfo; + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 0da1b9c81..80c1c3e21 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -29,10 +29,7 @@ import com.dtstack.flink.sql.util.ParseUtils; import com.dtstack.flink.sql.util.TableUtils; import com.google.common.base.Preconditions; -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import com.google.common.collect.*; import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlDataTypeSpec; @@ -64,12 +61,7 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.Set; +import java.util.*; import static org.apache.calcite.sql.SqlKind.*; @@ -130,8 +122,9 @@ public void exec(String sql, Map sideTableMap, StreamTabl List fieldNames = null; for(FieldReplaceInfo replaceInfo : replaceInfoList){ fieldNames = Lists.newArrayList(); - replaceFieldName(pollSqlNode, replaceInfo); - addAliasForFieldNode(pollSqlNode, fieldNames, replaceInfo.getMappingTable()); + //replaceFieldName(pollSqlNode, replaceInfo); + //TODO + //addAliasForFieldNode(pollSqlNode, fieldNames, replaceInfo.getMappingTable()); } } @@ -290,15 +283,17 @@ public AliasInfo parseASNode(SqlNode sqlNode) throws SqlParseException { return aliasInfo; } - public RowTypeInfo buildOutRowTypeInfo(List sideJoinFieldInfo, HashBasedTable mappingTable) { + public RowTypeInfo buildOutRowTypeInfo(List sideJoinFieldInfo, + HashBasedTable mappingTable) { TypeInformation[] sideOutTypes = new TypeInformation[sideJoinFieldInfo.size()]; String[] sideOutNames = new String[sideJoinFieldInfo.size()]; for (int i = 0; i < sideJoinFieldInfo.size(); i++) { FieldInfo fieldInfo = sideJoinFieldInfo.get(i); String tableName = fieldInfo.getTable(); String fieldName = fieldInfo.getFieldName(); - String mappingFieldName = ParseUtils.dealDuplicateFieldName(mappingTable, fieldName); - mappingTable.put(tableName, fieldName, mappingFieldName); + + String mappingFieldName = mappingTable.get(tableName, fieldName); + Preconditions.checkNotNull(mappingFieldName, fieldInfo + " not mapping any field! it may be frame bug"); sideOutTypes[i] = fieldInfo.getTypeInformation(); sideOutNames[i] = mappingFieldName; @@ -330,184 +325,9 @@ private TypeInformation convertTimeAttributeType(TypeInformation typeInformation return typeInformation; } - //需要考虑更多的情况 - private void replaceFieldName(SqlNode sqlNode, FieldReplaceInfo replaceInfo) { - SqlKind sqlKind = sqlNode.getKind(); - switch (sqlKind) { - case INSERT: - SqlNode sqlSource = ((SqlInsert) sqlNode).getSource(); - replaceFieldName(sqlSource, replaceInfo); - break; - case AS: - SqlNode asNode = ((SqlBasicCall) sqlNode).getOperands()[0]; - replaceFieldName(asNode, replaceInfo); - break; - case SELECT: - SqlSelect sqlSelect = (SqlSelect) filterNodeWithTargetName(sqlNode, replaceInfo.getTargetTableName()); - if(sqlSelect == null){ - return; - } - SqlNode sqlSource1 = sqlSelect.getFrom(); - if(sqlSource1.getKind() == AS){ - String tableName = ((SqlBasicCall)sqlSource1).getOperands()[0].toString(); - if(tableName.equalsIgnoreCase(replaceInfo.getTargetTableName())){ - SqlNodeList sqlSelectList = sqlSelect.getSelectList(); - SqlNode whereNode = sqlSelect.getWhere(); - SqlNodeList sqlGroup = sqlSelect.getGroup(); - - //TODO 暂时不处理having - SqlNode sqlHaving = sqlSelect.getHaving(); - - List newSelectNodeList = Lists.newArrayList(); - for( int i=0; i replaceNodeList = replaceSelectStarFieldName(selectNode, replaceInfo); - newSelectNodeList.addAll(replaceNodeList); - continue; - } - - SqlNode replaceNode = replaceSelectFieldName(selectNode, replaceInfo); - if(replaceNode == null){ - continue; - } - - //sqlSelectList.set(i, replaceNode); - newSelectNodeList.add(replaceNode); - } - SqlNodeList newSelectList = new SqlNodeList(newSelectNodeList, sqlSelectList.getParserPosition()); - sqlSelect.setSelectList(newSelectList); - - //where - if(whereNode != null){ - SqlNode[] sqlNodeList = ((SqlBasicCall)whereNode).getOperands(); - for(int i =0; i localTableCache, String table } if(table == null){ - throw new RuntimeException("not register table " + tableName); + throw new RuntimeException("not register table " + tableAlias); } return table; } - private List replaceSelectStarFieldName(SqlNode selectNode, FieldReplaceInfo replaceInfo){ - SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; - List sqlNodes = Lists.newArrayList(); - if(sqlIdentifier.isStar()){//处理 [* or table.*] - int identifierSize = sqlIdentifier.names.size(); - Collection columns = null; - if(identifierSize == 1){ - columns = replaceInfo.getMappingTable().values(); - }else{ - columns = replaceInfo.getMappingTable().row(sqlIdentifier.names.get(0)).values(); - } - - for(String colAlias : columns){ - SqlParserPos sqlParserPos = new SqlParserPos(0, 0); - List columnInfo = Lists.newArrayList(); - columnInfo.add(replaceInfo.getTargetTableAlias()); - columnInfo.add(colAlias); - SqlIdentifier sqlIdentifierAlias = new SqlIdentifier(columnInfo, sqlParserPos); - sqlNodes.add(sqlIdentifierAlias); - } - - return sqlNodes; - }else{ - throw new RuntimeException("is not a star select field." + selectNode); - } - } - - private SqlNode replaceSelectFieldName(SqlNode selectNode, FieldReplaceInfo replaceInfo) { - if (selectNode.getKind() == AS) { - SqlNode leftNode = ((SqlBasicCall) selectNode).getOperands()[0]; - SqlNode replaceNode = replaceSelectFieldName(leftNode, replaceInfo); - if (replaceNode != null) { - ((SqlBasicCall) selectNode).getOperands()[0] = replaceNode; - } - - return selectNode; - }else if(selectNode.getKind() == IDENTIFIER){ - SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; - - if(sqlIdentifier.names.size() == 1){ - return selectNode; - } - - //Same level mappingTable - String mappingFieldName = replaceInfo.getTargetFieldName(sqlIdentifier.getComponent(0).getSimple(), sqlIdentifier.getComponent(1).getSimple()); - if (mappingFieldName == null) { - throw new RuntimeException("can't find mapping fieldName:" + selectNode.toString() ); - } - - sqlIdentifier = sqlIdentifier.setName(0, replaceInfo.getTargetTableAlias()); - sqlIdentifier = sqlIdentifier.setName(1, mappingFieldName); - return sqlIdentifier; - }else if(selectNode.getKind() == LITERAL || selectNode.getKind() == LITERAL_CHAIN){//字面含义 - return selectNode; - }else if( AGGREGATE.contains(selectNode.getKind()) - || AVG_AGG_FUNCTIONS.contains(selectNode.getKind()) - || COMPARISON.contains(selectNode.getKind()) - || selectNode.getKind() == OTHER_FUNCTION - || selectNode.getKind() == DIVIDE - || selectNode.getKind() == CAST - || selectNode.getKind() == TRIM - || selectNode.getKind() == TIMES - || selectNode.getKind() == PLUS - || selectNode.getKind() == NOT_IN - || selectNode.getKind() == OR - || selectNode.getKind() == AND - || selectNode.getKind() == MINUS - || selectNode.getKind() == TUMBLE - || selectNode.getKind() == TUMBLE_START - || selectNode.getKind() == TUMBLE_END - || selectNode.getKind() == SESSION - || selectNode.getKind() == SESSION_START - || selectNode.getKind() == SESSION_END - || selectNode.getKind() == HOP - || selectNode.getKind() == HOP_START - || selectNode.getKind() == HOP_END - || selectNode.getKind() == BETWEEN - || selectNode.getKind() == IS_NULL - || selectNode.getKind() == IS_NOT_NULL - || selectNode.getKind() == CONTAINS - || selectNode.getKind() == TIMESTAMP_ADD - || selectNode.getKind() == TIMESTAMP_DIFF - || selectNode.getKind() == LIKE - - ){ - SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; - for(int i=0; i localTableCache, leftScopeChild.setAlias(joinInfo.getLeftTableAlias()); leftScopeChild.setTableName(joinInfo.getLeftTableName()); - SqlKind sqlKind = joinInfo.getLeftNode().getKind(); - if(sqlKind == AS){ - dealAsSourceTable(tableEnv, joinInfo.getLeftNode(), localTableCache, replaceInfoList); - } - Table leftTable = getTableFromCache(localTableCache, joinInfo.getLeftTableAlias(), joinInfo.getLeftTableName()); RowTypeInfo leftTypeInfo = new RowTypeInfo(leftTable.getSchema().getTypes(), leftTable.getSchema().getColumnNames()); leftScopeChild.setRowTypeInfo(leftTypeInfo); @@ -791,8 +469,13 @@ private void joinFun(Object pollObj, Map localTableCache, joinScope.addScope(leftScopeChild); joinScope.addScope(rightScopeChild); + HashBasedTable mappingTable = ((JoinInfo) pollObj).getTableFieldRef(); + //获取两个表的所有字段 + //TODO 抽取 List sideJoinFieldInfo = ParserJoinField.getRowTypeInfo(joinInfo.getSelectNode(), joinScope, true); + //通过join的查询字段信息过滤出需要的字段信息 + sideJoinFieldInfo.removeIf(tmpFieldInfo -> mappingTable.get(tmpFieldInfo.getTable(), tmpFieldInfo.getFieldName()) == null); String leftTableAlias = joinInfo.getLeftTableAlias(); Table targetTable = localTableCache.get(leftTableAlias); @@ -824,8 +507,6 @@ private void joinFun(Object pollObj, Map localTableCache, dsOut = SideAsyncOperator.getSideJoinDataStream(adaptStream, sideTableInfo.getType(), localSqlPluginPath, typeInfo, joinInfo, sideJoinFieldInfo, sideTableInfo); } - // TODO 将嵌套表中的字段传递过去, 去除冗余的ROWtime - HashBasedTable mappingTable = HashBasedTable.create(); RowTypeInfo sideOutTypeInfo = buildOutRowTypeInfo(sideJoinFieldInfo, mappingTable); dsOut.getTransformation().setOutputType(sideOutTypeInfo); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java index d399b533c..2c5ccd9ca 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java @@ -37,14 +37,12 @@ package com.dtstack.flink.sql.util; import com.google.common.collect.HashBasedTable; +import com.google.common.collect.HashBiMap; import org.apache.calcite.sql.*; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import static org.apache.calcite.sql.SqlKind.*; @@ -145,11 +143,42 @@ public static void fillFieldNameMapping(HashBasedTable m public static String dealDuplicateFieldName(HashBasedTable mappingTable, String fieldName) { String mappingFieldName = fieldName; - int index = 0; + int index = 1; while (!mappingTable.column(mappingFieldName).isEmpty()) { - mappingFieldName = mappingFieldName + index; + mappingFieldName = suffixWithChar(fieldName, '0', index); index++; } return mappingFieldName; } + + public static String dealDuplicateFieldName(HashBiMap refFieldMap, String fieldName) { + String mappingFieldName = fieldName; + int index = 1; + while (refFieldMap.inverse().get(mappingFieldName) != null ) { + mappingFieldName = suffixWithChar(fieldName, '0', index); + index++; + } + + return mappingFieldName; + } + + public static String dealDuplicateFieldName(Map refFieldMap, String fieldName) { + String mappingFieldName = fieldName; + int index = 1; + while (refFieldMap.containsKey(mappingFieldName)){ + mappingFieldName = suffixWithChar(fieldName, '0', index); + index++; + } + + return mappingFieldName; + } + + public static String suffixWithChar(String str, char padChar, int repeat){ + StringBuilder stringBuilder = new StringBuilder(str); + for(int i=0; i fieldMapping = joinInfo.getTableFieldRef(); + Map leftFieldMapping = fieldMapping.row(joinInfo.getLeftTableAlias()); + Map rightFieldMapping = fieldMapping.row(joinInfo.getRightTableAlias()); + + for(SqlNode oneSelectNode : sqlNode.getSelectList()){ + replaceSelectFieldTable(oneSelectNode, joinInfo.getLeftTableAlias(), newAliasName, null ,leftFieldMapping); + replaceSelectFieldTable(oneSelectNode, joinInfo.getRightTableAlias(), newAliasName, null , rightFieldMapping); + } + + //where中的条件属性为新的表名称和字段 + replaceWhereCondition(); sqlNode.setFrom(sqlBasicCall); } @@ -277,10 +294,24 @@ public static void getFromTableInfo(SqlNode fromTable, Set tableNameSet) } } - public static void replaceSelectFieldTable(SqlNode selectNode, String oldTbName, String newTbName) { + /** + * 替换select 中的字段信息 + * 如果mappingTable 非空则从该参数获取字段的映射 + * 如果mappingTable 为空则根据是否存在新生成字段 + * @param selectNode + * @param oldTbName + * @param newTbName + * @param fieldReplaceRef + * @param mappingTable + */ + public static void replaceSelectFieldTable(SqlNode selectNode, + String oldTbName, + String newTbName, + HashBiMap fieldReplaceRef, + Map mappingTable) { if (selectNode.getKind() == AS) { SqlNode leftNode = ((SqlBasicCall) selectNode).getOperands()[0]; - replaceSelectFieldTable(leftNode, oldTbName, newTbName); + replaceSelectFieldTable(leftNode, oldTbName, newTbName, fieldReplaceRef, mappingTable); }else if(selectNode.getKind() == IDENTIFIER){ SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; @@ -289,9 +320,9 @@ public static void replaceSelectFieldTable(SqlNode selectNode, String oldTbName, return ; } - if(oldTbName.equalsIgnoreCase(((SqlIdentifier)selectNode).names.get(0))){ - SqlIdentifier newField = ((SqlIdentifier)selectNode).setName(0, newTbName); - ((SqlIdentifier)selectNode).assignNamesFrom(newField); + String fieldTableName = sqlIdentifier.names.get(0); + if(oldTbName.equalsIgnoreCase(fieldTableName)){ + replaceOneSelectField(sqlIdentifier, newTbName, oldTbName, fieldReplaceRef, mappingTable); } }else if(selectNode.getKind() == LITERAL || selectNode.getKind() == LITERAL_CHAIN){//字面含义 @@ -338,7 +369,7 @@ public static void replaceSelectFieldTable(SqlNode selectNode, String oldTbName, continue; } - replaceSelectFieldTable(sqlNode, oldTbName, newTbName); + replaceSelectFieldTable(sqlNode, oldTbName, newTbName, fieldReplaceRef, mappingTable); } }else if(selectNode.getKind() == CASE){ @@ -349,16 +380,16 @@ public static void replaceSelectFieldTable(SqlNode selectNode, String oldTbName, for(int i=0; i fieldReplaceRef, + Map mappingTable){ + SqlIdentifier newField = sqlIdentifier.setName(0, newTbName); + String fieldName = sqlIdentifier.names.get(1); + String fieldKey = oldTbName + "_" + fieldName; + + if(mappingTable != null){ + String mappingFieldName = mappingTable.get(fieldName); + Preconditions.checkNotNull(mappingFieldName, "can't get any field from mappingTable with oldFieldName " + fieldName); + newField = newField.setName(1, mappingFieldName); + sqlIdentifier.assignNamesFrom(newField); + return; + } + + if(!fieldReplaceRef.containsKey(fieldKey)){ + if(fieldReplaceRef.inverse().get(fieldName) != null){ + //换一个名字 + String mappingFieldName = ParseUtils.dealDuplicateFieldName(fieldReplaceRef, fieldName); + newField = newField.setName(1, mappingFieldName); + fieldReplaceRef.put(fieldKey, mappingFieldName); + } else { + fieldReplaceRef.put(fieldKey, fieldName); + } + }else { + newField = newField.setName(1, fieldReplaceRef.get(fieldKey)); + } + + sqlIdentifier.assignNamesFrom(newField); + } + /** * 替换另外join 表的指定表名为新关联处理的表名称 * @param condition - * @param tableRef + * @param oldTabFieldRefNew */ - public static void replaceJoinFieldRefTableName(SqlNode condition, Map tableRef){ + public static void replaceJoinFieldRefTableName(SqlNode condition, Map oldTabFieldRefNew){ SqlKind joinKind = condition.getKind(); if( joinKind == AND || joinKind == EQUALS ){ - replaceJoinFieldRefTableName(((SqlBasicCall)condition).operands[0], tableRef); - replaceJoinFieldRefTableName(((SqlBasicCall)condition).operands[1], tableRef); + replaceJoinFieldRefTableName(((SqlBasicCall)condition).operands[0], oldTabFieldRefNew); + replaceJoinFieldRefTableName(((SqlBasicCall)condition).operands[1], oldTabFieldRefNew); }else{ Preconditions.checkState(((SqlIdentifier)condition).names.size() == 2, "join condition must be format table.field"); String fieldRefTable = ((SqlIdentifier)condition).names.get(0); - String targetTableName = TableUtils.getTargetRefTable(tableRef, fieldRefTable); - if(StringUtils.isNotBlank(targetTableName) && !fieldRefTable.equalsIgnoreCase(targetTableName)){ - SqlIdentifier newField = ((SqlIdentifier)condition).setName(0, targetTableName); + String targetFieldName = TableUtils.getTargetRefField(oldTabFieldRefNew, condition.toString()); + + if(StringUtils.isNotBlank(targetFieldName)){ + String[] fieldSplits = StringUtils.split(targetFieldName, "."); + SqlIdentifier newField = ((SqlIdentifier)condition).setName(0, fieldSplits[0]); + newField = newField.setName(1, fieldSplits[1]); ((SqlIdentifier)condition).assignNamesFrom(newField); } } @@ -401,6 +468,18 @@ public static String getTargetRefTable(Map refTableMap, String t return preTableName; } + public static String getTargetRefField(Map refFieldMap, String currFieldName){ + String targetFieldName = null; + String preFieldName; + + do { + preFieldName = targetFieldName == null ? currFieldName : targetFieldName; + targetFieldName = refFieldMap.get(preFieldName); + } while (targetFieldName != null); + + return preFieldName; + } + public static void replaceWhereCondition(SqlNode parentWhere, String oldTbName, String newTbName){ if(parentWhere == null){ @@ -490,18 +569,13 @@ private static void replaceConditionNode(SqlNode selectNode, String oldTbName, S /** * 获取条件中关联的表信息 * @param selectNode - * @param tableNameSet + * @param fieldInfos */ - public static void getConditionRefTable(SqlNode selectNode, Set tableNameSet) { + public static void getConditionRefTable(SqlNode selectNode, Set fieldInfos) { if(selectNode.getKind() == IDENTIFIER){ SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; - if(sqlIdentifier.names.size() == 1){ - return; - } - - String tableName = sqlIdentifier.names.asList().get(0); - tableNameSet.add(tableName); + fieldInfos.add(sqlIdentifier.toString()); return; }else if(selectNode.getKind() == LITERAL || selectNode.getKind() == LITERAL_CHAIN){//字面含义 return; @@ -547,7 +621,7 @@ public static void getConditionRefTable(SqlNode selectNode, Set tableNam continue; } - getConditionRefTable(sqlNode, tableNameSet); + getConditionRefTable(sqlNode, fieldInfos); } return; @@ -558,4 +632,10 @@ public static void getConditionRefTable(SqlNode selectNode, Set tableNam throw new RuntimeException(String.format("not support node kind of %s to replace name now.", selectNode.getKind())); } } + + public static String buildTableField(String tableName, String fieldName){ + return String.format("%s.%s", tableName, fieldName); + } + + } From 48cb34ede5f5fa5f8373486d24cdb2d2c610ef14 Mon Sep 17 00:00:00 2001 From: xuchao Date: Sun, 22 Mar 2020 21:55:10 +0800 Subject: [PATCH 163/523] =?UTF-8?q?=E5=88=9D=E6=AD=A5=E9=87=8D=E6=9E=84=20?= =?UTF-8?q?sql=E8=AF=AD=E5=8F=A5=E8=A7=A3=E6=9E=90=E9=80=BB=E8=BE=91?= =?UTF-8?q?=EF=BC=9B=20=E5=AD=97=E6=AE=B5=E6=9B=BF=E6=8D=A2=E5=92=8C?= =?UTF-8?q?=E9=87=8D=E5=91=BD=E5=90=8D=E7=9B=B8=E5=85=B3=E7=9A=84=E9=80=BB?= =?UTF-8?q?=E8=BE=91=E5=9C=A8=E8=A7=A3=E6=9E=90=E9=98=B6=E6=AE=B5=E5=AE=8C?= =?UTF-8?q?=E6=88=90=EF=BC=9B=20=E7=9B=AE=E6=A0=87=EF=BC=9A=E7=BB=8F?= =?UTF-8?q?=E8=BF=87=E8=A7=A3=E6=9E=90sql=E4=B9=8B=E5=90=8E=E7=9A=84?= =?UTF-8?q?=E6=89=80=E6=9C=89sql=E6=98=AF=E7=A1=AE=E5=AE=9A=E7=9A=84?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/FieldReplaceInfo.java | 24 --------- .../flink/sql/side/JoinNodeDealer.java | 6 +-- .../dtstack/flink/sql/side/SideSqlExec.java | 49 +++---------------- .../dtstack/flink/sql/util/TableUtils.java | 36 +++++--------- 4 files changed, 23 insertions(+), 92 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/FieldReplaceInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/FieldReplaceInfo.java index 696a10af2..dfab231ca 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/FieldReplaceInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/FieldReplaceInfo.java @@ -20,13 +20,9 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.util.ReflectionUtils; import com.google.common.collect.HashBasedTable; import org.apache.commons.lang3.StringUtils; -import java.lang.reflect.Field; -import java.util.HashMap; -import java.util.Map; /** * 用于记录转换之后的表和原来表直接字段的关联关系 @@ -89,26 +85,6 @@ public String getTargetFieldName(String tableName, String fieldName) { return targetFieldName; } - - /* Field field = ReflectionUtils.getDeclaredField(mappingTable, "backingMap"); - field.setAccessible(true); - HashMap> map = null; - try { - map = (HashMap) field.get(mappingTable); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } - - if(map.size() == 1 && tableName.equalsIgnoreCase(targetTableAlias)){ - for(Map tmp : map.values()){ - targetFieldName = tmp.get(fieldName); - if(targetFieldName != null){ - return targetFieldName; - } - } - }*/ - - if(preNode == null){ return null; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index 1d2226fb1..b8afad67c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -277,7 +277,7 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, Set sideTableSet, for(SqlNode sqlNode : parentSelectList.getList()){ for(String tbTmp : fromTableNameSet) { //TODO - TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldReplaceRef, null); + TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldReplaceRef); } } @@ -330,7 +330,7 @@ public void addSideInfoToExeQueue(Queue queueInfo, //TODO for(SqlNode sqlNode : parentSelectList.getList()){ for(String tbTmp : fromTableNameSet) { - TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldReplaceRef, null); + TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldReplaceRef); } } @@ -393,7 +393,7 @@ private void extractTemporaryQuery(SqlNode node, String tableAlias, SqlBasicCall HashBiMap fieldReplaceRef = HashBiMap.create(); for(SqlNode tmpSelect : parentSelectList.getList()){ for(String tbTmp : fromTableNameSet) { - TableUtils.replaceSelectFieldTable(tmpSelect, tbTmp, tableAlias, fieldReplaceRef, null); + TableUtils.replaceSelectFieldTable(tmpSelect, tbTmp, tableAlias, fieldReplaceRef); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 80c1c3e21..0a9668916 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -108,25 +108,11 @@ public void exec(String sql, Map sideTableMap, StreamTabl Queue exeQueue = sideSQLParser.getExeQueue(sql, sideTableMap.keySet()); Object pollObj = null; - //need clean - boolean preIsSideJoin = false; - List replaceInfoList = Lists.newArrayList(); - while((pollObj = exeQueue.poll()) != null){ if(pollObj instanceof SqlNode){ SqlNode pollSqlNode = (SqlNode) pollObj; - if(preIsSideJoin){ - preIsSideJoin = false; - List fieldNames = null; - for(FieldReplaceInfo replaceInfo : replaceInfoList){ - fieldNames = Lists.newArrayList(); - //replaceFieldName(pollSqlNode, replaceInfo); - //TODO - //addAliasForFieldNode(pollSqlNode, fieldNames, replaceInfo.getMappingTable()); - } - } if(pollSqlNode.getKind() == INSERT){ System.out.println("----------real exec sql-----------" ); @@ -137,7 +123,7 @@ public void exec(String sql, Map sideTableMap, StreamTabl } }else if(pollSqlNode.getKind() == AS){ - dealAsSourceTable(tableEnv, pollSqlNode, tableCache, replaceInfoList); + dealAsSourceTable(tableEnv, pollSqlNode, tableCache); } else if (pollSqlNode.getKind() == WITH_ITEM) { SqlWithItem sqlWithItem = (SqlWithItem) pollSqlNode; @@ -166,8 +152,7 @@ public void exec(String sql, Map sideTableMap, StreamTabl }else if (pollObj instanceof JoinInfo){ System.out.println("----------exec join info----------"); System.out.println(pollObj.toString()); - preIsSideJoin = true; - joinFun(pollObj, localTableCache, sideTableMap, tableEnv, replaceInfoList); + joinFun(pollObj, localTableCache, sideTableMap, tableEnv); } } @@ -401,8 +386,7 @@ public List getConditionFields(SqlNode conditionNode, String specifyTabl protected void dealAsSourceTable(StreamTableEnvironment tableEnv, SqlNode pollSqlNode, - Map tableCache, - List replaceInfoList) throws SqlParseException { + Map tableCache) throws SqlParseException { AliasInfo aliasInfo = parseASNode(pollSqlNode); if (localTableCache.containsKey(aliasInfo.getName())) { @@ -424,19 +408,13 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, Set fromTableNameSet = Sets.newHashSet(); SqlNode fromNode = ((SqlBasicCall)pollSqlNode).getOperands()[0]; TableUtils.getFromTableInfo(fromNode, fromTableNameSet); - for(FieldReplaceInfo tmp : replaceInfoList){ - if(fromTableNameSet.contains(tmp.getTargetTableName()) - || fromTableNameSet.contains(tmp.getTargetTableAlias())){ - fieldReplaceInfo.setPreNode(tmp); - break; - } - } - replaceInfoList.add(fieldReplaceInfo); + } - private void joinFun(Object pollObj, Map localTableCache, - Map sideTableMap, StreamTableEnvironment tableEnv, - List replaceInfoList) throws Exception{ + private void joinFun(Object pollObj, + Map localTableCache, + Map sideTableMap, + StreamTableEnvironment tableEnv) throws Exception{ JoinInfo joinInfo = (JoinInfo) pollObj; JoinScope joinScope = new JoinScope(); @@ -518,17 +496,6 @@ private void joinFun(Object pollObj, Map localTableCache, replaceInfo.setTargetTableName(targetTableName); replaceInfo.setTargetTableAlias(targetTableAlias); - //判断之前是不是被替换过,被替换过则设置之前的替换信息作为上一个节点 - for(FieldReplaceInfo tmp : replaceInfoList){ - if(tmp.getTargetTableName().equalsIgnoreCase(joinInfo.getLeftTableName()) - ||tmp.getTargetTableName().equalsIgnoreCase(joinInfo.getLeftTableAlias())){ - replaceInfo.setPreNode(tmp); - break; - } - } - - replaceInfoList.add(replaceInfo); - if (!tableEnv.isRegistered(joinInfo.getNewTableName())){ Table joinTable = tableEnv.fromDataStream(dsOut); tableEnv.registerTable(joinInfo.getNewTableName(), joinTable); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 978af2df3..46eff4b6c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -256,13 +256,14 @@ public static void replaceFromNodeForJoin(JoinInfo joinInfo, SqlSelect sqlNode) Map leftFieldMapping = fieldMapping.row(joinInfo.getLeftTableAlias()); Map rightFieldMapping = fieldMapping.row(joinInfo.getRightTableAlias()); - for(SqlNode oneSelectNode : sqlNode.getSelectList()){ + /* for(SqlNode oneSelectNode : sqlNode.getSelectList()){ replaceSelectFieldTable(oneSelectNode, joinInfo.getLeftTableAlias(), newAliasName, null ,leftFieldMapping); replaceSelectFieldTable(oneSelectNode, joinInfo.getRightTableAlias(), newAliasName, null , rightFieldMapping); - } + }*/ //where中的条件属性为新的表名称和字段 - replaceWhereCondition(); + FieldReplaceUtil.replaceFieldName(sqlNode, joinInfo.getLeftTableAlias(), newAliasName, leftFieldMapping); + FieldReplaceUtil.replaceFieldName(sqlNode, joinInfo.getRightTableAlias(), newAliasName, rightFieldMapping); sqlNode.setFrom(sqlBasicCall); } @@ -296,22 +297,18 @@ public static void getFromTableInfo(SqlNode fromTable, Set tableNameSet) /** * 替换select 中的字段信息 - * 如果mappingTable 非空则从该参数获取字段的映射 - * 如果mappingTable 为空则根据是否存在新生成字段 * @param selectNode * @param oldTbName * @param newTbName * @param fieldReplaceRef - * @param mappingTable */ public static void replaceSelectFieldTable(SqlNode selectNode, String oldTbName, String newTbName, - HashBiMap fieldReplaceRef, - Map mappingTable) { + HashBiMap fieldReplaceRef) { if (selectNode.getKind() == AS) { SqlNode leftNode = ((SqlBasicCall) selectNode).getOperands()[0]; - replaceSelectFieldTable(leftNode, oldTbName, newTbName, fieldReplaceRef, mappingTable); + replaceSelectFieldTable(leftNode, oldTbName, newTbName, fieldReplaceRef); }else if(selectNode.getKind() == IDENTIFIER){ SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; @@ -322,7 +319,7 @@ public static void replaceSelectFieldTable(SqlNode selectNode, String fieldTableName = sqlIdentifier.names.get(0); if(oldTbName.equalsIgnoreCase(fieldTableName)){ - replaceOneSelectField(sqlIdentifier, newTbName, oldTbName, fieldReplaceRef, mappingTable); + replaceOneSelectField(sqlIdentifier, newTbName, oldTbName, fieldReplaceRef); } }else if(selectNode.getKind() == LITERAL || selectNode.getKind() == LITERAL_CHAIN){//字面含义 @@ -369,7 +366,7 @@ public static void replaceSelectFieldTable(SqlNode selectNode, continue; } - replaceSelectFieldTable(sqlNode, oldTbName, newTbName, fieldReplaceRef, mappingTable); + replaceSelectFieldTable(sqlNode, oldTbName, newTbName, fieldReplaceRef); } }else if(selectNode.getKind() == CASE){ @@ -380,16 +377,16 @@ public static void replaceSelectFieldTable(SqlNode selectNode, for(int i=0; i fieldReplaceRef, - Map mappingTable){ + HashBiMap fieldReplaceRef){ SqlIdentifier newField = sqlIdentifier.setName(0, newTbName); String fieldName = sqlIdentifier.names.get(1); String fieldKey = oldTbName + "_" + fieldName; - if(mappingTable != null){ - String mappingFieldName = mappingTable.get(fieldName); - Preconditions.checkNotNull(mappingFieldName, "can't get any field from mappingTable with oldFieldName " + fieldName); - newField = newField.setName(1, mappingFieldName); - sqlIdentifier.assignNamesFrom(newField); - return; - } - if(!fieldReplaceRef.containsKey(fieldKey)){ if(fieldReplaceRef.inverse().get(fieldName) != null){ //换一个名字 From 8e2e91d6307f51de1bbfaa41e2e0711e2f1073e4 Mon Sep 17 00:00:00 2001 From: xuchao Date: Sun, 22 Mar 2020 22:42:52 +0800 Subject: [PATCH 164/523] =?UTF-8?q?=E5=88=9D=E6=AD=A5=E9=87=8D=E6=9E=84=20?= =?UTF-8?q?sql=E8=AF=AD=E5=8F=A5=E8=A7=A3=E6=9E=90=E9=80=BB=E8=BE=91?= =?UTF-8?q?=EF=BC=9B=20=E5=AD=97=E6=AE=B5=E6=9B=BF=E6=8D=A2=E5=92=8C?= =?UTF-8?q?=E9=87=8D=E5=91=BD=E5=90=8D=E7=9B=B8=E5=85=B3=E7=9A=84=E9=80=BB?= =?UTF-8?q?=E8=BE=91=E5=9C=A8=E8=A7=A3=E6=9E=90=E9=98=B6=E6=AE=B5=E5=AE=8C?= =?UTF-8?q?=E6=88=90=EF=BC=9B=20=E7=9B=AE=E6=A0=87=EF=BC=9A=E7=BB=8F?= =?UTF-8?q?=E8=BF=87=E8=A7=A3=E6=9E=90sql=E4=B9=8B=E5=90=8E=E7=9A=84?= =?UTF-8?q?=E6=89=80=E6=9C=89sql=E6=98=AF=E7=A1=AE=E5=AE=9A=E7=9A=84?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/JoinInfo.java | 23 --- .../flink/sql/side/JoinNodeDealer.java | 192 ++++++++++-------- .../dtstack/flink/sql/util/TableUtils.java | 2 + 3 files changed, 104 insertions(+), 113 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java index c3a8656ff..8254b763e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java @@ -21,7 +21,6 @@ package com.dtstack.flink.sql.side; import com.google.common.collect.HashBasedTable; -import com.google.common.collect.HashBiMap; import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlNode; @@ -34,7 +33,6 @@ * Join信息 * Date: 2018/7/24 * Company: www.dtstack.com - * * @author xuchao */ @@ -45,8 +43,6 @@ public class JoinInfo implements Serializable { //左表是否是维表 private boolean leftIsSideTable; - private boolean leftIsTmpTable = false; - //右表是否是维表 private boolean rightIsSideTable; @@ -70,8 +66,6 @@ public class JoinInfo implements Serializable { private JoinType joinType; - private HashBiMap fieldRefInfo = HashBiMap.create(); - /** * 左表需要查询的字段信息和output的时候对应的列名称 */ @@ -210,22 +204,6 @@ public void setJoinType(JoinType joinType) { this.joinType = joinType; } - public boolean isLeftIsTmpTable() { - return leftIsTmpTable; - } - - public void setLeftIsTmpTable(boolean leftIsTmpTable) { - this.leftIsTmpTable = leftIsTmpTable; - } - - public HashBiMap getFieldRefInfo() { - return fieldRefInfo; - } - - public void setFieldRefInfo(HashBiMap fieldRefInfo) { - this.fieldRefInfo = fieldRefInfo; - } - public Map getLeftSelectFieldInfo() { return leftSelectFieldInfo; } @@ -259,7 +237,6 @@ public HashBasedTable getTableFieldRef(){ public String toString() { return "JoinInfo{" + "leftIsSideTable=" + leftIsSideTable + - ", leftIsTmpTable=" + leftIsTmpTable + ", rightIsSideTable=" + rightIsSideTable + ", leftTableName='" + leftTableName + '\'' + ", leftTableAlias='" + leftTableAlias + '\'' + diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index b8afad67c..d42bf6774 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -81,12 +81,15 @@ public JoinNodeDealer(SideSQLParser sideSQLParser){ * @param tableRef 存储构建临时表查询后源表和新表之间的关联关系 * @return */ - public JoinInfo dealJoinNode(SqlJoin joinNode, Set sideTableSet, - Queue queueInfo, SqlNode parentWhere, + public JoinInfo dealJoinNode(SqlJoin joinNode, + Set sideTableSet, + Queue queueInfo, + SqlNode parentWhere, SqlNodeList parentSelectList, Set> joinFieldSet, Map tableRef, Map fieldRef) { + SqlNode leftNode = joinNode.getLeft(); SqlNode rightNode = joinNode.getRight(); JoinType joinType = joinNode.getJoinType(); @@ -95,8 +98,8 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, Set sideTableSet, String leftTbAlias = ""; String rightTableName = ""; String rightTableAlias = ""; - boolean leftTbisTmp = false; - //TODO 这个是啥东西 + + //TODO 含义需要更明确 HashBiMap fieldReplaceRef = HashBiMap.create(); //如果是连续join 判断是否已经处理过添加到执行队列 @@ -109,14 +112,12 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, Set sideTableSet, dealNestJoin(joinNode, sideTableSet, queueInfo, parentWhere, parentSelectList, joinFieldSet, tableRef, fieldRef, parentSelectList, fieldReplaceRef); leftNode = joinNode.getLeft(); - } if (leftNode.getKind() == AS) { AliasInfo aliasInfo = (AliasInfo) sideSQLParser.parseSql(leftNode, sideTableSet, queueInfo, parentWhere, parentSelectList); leftTbName = aliasInfo.getName(); leftTbAlias = aliasInfo.getAlias(); - } boolean leftIsSide = checkIsSideTable(leftTbName, sideTableSet); @@ -146,69 +147,20 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, Set sideTableSet, tableInfo.setRightTableAlias(rightTableAlias); } - - tableInfo.setLeftIsTmpTable(leftTbisTmp); tableInfo.setLeftIsSideTable(leftIsSide); tableInfo.setRightIsSideTable(rightIsSide); tableInfo.setLeftNode(leftNode); tableInfo.setRightNode(rightNode); tableInfo.setJoinType(joinType); - tableInfo.setFieldRefInfo(fieldReplaceRef); tableInfo.setCondition(joinNode.getCondition()); TableUtils.replaceJoinFieldRefTableName(joinNode.getCondition(), fieldRef); //extract 需要查询的字段信息 - //TODO 抽取=========================== if(rightIsSide){ - Set fromTableNameSet = Sets.newHashSet(); - TableUtils.getFromTableInfo(leftNode, fromTableNameSet); - Set extractCondition = Sets.newHashSet(); - extractWhereCondition(fromTableNameSet, (SqlBasicCall) parentWhere, extractCondition); - Set extractSelectField = extractSelectFields(parentSelectList, fromTableNameSet, tableRef); - Set fieldFromJoinCondition = extractSelectFieldFromJoinCondition(joinFieldSet, fromTableNameSet, tableRef); - - extractSelectField.addAll(extractCondition); - extractSelectField.addAll(fieldFromJoinCondition); - - Set rightFromTableNameSet = Sets.newHashSet(); - TableUtils.getFromTableInfo(rightNode, rightFromTableNameSet); - Set extractRightCondition = Sets.newHashSet(); - extractWhereCondition(rightFromTableNameSet, (SqlBasicCall) parentWhere, extractRightCondition); - Set rightExtractSelectField = extractSelectFields(parentSelectList, rightFromTableNameSet, tableRef); - Set rightFieldFromJoinCondition = extractSelectFieldFromJoinCondition(joinFieldSet, rightFromTableNameSet, tableRef); - rightExtractSelectField.addAll(extractRightCondition); - rightExtractSelectField.addAll(rightFieldFromJoinCondition); - - //重命名right 中和 left 重名的 - Map leftTbSelectField = Maps.newHashMap(); - Map rightTbSelectField = Maps.newHashMap(); - String newTableName = tableInfo.getNewTableAlias(); - - for(String tmpField : extractSelectField){ - String[] tmpFieldSplit = StringUtils.split(tmpField, '.'); - leftTbSelectField.put(tmpFieldSplit[1], tmpFieldSplit[1]); - fieldRef.put(tmpField, TableUtils.buildTableField(newTableName, tmpFieldSplit[1])); - } - - for(String tmpField : rightExtractSelectField){ - String[] tmpFieldSplit = StringUtils.split(tmpField, '.'); - String originalFieldName = tmpFieldSplit[1]; - String targetFieldName = originalFieldName; - if(leftTbSelectField.containsKey(originalFieldName)){ - targetFieldName = ParseUtils.dealDuplicateFieldName(leftTbSelectField, originalFieldName); - } - - rightTbSelectField.put(originalFieldName, targetFieldName); - fieldRef.put(tmpField, TableUtils.buildTableField(newTableName, targetFieldName)); - } - - tableInfo.setLeftSelectFieldInfo(leftTbSelectField); - tableInfo.setRightSelectFieldInfo(rightTbSelectField); - + extractJoinNeedSelectField(leftNode, rightNode, parentWhere, parentSelectList, tableRef, joinFieldSet, fieldRef, tableInfo); } - //========================================= if(tableInfo.getLeftNode().getKind() != AS){ return tableInfo; } @@ -222,19 +174,80 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, Set sideTableSet, return tableInfo; } + /** + * 获取join 之后需要查询的字段信息 + */ + public void extractJoinNeedSelectField(SqlNode leftNode, + SqlNode rightNode, + SqlNode parentWhere, + SqlNodeList parentSelectList, + Map tableRef, + Set> joinFieldSet, + Map fieldRef, + JoinInfo tableInfo){ + Set fromTableNameSet = Sets.newHashSet(); + TableUtils.getFromTableInfo(leftNode, fromTableNameSet); + Set extractCondition = Sets.newHashSet(); + extractWhereCondition(fromTableNameSet, (SqlBasicCall) parentWhere, extractCondition); + Set extractSelectField = extractSelectFields(parentSelectList, fromTableNameSet, tableRef); + Set fieldFromJoinCondition = extractSelectFieldFromJoinCondition(joinFieldSet, fromTableNameSet, tableRef); + + extractSelectField.addAll(extractCondition); + extractSelectField.addAll(fieldFromJoinCondition); + + Set rightFromTableNameSet = Sets.newHashSet(); + TableUtils.getFromTableInfo(rightNode, rightFromTableNameSet); + Set extractRightCondition = Sets.newHashSet(); + extractWhereCondition(rightFromTableNameSet, (SqlBasicCall) parentWhere, extractRightCondition); + Set rightExtractSelectField = extractSelectFields(parentSelectList, rightFromTableNameSet, tableRef); + Set rightFieldFromJoinCondition = extractSelectFieldFromJoinCondition(joinFieldSet, rightFromTableNameSet, tableRef); + rightExtractSelectField.addAll(extractRightCondition); + rightExtractSelectField.addAll(rightFieldFromJoinCondition); + + //重命名right 中和 left 重名的 + Map leftTbSelectField = Maps.newHashMap(); + Map rightTbSelectField = Maps.newHashMap(); + String newTableName = tableInfo.getNewTableAlias(); + + for(String tmpField : extractSelectField){ + String[] tmpFieldSplit = StringUtils.split(tmpField, '.'); + leftTbSelectField.put(tmpFieldSplit[1], tmpFieldSplit[1]); + fieldRef.put(tmpField, TableUtils.buildTableField(newTableName, tmpFieldSplit[1])); + } + + for(String tmpField : rightExtractSelectField){ + String[] tmpFieldSplit = StringUtils.split(tmpField, '.'); + String originalFieldName = tmpFieldSplit[1]; + String targetFieldName = originalFieldName; + if(leftTbSelectField.containsKey(originalFieldName)){ + targetFieldName = ParseUtils.dealDuplicateFieldName(leftTbSelectField, originalFieldName); + } + + rightTbSelectField.put(originalFieldName, targetFieldName); + fieldRef.put(tmpField, TableUtils.buildTableField(newTableName, targetFieldName)); + } + + tableInfo.setLeftSelectFieldInfo(leftTbSelectField); + tableInfo.setRightSelectFieldInfo(rightTbSelectField); + } + /** * 处理多层join - * 判断左节点是否需要创建零时查询 + * 判断左节点是否需要创建临时查询 * （1）右节点是维表 * （2）左节点不是 as 节点 */ - private JoinInfo dealNestJoin(SqlJoin joinNode, Set sideTableSet, - Queue queueInfo, SqlNode parentWhere, - SqlNodeList selectList, Set> joinFieldSet, - Map tableRef, Map fieldRef, - SqlNodeList parentSelectList, - HashBiMap fieldReplaceRef){ + private JoinInfo dealNestJoin(SqlJoin joinNode, + Set sideTableSet, + Queue queueInfo, + SqlNode parentWhere, + SqlNodeList selectList, + Set> joinFieldSet, + Map tableRef, + Map fieldRef, + SqlNodeList parentSelectList, + HashBiMap fieldReplaceRef){ SqlJoin leftJoinNode = (SqlJoin) joinNode.getLeft(); SqlNode parentRightJoinNode = joinNode.getRight(); @@ -249,7 +262,6 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, Set sideTableSet, boolean rightIsSide = checkIsSideTable(rightTableName, sideTableSet); SqlBasicCall buildAs = TableUtils.buildAsNodeByJoinInfo(joinInfo, null, null); - //TODO 抽取相同代码 if(rightIsSide){ addSideInfoToExeQueue(queueInfo, joinInfo, joinNode, parentSelectList, parentWhere, fieldReplaceRef, tableRef); } @@ -262,30 +274,9 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, Set sideTableSet, extractTemporaryQuery(newLeftNode, leftTbAlias, (SqlBasicCall) parentWhere, parentSelectList, queueInfo, joinFieldSet, tableRef, fieldRef); - //记录表之间的关联关系 - String newLeftTableName = buildAs.getOperands()[1].toString(); - Set fromTableNameSet = Sets.newHashSet(); - TableUtils.getFromTableInfo(newLeftNode, fromTableNameSet); - for(String tbTmp : fromTableNameSet){ - tableRef.put(tbTmp, newLeftTableName); - } - //替换leftNode 为新的查询 joinNode.setLeft(buildAs); - - //替换select field 中的对应字段 - for(SqlNode sqlNode : parentSelectList.getList()){ - for(String tbTmp : fromTableNameSet) { - //TODO - TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldReplaceRef); - } - } - - //替换where 中的条件相关 - for(String tbTmp : fromTableNameSet){ - TableUtils.replaceWhereCondition(parentWhere, tbTmp, newLeftTableName); - } - + replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, parentSelectList, parentWhere, fieldReplaceRef); } return joinInfo; @@ -293,6 +284,7 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, Set sideTableSet, /** * 右边表是维表需要重新构建左表的临时查询 + * 并将joinInfo 添加到执行队列里面 * @param queueInfo * @param joinInfo * @param joinNode @@ -319,15 +311,34 @@ public void addSideInfoToExeQueue(Queue queueInfo, //替换左表为新的表名称 joinNode.setLeft(buildAs); + replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, parentSelectList, parentWhere, fieldReplaceRef); + } + + /** + * 替换指定的查询和条件节点中的字段为新的字段 + * @param buildAs + * @param leftJoinNode + * @param tableRef + * @param parentSelectList + * @param parentWhere + * @param fieldReplaceRef + */ + public void replaceSelectAndWhereField(SqlBasicCall buildAs, + SqlNode leftJoinNode, + Map tableRef, + SqlNodeList parentSelectList, + SqlNode parentWhere, + HashBiMap fieldReplaceRef){ + String newLeftTableName = buildAs.getOperands()[1].toString(); Set fromTableNameSet = Sets.newHashSet(); TableUtils.getFromTableInfo(leftJoinNode, fromTableNameSet); + for(String tbTmp : fromTableNameSet){ tableRef.put(tbTmp, newLeftTableName); } //替换select field 中的对应字段 - //TODO for(SqlNode sqlNode : parentSelectList.getList()){ for(String tbTmp : fromTableNameSet) { TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldReplaceRef); @@ -351,8 +362,10 @@ public void addSideInfoToExeQueue(Queue queueInfo, * @param tableRef * @return 源自段和新生成字段之间的映射关系 */ - private void extractTemporaryQuery(SqlNode node, String tableAlias, SqlBasicCall parentWhere, - SqlNodeList parentSelectList, Queue queueInfo, + private void extractTemporaryQuery(SqlNode node, String tableAlias, + SqlBasicCall parentWhere, + SqlNodeList parentSelectList, + Queue queueInfo, Set> joinFieldSet, Map tableRef, Map fieldRef){ @@ -389,7 +402,6 @@ private void extractTemporaryQuery(SqlNode node, String tableAlias, SqlBasicCall queueInfo.offer(sqlBasicCall); //替换select中的表结构 - //TODO HashBiMap fieldReplaceRef = HashBiMap.create(); for(SqlNode tmpSelect : parentSelectList.getList()){ for(String tbTmp : fromTableNameSet) { diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 46eff4b6c..39d79c065 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -297,6 +297,8 @@ public static void getFromTableInfo(SqlNode fromTable, Set tableNameSet) /** * 替换select 中的字段信息 + * 如果mappingTable 非空则从该参数获取字段的映射 + * 如果mappingTable 为空则根据是否存在新生成字段 * @param selectNode * @param oldTbName * @param newTbName From f3ee3ce71c9c7ef42451839e40c3a7021a7f05c6 Mon Sep 17 00:00:00 2001 From: xuchao Date: Sun, 22 Mar 2020 22:51:17 +0800 Subject: [PATCH 165/523] =?UTF-8?q?=E5=88=9D=E6=AD=A5=E9=87=8D=E6=9E=84=20?= =?UTF-8?q?sql=E8=AF=AD=E5=8F=A5=E8=A7=A3=E6=9E=90=E9=80=BB=E8=BE=91?= =?UTF-8?q?=EF=BC=9B=20=E5=AD=97=E6=AE=B5=E6=9B=BF=E6=8D=A2=E5=92=8C?= =?UTF-8?q?=E9=87=8D=E5=91=BD=E5=90=8D=E7=9B=B8=E5=85=B3=E7=9A=84=E9=80=BB?= =?UTF-8?q?=E8=BE=91=E5=9C=A8=E8=A7=A3=E6=9E=90=E9=98=B6=E6=AE=B5=E5=AE=8C?= =?UTF-8?q?=E6=88=90=EF=BC=9B=20=E7=9B=AE=E6=A0=87=EF=BC=9A=E7=BB=8F?= =?UTF-8?q?=E8=BF=87=E8=A7=A3=E6=9E=90sql=E4=B9=8B=E5=90=8E=E7=9A=84?= =?UTF-8?q?=E6=89=80=E6=9C=89sql=E6=98=AF=E7=A1=AE=E5=AE=9A=E7=9A=84?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/JoinNodeDealer.java | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index d42bf6774..33bd2b293 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -185,24 +185,9 @@ public void extractJoinNeedSelectField(SqlNode leftNode, Set> joinFieldSet, Map fieldRef, JoinInfo tableInfo){ - Set fromTableNameSet = Sets.newHashSet(); - TableUtils.getFromTableInfo(leftNode, fromTableNameSet); - Set extractCondition = Sets.newHashSet(); - extractWhereCondition(fromTableNameSet, (SqlBasicCall) parentWhere, extractCondition); - Set extractSelectField = extractSelectFields(parentSelectList, fromTableNameSet, tableRef); - Set fieldFromJoinCondition = extractSelectFieldFromJoinCondition(joinFieldSet, fromTableNameSet, tableRef); - extractSelectField.addAll(extractCondition); - extractSelectField.addAll(fieldFromJoinCondition); - - Set rightFromTableNameSet = Sets.newHashSet(); - TableUtils.getFromTableInfo(rightNode, rightFromTableNameSet); - Set extractRightCondition = Sets.newHashSet(); - extractWhereCondition(rightFromTableNameSet, (SqlBasicCall) parentWhere, extractRightCondition); - Set rightExtractSelectField = extractSelectFields(parentSelectList, rightFromTableNameSet, tableRef); - Set rightFieldFromJoinCondition = extractSelectFieldFromJoinCondition(joinFieldSet, rightFromTableNameSet, tableRef); - rightExtractSelectField.addAll(extractRightCondition); - rightExtractSelectField.addAll(rightFieldFromJoinCondition); + Set extractSelectField = extractField(leftNode, parentWhere, parentSelectList, tableRef, joinFieldSet); + Set rightExtractSelectField = extractField(rightNode, parentWhere, parentSelectList, tableRef, joinFieldSet); //重命名right 中和 left 重名的 Map leftTbSelectField = Maps.newHashMap(); @@ -231,6 +216,25 @@ public void extractJoinNeedSelectField(SqlNode leftNode, tableInfo.setRightSelectFieldInfo(rightTbSelectField); } + public Set extractField(SqlNode sqlNode, + SqlNode parentWhere, + SqlNodeList parentSelectList, + Map tableRef, + Set> joinFieldSet){ + Set fromTableNameSet = Sets.newHashSet(); + TableUtils.getFromTableInfo(sqlNode, fromTableNameSet); + Set extractCondition = Sets.newHashSet(); + + extractWhereCondition(fromTableNameSet, (SqlBasicCall) parentWhere, extractCondition); + Set extractSelectField = extractSelectFields(parentSelectList, fromTableNameSet, tableRef); + Set fieldFromJoinCondition = extractSelectFieldFromJoinCondition(joinFieldSet, fromTableNameSet, tableRef); + + extractSelectField.addAll(extractCondition); + extractSelectField.addAll(fieldFromJoinCondition); + + return extractSelectField; + } + /** * 处理多层join From 8ee1f7a274a539f90e6e5d7f99e72171103b22af Mon Sep 17 00:00:00 2001 From: whiletrue <670694243@qq.com> Date: Mon, 23 Mar 2020 10:25:05 +0800 Subject: [PATCH 166/523] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 491888c90..e11a05212 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ * 维表：mysql, SQlServer,oracle, hbase, mongo, redis, cassandra, serversocket, kudu, postgresql, clickhouse, impala, db2, sqlserver * 结果表：mysql, SQlServer, oracle, hbase, elasticsearch5.x, mongo, redis, cassandra, console, kudu, postgresql, clickhouse, impala, db2, sqlserver -# FlinkX & FlinkStreamSQL钉钉交流群 +# 钉钉交流群
From 059454fc0abbaee9292545a6ca9cf90361e1aedb Mon Sep 17 00:00:00 2001 From: whiletrue <670694243@qq.com> Date: Mon, 23 Mar 2020 10:49:00 +0800 Subject: [PATCH 167/523] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e11a05212..109cfb9b0 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ * 维表：mysql, SQlServer,oracle, hbase, mongo, redis, cassandra, serversocket, kudu, postgresql, clickhouse, impala, db2, sqlserver * 结果表：mysql, SQlServer, oracle, hbase, elasticsearch5.x, mongo, redis, cassandra, console, kudu, postgresql, clickhouse, impala, db2, sqlserver -# 钉钉交流群 +# 钉钉交流群（可以搜索群号30537511，也可以扫下面的二维码）
From 837dd1f0708bac5513a276946b442bbb0b817d57 Mon Sep 17 00:00:00 2001 From: whiletrue <670694243@qq.com> Date: Mon, 23 Mar 2020 11:03:35 +0800 Subject: [PATCH 168/523] Update README.md --- README.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 109cfb9b0..1f2306336 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,14 @@ -# flinkStreamSQL + +## 技术交流 + > * 招聘大数据平台开发工程师，想了解岗位详细信息可以添加本人微信号ysqwhiletrue,注明招聘，如有意者发送简历至sishu@dtstack.com + + > * 可以搜索群号30537511或者可以扫下面的二维码进入钉钉群 +
+ +
+ + +## flinkStreamSQL > * 基于开源的flink，对其实时sql进行扩展 > > * 自定义create table 语法（包括源表,输出表,维表） > > * 自定义create view 语法 @@ -7,16 +17,11 @@ > > * 支持原生FLinkSQL所有的语法 > > * 扩展了输入和输出的性能指标到promethus -# 已支持 +## 已支持 * 源表：kafka 0.9、0.10、0.11、1.x版本 * 维表：mysql, SQlServer,oracle, hbase, mongo, redis, cassandra, serversocket, kudu, postgresql, clickhouse, impala, db2, sqlserver * 结果表：mysql, SQlServer, oracle, hbase, elasticsearch5.x, mongo, redis, cassandra, console, kudu, postgresql, clickhouse, impala, db2, sqlserver - -# 钉钉交流群（可以搜索群号30537511，也可以扫下面的二维码） - -
- -
+ ## 1 快速起步 ### 1.1 运行模式 @@ -306,6 +311,5 @@ into and a.pv=10 ) as d ``` -# 招聘 -1.大数据平台开发工程师，想了解岗位详细信息可以添加本人微信号ysqwhiletrue,注明招聘，如有意者发送简历至sishu@dtstack.com。 + From 3cb8db43a06a41ac20431079cba311ca193d5c12 Mon Sep 17 00:00:00 2001 From: whiletrue <670694243@qq.com> Date: Mon, 23 Mar 2020 11:15:56 +0800 Subject: [PATCH 169/523] Add files via upload --- images/IMG_1573.JPG | Bin 0 -> 199826 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 images/IMG_1573.JPG diff --git a/images/IMG_1573.JPG b/images/IMG_1573.JPG new file mode 100644 index 0000000000000000000000000000000000000000..4a32461c10c6a5256af92e7655010d9d8a03077e GIT binary patch literal 199826 zcmeFZc|27A|2TSN-=eaMQIws^T4Yk$5>laTQ%UwFBxITqA$tfRrcy~+vt`MUt(CH7 zA46ozOfn8LbLPH!|L%9Wzx%uQ{_gK_??1oa=Zte4&Uu~JbKhRiGwkndG9-G!+{zr{ z;D8`Y@PXJ<&^fa(pDPe#V*{x{5X1*<;n)swf-?^Af#Q51u7A!UXfOB;L7a~`{^RPS zE&q0vbLtW2znydcb0d2hGCbvty^am`#$Hp<(b@+Y9<{RJ`ujAn{d2zOpHtC(xuoG# z=rz}TB>#gY_$Z#;0&U~thB&7+G9S&qm`}Vr@v8^ zbS?!&@e4>uN=eJ?*{iIgs-~-_Z(w-v(6QrY<`$M#C(fL8aCAC%-r3`_=M^t+AK&2X zAvZ$9!f!_3iHVK78=sJ#aX&LF`$10blftLZo)^6+e)+npx~8`7O?|_=w)T$BuJ<24 z_V)D;3=VxC9+{#{&-|F3`#HZr`@Q;SjlK?VZ2rXy(DT2b1&;p(v;PXOZ2+$=TwI)7 zynpfH*b@2|@oik(yY};JH?`-z7`S8if!lmyN7Eixw(=|LoTiFj3i>7>p`<&thxQk> ze=z&+LoDil2(y1j>_6}tg^mFs`@3xcKb%{(09J8=jSHXvHf|p7e>R@~edGOSjw?SwKK~$b3K>w@m?63ce z7o`7-_61C0UBlkRc^66XlBYy7}um5=d+~TaJ z@mBgKl=ZKR|9=u8cx!cdGD2XT<<=VEH&$h{$bL3v)7%n*a;#b#Xq?#RUUMf5LCW~VJ z>tuI5Q!ElKfQ{*7sFvAjYPNTz2FIzP752Mn9*&$NOiB0M@?t}z-)x9>mhevJ`6d8| z@PiFeUOjp8YGmKRN!a}Qw|YWGJB5Wu9+@P-Y$Dx z&xUq9$=|Yk_k-o9)Jw#DEHmrOmhTHX7RPeSx$bn9*qzdGROL{s4b$DcZ7X&54`gp9 z_HR1}yZ=9YshT8XxcDW{g7+2!e<6V)jXMtUR?;QIqdQM=;l<{xl#x6eHdH9n;5>`l zOQ6`Zp@8RX=v%Uz-odLJW@4wICkZP5)9fYMI|%Pd5p2j+32|sVhu_PF((vo()mAo? z329d6#9KTPe~eOuW-CgSSR*WMfAbb{0#uc>uXsTyz0IWNFeGMg_U}7^J@B8%%-P?+ z#sR2Y{dQ5tqg(xwZEU5N;lW4zxZS~pY-rSn^vA==#+7sjJtIQg^fN$`d>7Q7y#XQ6 zX+Y*v|I_T}-2V$vkvCS%S2rq}fNpQ1Z;-bnU-**i;Mm{eSFUDJHgwN`Rh!3#%=NCXeGWT@ z9GRq#U|CWFeFJva1XP|p-IAd4XiHNgWv;X54hCwHXG5aR24Q(8saer-?GoiL7S{KV zxM&P%DaOd0KYvOx2>KxZR{bvP1;Mvxm_gTp&5Ci?SSp*Y5vXK9Y4BA;fds#=EsX3dw+_UAeR zk~VsRKJVT}o(uH9KHD@+=3RDT7$O5VO^PVm!&RM|1}WEBGC$8vjQy;$AMX%uHM?)# zYkp#;kca%|u;0`NWE%j=hz(sLz{(Ba^e~e!-6Ilqti@-_k0ksNQ5aa(^eQm=c{ycv zs<}@sL-+NG-&5wC_e3EG)mqM!9cM$_2U+*o5cvV3%X&<>f*rMPx6^JTO1cPoYgeCb zXtHi>HTs1Fp+9Y`fi5^*ge7zps3c%UKG!w z$P%UWIs30lS#pQ0(+gEEWT$@!RLYVR3fxy;7j7S1ne(xlFdoVj?L!1#(KrzS22Gh_ z5zE9-JJE5u*cc6e?De9A*@%_u_-9$_nOQ%`QM*4do+RAeDqZ!E^9br2?9rmzmFaQx zx9Fn~ z0}pTPCB+lJAES%1o&s7Gp{la-Xe;zs7U?l{Z%+Bxgp17-i_jkGBuOhR>EZAR(9)Cj zj`~n{Q;7VS*Z3>%W`GYHlEjNIAO|z1Qqm{^dl%d9wG5s1**DR;=l7A4m%TNw>+hRO z--Yt~4P^BdhtDH~+SepwTa2jXIdVvlYx1PR|&+zggaQd&gsoSN9 z2u&{ZoompncDEkWIEsv)__$z8e)w=EU4@guqr`@YY{+f*{;M^JJE~5SxqW3?aw(jl z5mrUNM_pNF6fuu9QQP-f%in$)7CTjr#Vy`_q|Rq^!f_8*Ot&cWrQariG}@jSI-Po# z4N)!K1UpcoYk91QGa2m-6upVUV9ov?r$YpuTjk_I*QHw?iOj3N)znSjst(@(FyBVH zNz@QR0y+S77g1oP;9OpvXKFVNdSDk~-9e+CBw9X;^e>=wOWxX5sc>wU^322kv&eRp(Qsq@Z!3d$rr}XAlRAM@Gk? zoa=e7JrC|n+||Q@U5l7G(lq8ah8linLT6xxS{J|8`mAQHwTjb`x|v#)Bm1$%_;!_? z`K=e6EP(?m@5_J3vmO$#1o)sAy@K_;$&{gh6NKw1!X90SB!fWDdN?F=cFT_Ewwh(6 zTe{Xe{O`wXocpzKK>bf1B4)sp&u2r&%o_p02_l4Ki0~xVO!^DqzB1Ebj=8gv4GF&n z8pnoU7rX)+im6^NvXeoN=$4YaU4pMWEt@!{r7B5uGjA+*5G)AP6NEc}VZn2;=&R-2 zi)iYpWP)Q8Y%Ma1uAQpy*Y%6lX{W6&2mG=7s-eH-W5m>mKhrq`#S23j4TYsi)BmbSj+BqZO?es-m^n= zLGDEQK`y6_t}$j1AbBI9wVYVP>O;O7IFTfp`EW-D%e1d*Ybk{X=*4>x4n(}z8?;ii zFdDS7BR~D&jO8QCN2qzrvUAM0YA4vxAq;#7g$T%%Ft1ZBmI@ii@f~;*8Y+&dO5-)B zarXyY42^jDVV8BX$>F;F=CrJ^oef^Iu5%vZ2M-V4Wv!47Nohx+``Jm83as=j7WCoYDek5`gGfed$vA8IBC1B!^7k0w(m=RIOKUBY*DTWBE43A zN0P;Hva;|frYu}knkouMzsu!L$thmwwDOe9*00bA>b^`|FMWHzcHYFJAiVrX1|nkD zf1S|-E2p;IuP!)vE{wkY=su6srNzmk-kh6&`xdFjvNY{?hp_D;aW_k!*FRpAm=E6CKZ&ITl8V?P_>3hejiWp4XK)~t5gw(*I&cydF$ zlWg{2X{)I?f}NzE#C|uRgK|t%Bknq!N}VLnbw-r)+mPE2-TCb&Q}Dtg{ot=$aZ9B8 zvO=-+LFiH>dZ?%;nz6Xi=S&eb3}JQ<3Q)|WE`Z>dKur7i8vxX^4AgoWM3^%F{FjqB z68G0!W&u^`{K?4@RSewy>$FJUGmo&u_6d1vY|JeRQG9aLC)YiPE88hMJLsuCoqF`y zOqS1OwY1IzL7n{^N0jbyav(;VCM*gAG-Q$zb2jU!sY`dC_zy zq-bK+^2w?N@7-Bt*5$9GF6=hDUw+?8NUV5EB)Sn3O|14>Vnd8$-zOG`5H4hDa_7kW zYYnH2u!ytO{k;)RbvUQj)aTon1X|#sugvM z$%9Rl5s6=O@#j|bkbN1CAxnnTs}p8wvddXw9uA(f6T|yGlK7ymwc}w0 zAW~n#8<7N*I9{5$gK9I^UY;bstFK9OYTU?cq~@^mth$k<@zA-x&$(BQyLEqjm42X| z<3qVRD-{<`nM}aEws^;e;tLGx8CUBlDE^-fS&sv2&U|TTsEqY_sK)Iuwqy&n^z0O1 zY5`ob*^u|j@Ja`XYJpB7ZDndt_iSCjD@;B;#>(+~>1-1|Yg#g#b7pCNSH|^=E;pAy zlINs!Iaor>J@6N5>hw_bB{-gH(Z1=F_=kTcU208J=9lcKMf%>+-5OtY9$&sH#d$AG zZ$5hd~%;hFp9IdRnD7p zos+V%Y{t;T_tcHxb7(6R9rtwjEEuAD<+-)QDQQqLFw42oB18J>XL44!{Hl%@q4~8NFH4Iq*z73+m6sHlEa$YUS zr|%LT#NHuJ84+^Z1aFFaVw@Q z@8T zj>dh*X~+2w!OzD!LIlQK4yJfJTb}Q?+7VTD=a-|Df(&w#K)y}nX6|7e2Y5D4ZZ+YC zf4O|Rh3dFzaaVV={`z8$#;@s=TkadyZN?hqvgM~tTRWvhEzw_6m(-{KYOFAb`J-yf zmpG^28aUF{-=)TH-q|}ida@>c!RF}5jr}f*GS|1oT@1x0XW!GFIJ@h1yB37*y^@d^ zei74+VoFFNTlWHfY*(j8f5mfzRZSHr8n|ka|9noz>e}QK%dI=uGSTG|?1;%bYAL=C*090M3${k~6)s#aDl`Ilk1nBjVyM zUy7xc=G-aTC)u)N>cw2!MjBqSp#vybiNKUQ!-ls0W+h%n4=Ya_w7s?HTAp&ryWguP z8(^;PA(5xUf2k%UOZn2pw5NQ-ElE$HEvuV8Y-n2xPMi%r;VlKSv;Z(n`^m|8`7+xn zi|+W_-TnQ&R)MOflvW1^^Yc0JEs#+9O##FSO@078)Ai6cG_2~ioH14Y9Y_O#Kg3R8 z98M9O&p2J*Pp(hVt{AbtZkoK?+^9XmT!d?ue?E_muX%uH(FReOwm8-yfbBogD27_1bMM^iuwyXsn~7 zw9wOQrgD6Sin(=|Igkc$>9A_hvifZ5-`sW8mHFzJAnCcv z|GR=`;KRWVh;W4sRnILiGo+DDJ5P~iE1LEZs_|yJbb#iXQFRI>tj+&`1@yV(_8SdJ z@576Wg0hQ;Z9C!JnbWwKk< zUF7_xm_ys!io?7ym~KsMRLMQtyq$c*6pPb=tEydRFr}m_JLb+s;D#JoMABc#aYFs1 zD;vh=weRTWxheUs*XDi>HSqC~iamJcY$Eo7!`c0EmdbA-Lg5d+2T{Oz4>7m5AzS5H zZ_!stqeZQl*K?i3;a}REVSeqeie=vhTpSP`y_ITP-;|j5p#9kAbn(<##g)Kr^-XC2 za5nOpAc^)*{Z8zhtRl3OOQ9oi4iw9!-Y)kPm%GlNN8MY4cTM_!+PA70U$`M+R1vl} z(q!qh<;*zjUPWOQLTqS!6Br4o$;CW&A?<8T82Sx(vZQe!GucE1HOpN?)a~>>= z9bL|j{lN7~V{}wVP_BMDrOD|v?h31bKz@X#T4P{!uNJ#u!ZdoBgMyM1*}iy)r&#g) zqRi`2y{65g;m_MU0u6eID!-)EOuI@U!V6L&&TmQ^3JY54K#T_b|JbN?7=OweKN_}N z`q-X%+FD&rjgiiMb#&rbI9I4+kW%G|mp^}os;dh{0U6sFbtnPx1-kLG{QIX9tk(o` zflSRxOh`u^jr48~yTfPm)Z1;sM+YP&D-KuQ(?|~&@VuQMvE8DEZ@WGo_>HJ*K>dA; zBdBeN@OK)zD|KX#RshbET7ELSty!k2-!!>~rdHaVtSj3b-t;_EA(al-RS?n~a%&#jX=X^}tu z7}Fqs|9`08MMfVH{BKqH{_m<~4yoJ9jBYbyux(!*xwFUY9wA5TrhckynF}di*t&S; z|1c#ld+7h8TA9dfAtIOoEP^7^6`dmOM*pBAjBr6$B}fZQmG7q^4zwFJ^)ULClNj;C^VZ%Rg6B`qS?lOp z4%G#JsUP4wZ9}r3AZZZD4Fu}dlsGV36O|B6@Y5ca=y8B=Qzwt7t1vJ6m0M;XXgzSU zRA`lzWfV+Te`a$0*_Mry9AjM3*d;*03}mY_E3sJ{>A>&$3iO}=EvE;F!*qO|JF;>j zeaO5qdiiBK_nYc;m&Bm7xR~M0-%k%|8@?|GcSV^3;qOLO{>otRm{E@*0!7oB7H4QR znX`|J#_b5tU41{jns++snOr8L^3?p@NBUBFkOPk?Po&A8zk`_}AVdu|F~W)UHAPV4 zS|)|@GH_?_fahSf$o4%%dJ6Gc^nq$gna5Y&+z~ym9FzLqoRypj;&d($Y(&j7f!ao; z;zu!JXipWK$P-ct>t~SAg^xcD^RYen|q^7nlI6p6c`LoX1r!met!D>6G?0$@r+BV~9w?8N6 zK^FEtB=QXlAL5Dyz$!D{yUJ48P?5DWB8Vmv(N&>L*#Vj&-ThmW6EK@x@aQy(8+R&q zb}d9$*&_1o(^E=IwI#`{?2vS7%oW!BOabo+He_Z8Ypt}SqKP2u$wqB&xyae?)KN+D z5%}_?D9-ei_f33qk>8>2#@%J}1rB8lo3%G?o6 zfBFsWhmImv+a=YYMZNdAjkaP-&h=;(YA%UjcZI}R#vVO-P(5zv+LxQkOtH~vC#EdO zN4Xv0wIV!_F%-y3&=Ni}oq2_APr7*GE(QzNNAY{+oX?uJdK1KW?tic4!d7(``YRaR z$@2kc0OrPry`nF%Qt?tSVfrZSIBgrdc(jgY^6L3`dqd3FjmN({(tgy}3AkozV{6{O zD_A3=IM;}<-ph4526h5<&JI|MB)Eo62=d$8XZd;{z&Sgg{4PT4Nn;U!!%w7Z@^xwy$qzj&NluN! z8~rR$AIdwH?uE~6f+HZ)%UqRF-oxq)34rxU<~|k?kwIE8TeZ>0t(Nvs^S2SLCG zImmj!^m`tLvK+XIwEBd;JaH+-FGKdz4^`+#t)l*6j`Bpp)F!AP?`mZ|V(wx?w}B8>oGE&-{0NGqvg<*u;}ZAX{P{Tr2wq69e#m-(70Uay{nbFK3rZ z={$Rr_0i&Y*2$L=H!k5JEG}OtfLGROWUG9|F>7 zK>?`AIHLs>L=v!LN?QQ?L*1%~7DIOZQn+WltY|1D-9;z7bq6(zXjKxqfSzsd-bJ&Z zNbf)NhR02rtGI{7`GS+75H&0K;xuJE`e}lk<%JIy8U4~1PTPMfP|t35xYt@VGv2$r44N!bAo$-((f|{0Kr)1!c3LlLv-2Ewwqt5uNF&6{dI|f)YlxO)f`}VBhIi)qZ$wf25BN zY)+NVtygGIHk4}9cCgcM5%cvP>j9el$8{#G@2_1#;@MEgDcmUZ!{t!v76`Du|OC zU_+M)BWK<@9>OM+)!BuUZ(Z>)4!t(|UMXS4rpX}&l>7zn4C{J@^60z)c^!fG`ikrb zQ=VFuPc!|DD8`vrCOU-+)JOcNGYvYQ{louLqtdmAK+H!;rdT2(jH`l68D3$E^cJ9_ z*sv5-bZ!Zvkg(dJ>g}vlxhvOe;Lp>e`mR@Ho%;jghoglf4KIj*nk;;n_g%9TRXK&= ziQx`A@NV)%@AZ6b_v)&un&eASe7o|jEh))j+-J=o=n#kdVZg4n-X?pRA0l{=DoMv; z@2-Ok5O)~32-w=TX<%1ly1E}%V|_z{bx~?hlwxJLQ9cJ}miRo08Xh{j8wN!K^RupV`E1hA|ET{#dJf0XNHt~{YS$Myi|3Y+g9tXPwNgZPGMpRkRJNG0kTzqMI>Ay#Gz4GL z2QD^YtBhPfslBc05?&rVdFFzWJuU~j9R@W4-zWUW)T}w^IKgdceRzn z%5iNd*wBxDgWAK~f$9jw?}vRu#ntVp=mU*)<)fRboHHiX``B)PN7s5fod!6+X z5%EP;8!WZaB+2p_xLmiPWTwt5?|l;|p7l=Hn0LE)W@cJm;kzt!MB!l|W(MTSe^o0= zB`g);1Jzns9(@-hUcIu@&%`iyY1AjnF;rqjxHP>eCul{zVR=>QxC1%eN=}$i;gSE+ z?inG?iMca^;kt;a8)Ax`=^~0Vxo0srL)byHJo3*d5YW_MUCltH&q;SXhAw>edaDt&_0&PL%ldQcT-=fqc_|NZj)BtWIZ$@F7b7Xb!VYo)NyM zov6}!dY>>(68M>JoB+*h*@%J~MbmFmx4yyYl)#@_e*5S8kUr+^EQl+CL9er8ZttPnzG1yG_G4JM!LL6~I=t>pj>+sS=6bdg8guKnt&?DtQi#G( zDbhluUY%TCXW;Pf-Gx}COl^jZ%@jjKdgMch1TjRvRJVd}HeIf4-te^5$(gOOO=v`r z6EG|bFzh)phN>*>YMC6FrUzwArAE&TYG@N(0%Tn)1%t1;wu$89PokT~#!wHYf6QZ! z0U~IU$Zyb8M>aIvMYo5&82U^>)`J;Zc4YOnT1WhDH$hCjq{OP5k(D;l7vDW9Sn-ir zh&?zPiT)N%u8Uej#}asP%G7Dsw;ZB@4A#I(D2NpL^ESoY(vubXf#{jC=`pR9q zkKidXvU|my*42qgs?EJo^kVUlUb@@?^&5=^-Gklz)=EJoK5FlKjkM{Mx70Y3`6f0lgbTIPDV<)*-9 zHX^bgW>A&EVsRVQ&jz-b4&dxC(05lQrBgH&PuDc3EBR?Ye(Dmw^JPYh5=VjhJTYOC zdgZTK!B7bCCMv#YY33Q9K#GmyszW;8l19Su^^B&a1Un^F+?Mc`vcLGGpIKrP%ET(mxDY1wM6G5^X&oIV$0KrZ>$8>u;qff7i`cn6(2lev|)oAKQK2@7se#b;`J~U|3ISg%9 z`R#FizQg5QBf;x+=XYv8q6~Z; zTdF_{>&Km^SO#G1-=tcR`W>w+L?6|@j#0ScRB(F9N>*QA=NqB53&jO2PzkDHXjzqF zH`0pigEQ=449%b;`x}fOI!4nf-tMz-rYB*-<#Lwc)aQ!{8sV7xDJq;GCXFrxLqRpX zGMl+$Zko5HM-X{GA%x){4r}Oh$Z$y3x>{z|=Hfd1)WTr<`?D`UPXACWKm2O*HyjR7 z><1)~tm%Qgox6*uzLNEgeWh1&x{HgGd#wbjb_zb?P@+Rlzs;T#TB}Lab0VWfQ`u3? zhr_a`O%kWa70>quthouSxX^ftURVj2Jr)a2)fh9|b+=;l?b+0v@Ocafbz5zLV!-O_ z%mv@%Os;Mq4^FpesyeZ=|Bp->IFJ?#* zT1`loNw9KVbO-^$zc1Kf$M17!jj|y5^;t(pPu{i4yU|(o>E=>LcsMvmNFa|P*EfT3 zF#7f%?1=iiS`b58X&Rksq*{A()Ob}Vny2v;(Q{0>7Bu$sq$@HCu_pE9-_fwr^B7u0 z8-Z%09p#Jx1=`UbDK_*@`J#TY{xhSo@#?;=xhvsMy+SkmUR?TY8&>{J00ONISlEt4 zrfY9*S;WA?Q&>ceR3I|ig$TR_#>ACXg4;n689qb*O1qy+Pn{~gh;pc~);#D@+pqC7 z%<0ybBvacVFZ;-t%i2%jtWE zwa-VGT~3`%5A*s?>`G(G`2pnYuwwWxBx=B9JjoxMSbM-FtZLdR)%bAWWvy(TvKZgG zfB;pcA2u_+R{MK2rQ#c@9IH-CfAd_hwj0&Y)c@AR!QevLoA(tA3JyjP1ii0xS@Yaz z%Jqw?bj2L!4A0PQ47L8mOzf5*`CSE?PhJf;)_%iXr{7rdmPzIF4qb`R-g9AAzXB+4 zM(32y?AMn0R2lm6o~-Z8P66179?gm|QE(-+@{>GKuv*d5s*jSq5`0e2T3Zxze#Z6K zmFC@zc_-f*&LmntzpP>M^Zt(CDd?|YeNG&Jr~+fb(TtVUbeLfZpYSHrxEuE7QIa+G z9q`#~QfUqnCvTjd`*NEWYtFf%4smR(fe>LQO@t|la;Bl$L0m+xkT4j$iMui4L+x|7 zc-kp*B^())WauUCR7o_sz{w~oLtjUf@L$kX*`02eP}^{qsJyXErx&fFQR<~$C#p!f zhPO2ot+aOE;FHRz-g!BrKbO94c-n@bL81nunPNjAVPHiz?|>~@NtH!0&#LR-IHd;v z34<8!8M;CJ79&CHQ)Sy1uJ0+#HM?-_-JXeO_c`YO!qPHk-rtw=4Z+ekH zJIK~GVoJkWu}fG+TkgrP@J)UT$-0*T2|0IXC&pi&o0`FB0I3Tmv3qc`rBFe0Uc))cVP zx-#NH7hq+rt-u-}^4l?WYp6q8+(+`>g1$WOptj(xhu!V@5$)}g1!0fx7s@Ibbz8gY zdT&M7Ov1ZDnc{`-h*U?^u#AZO!jnF0IC;uS(w255&G*^v*x(HAfrkA`F0P&Rb`O?9 z!anRVRgmLliec!oHN7McqBJnYt%vd4xcwx!toA{#$AB^V*^@}yJrURFoyrYoKiuBy zdAHdhoo5hnB#;w{R42ml-f4<5b6*@&LXfXC?etOF5bIjJzf}2xHR}g z;%@EguXm;0{pQBjHKee~Aq-Pu2O@BuYzOb@Svo_}9uag&7fcm%Q|-g4KRYq0M=w1< zR1sge7j6#qVt{@bna+4a*NlIMz9e zV&d=PUL2paH(z=~f+!)hde9gHDx$f(<+#b2d$!_jD#%TsgFxW5%46`dINg3U`5~sV zC_Z2fEeBWY6Di|%w)y9EBc!jDU&tu_uJtS0KU?I%?lD=vH|q3t<_<|jxRZ)u?(jnd z_D`cndZyOfZOpMed5(3|uGr-tVKXV}Kd{xuUXiRHly;kZMBk-Yv_0abo&hypK+b0X z?0xtiBGsNj;zoKwD>$b`5++~@n+Vzlmf2@L(1112D?xM1V5OF*(Q>Q9`Q$+BZ8KXY^8i0TKI}MgKf*nY5>-Ut)FXn+C}% zO}|=X|FL?vvzfkRG$<9rN<&Q9YykWK>pAlvLj~!_?}BT};qvwySx$qALRqQ)-!QHp z{P@BH0s`KfT()_`ac^J5>N!ICo^R$y#SQJW=18$pXj}?2Tfw3ZR*FSjYzDq@)bej z2K?jg@9CS>Rk4$DB~NcXG(VZ-VI^<6s)z^0omK~&B;$J6DqV%99RtMb;h~0~1c_f- z*Xh0GG7%Bi()Ntm+%WzGiYAMJ3LDwVhFm*O@ovQ82PeU@h}a;I-1Rm#^gUyG5;Ucr zY}T7J$&g4cH7O1Zl`ptD<5OA{vYUVR?`u~AUfzgK%Qy*uyo+>TsAq_`K`$}}%m_+T zfJ5;F>V4*U+HN1OwubouEiI%|OS?eUo}OcWIq}mQA&~y(*N+_g`mLl^TG!RPvRCIWGzM99g7QHv z!FvVPGoYuj8g{gx#XvS;mwN4d*s2*7pFhP4oay^9C_FH5KU?Uk$?%?mMw7Kni!bu# zhjI}+K+Y#XZY^#dqXR6XlEP#Wepcx8d#0U1vSzj4?NZHX=TZ{=?b2X`O?|;>#6A&^ zYi%@a>9pJA1f021gN3#5`4l!32Wo{}x2o&w`{euVK@aVN(Xx9jwo1`UIY{4y$>1++ z$f>IwP|*a4x(Xgdt=zb#zoHZ-vr7L{TOT*QFzO8eNwL^H|Ku3o+XwTFW@!v<19$I@ z?%?$|U;*5jP?^eR}aj2aF#)zrK$lT#dZ{V5fz%5m@b&F**hO|gB;r}-}KI8>CQc>Va8`Rw7y z4|=l&yWrNTM3tqit6ngDamVcg;E;Pv*maFQS?YS?#1x4>BXw^ zr9&S=_mm|XJN}ZDzp(4UD~sj#U8NU%@rV1DlRIi^Qv-z(eSREoy}q*}Z0nw;Q7 zO5l*>naE+oqt;pn%N?j9T{dS9576i4BW2aUys{%4{A}M zG#{dh=XaJ;7S##ctmsDj86Pe;)KJ&&xALrLHq~kwfA;^F6m>JDriyZzc6cRr`EpHN zea^4vO{5GK>d3v9Jlmo=@_ZnzWpYMz>|DaH$HyLSXvl}&O{VP+P=BlE7t;55v%@pO zFXC!;ZIDLA$H%?LcV%YQb8Q2SQzAFN?p(#>5qO)8!`j*m?#;Vvq^zH=uHSz=*?9B# zwh@Ir)|VzOoQP3W@Pe-CU!9zILJ$&y>j>Nh9yuWvU z2=4rb>Fiowqgf!r`&p?0}FkCqe3noZ@ThL}559H@)1{)psA!$`0<* znoY9I+e$tai-Z%%btI~pHoUiox!nlif4ozkcL;nrMQ6FF-%bA;Be8W6G7CF~U%kJY zoR!aiHerk-_Ew68KySJkgzEx^<^`COi)jgtEx7Y`qr4PM%u=%;zQs$#je?vF95>t=HvceWr&2pf-+Pd)I#Z5@ zjnr!5c!ATee~lGd1dTVLFNd3-?JUVwtrU0EPaiIPQ@g$4{ewN3ba9Bu0F%84!y>{l ztVldxM+r^^_Irf0h2tmzdfH6~pUgYg%+L|Hk33PUrwY{573xMp@4U})4RC<6F_i@J zfXFb0yq=2ZJ&zx4J^*uKsf@1NlS8@qcnZU(Ywp9hhk`V^ooQb=G$G#Q3KDWh!SPy;41t-tz@= zE#7O9NI1U8S_TBxL6c`iU}elQiGt=szKm#Pyu#_xnoAV>{!EJ4dCYy6w%=Dv50y}R zzsrNsmn1CnRA##p0nFH8rlgQaj+E|>MJcJXn@rji{*I+C%zLyJx027Q)Ic1 zYRUBM#a)~h5o@br=uiCg$@*%I$~iFRreZmhZeez=*LytjMzCg?)ML^Y<`_r$lkBw}FN)dD2u+=>B+DQHBytLh>b3&!EnngW z`1eGrc`85ho=|(07)awhXI&cPcI;iH z_lM%6j&r99ytzulJZibWwW;&&3I3h}nV5o6wXg-ke+#shZENB!Bebf5FC1XX1=DW& zjbGG5?_1C}Sih;)Ici;~7RPE=-QJOv@#Ek`M?4zjh0N`|u62VXayACSJ_#{miab@3 zp4o7mm+tynuE+Gj_a-ys@e^ojz+8v6Jdmw}i1;{DpE}tl5+Ci}@b#CL$-uSIc9d_U zhTs%;4J9VoZBT{sUDh{nV`FLsRt@dIreb&qprxU*XUUW@j%k||$8CEt>7Cm?5}0Lh zdnH6FKqH#$Um#Jns+1{vlMjIxz#9W+GwXzg-eIVxv7>EM<;)!f`e7Pj*@321GJU-6 zcl8(JedqOrZ3*eU0|zW7??vyD9+nDi9Zg_(oqBuAz@}hhcSJN+mXs}TP=CaBpqs}< zN?O6Im-Idk5!?mne1DKYE+y1-$D=yb1N>Sixtiqg3RTp#iWHj@hIKJbOj}D=v*0nh4B4dp6a6(Ew2>AdCs(=Hnh0vEp5BThRjmQ6Bp8a zN48s}3E6Dh?Y-~NgUbP*OEHs9Eu>_k2-*`tp{8$ypoQQ@+qwbW(af$M?svRy zEjwz4jNER%+ESE;nLWn1j-NzRUCUuji&hxn4(p+N=Msbke}>2&{3F7X>oNer9s*5wkvU=&mwF@_G zhgwft>U2q=)Jw*;w?U{$kOhDYtJNOu!;+=z13|yVbY?x#v+A9=CD@kzs_55+=QifE zidYY+neEOL%hjZN^$LNQ3ruS`jf{?-6j>uC5XD0Lz|GC5>MS;7->)_KDQaXwTw5t+ zJd$(aCS*g%6=6!mAp(;$EF##$F#p@BJ_T$6{cDZzv6eJHLAtKWx1RC^g$IM1Q|KQP zox{@k#-A@*TWj@-pW^&OM1=0K@G zYNe$=T@G8zxA|5Z zwbD6R#^k#L<~;i`kM46I0mRh?2jG8;KVn$>K4IqN#kLWh*^q(A#<|b_z#3i?T+OecH&`$?;0c{nFlF}8v<3XzE0WC*H6C*xNYvN zn)${)^HjvpA3!p0e;A}NT!#H5@-u{ol>=I{fmcdx$0WM%@BQ=Eq?2(nrp$3%8|xYM zt~(^i^$GhQqIo zAKyg3&r4KvBsHyEwTj)fign#;Zs7)|yj?@%T^<4u+=D@>gZm+i^h*;jAErH(@=9UV z_{;&H+CiDOAM-SoPljs?d8c~#efp(P5~Uc~UCr?u*jaH5!p~qRFoCOxjzf6yT?GCn zalb;QA4DVtb!#Sqiog^MqRXRRGh_)lc@wFBJiNj(0bVz&EUly zf@Q;GwlE5?Br1sTV{ z=stASWP%ANoOhCzmFY=A#SI2c{;WGS>AXSno!7lv)zTv?W^tNvZh1!DlyjvVHkPFy zAHs3I!tDioDrO2<>Gp_+VH|3{)>w}!oF*JFFvvCgb{Uw@Zy!XlO`%Wb1XMUTOToR* z=h4;r^?+pa@Tcd<$Ly7&Fcwo2KXUSGvCk(9`HjN5(T`)+TR!X*I(YuPRFFCo&<6|F z8){bmmdSahadK~I9d()0*aV_zDbA&I#noCU^pM%tis~S|e=9cD6Mf^n=UVxnL{jZWmSoD8 zyd+U=Cu66~A=(mp^Xl72B11i}66PH`7N`?*dlx+b?kM$kbaGb%vF5u)KRE-3baB5| z)_#kY$)8WXH=j9w(yD1|qZvUFP4y;qU}FiKAn`9B!BR+_hw*J~^0K|xjr8C4uN~|i z=^ej3kYwqZb@b)!ry)nIX6_j!ARj@HkwW?%n;J=?+g|TsDT*V3lT?ezL=@HT)|M4| z9*c-37Z&J}sCH4!YCew6m)z#ltoMw5x>)iI`JSWs%Ykp-bL&o1*V|tp!YPouUnhVr z3p$EbNFry3P6{I&I2S50sp-gs+P5Nq+Yi0M%y{5P~HVFfs$-}g`PZk~lvU3fvboSY1L584j9 z54n5|`Q*^pS0gL!7~%I|k8794>0N3+TV_Np^_VCo2s)$vLE~z(I-FPz7qyXU#NR$A z7W+DdI#!@|z{w|1ebH(1skx-4U(CsQJ4Jo76G+=BoC_UX^mHZ_t>N46uafO|8=Dj-=_uY8wp?(aK){Yn`K0b+ZlQY_&b=XHK5hx zC%4pK)z3WRd;H#b;RAQ00C}Ep&j*$QFLH*2_FcMqpX2X)%iI#+{)>dx)8KUoH0LvT zlwpJaZhu^vplJQAq{*Hl9dh_InGJnXKPM+DT9~7BW!n?h><0+hj>Exb3?ITfCu&hU zy1MLtWADx5q3r+t@ex9nNo8NAEKx!wWXX`*Qi`-QBy?xp}#$D6Swrzmeo#I8<}XRyhZKV@`yr_&wkjOxJ>^N`QpR_EjNFP+%&%%|;n+mC5n zksZPv4`{O%^Ru)JqVLxPMs2;*I%qfx>;*Sv_XD9i(s{ZJy4IED(nVNg%sMdK1HZFD z=t7xcNpy$lAtuTC*W%}E4r}}UnD~sDG$7;Y0QoZ{%-c6-lrmlRr9!dh|Y%w&K?Zp5X&-Hx1aAdMxioB ze8yUmx;OTovqDg2|HV4nR~7@f8Op_++dr0Wlz7ii?iW0AQsj9ks%Pyg4TTKi*@@(I zDBn7Pc81QK*L`FIg`UYj?JRnC@J0T!CYR$)c`hD_%D2xLi5ymvL*33mPq`vI=fLCA z{i+ZUrwg>DUuMM9IjM3+-CB~DW|IA)eX&lHBA;BA8D}}(CEEAxdRd#NW%0Z$k4dVP zt#z1U-yNE63EoZ^K$N|&_@6l<>3cph+k{pX^OAClkoaq4p{^{d+^vK6*6}lw0a9{ju(Av7uH^} zA9p$8WcyrO_t!Sd(?2qA@A_b6dED^um#Jv23vBKQXHL6s)@+07!&N2lrEu@PI1#@_ zzoQ?h`};CnvmE*kI+}^Q+=<9m+tS2uxCh577*v?Dm6(Eo^eu;sSO~=b7v)eRe`CO-HJ|bN!g5RiVDg1M(N27Jv;+M6CoMdTnA@gGP|D-qwr9K_uT5Z}A<~e#aSBNJ zk%Y~4+nmP=K76411+bQH&oUehKl+1@ch{}AZ&9dQY5ahQXJaP-dM^}O$zShb9uC=?JuJd|^-+aHgT!d4U6{2S2smW)1jT@4-mM&uC zV96c8GTcC|5<;7MBD7e2b)Z96517f@7#gSrH~y69@>ozBcZn1fw@L6vsv&Y83N%;l zBqu^sA^D(&KYNC;i%|hx!T0qb+mvZqj>LTYykv$n^3*$5I%?VR<=Uz8ZNGTkx&7$W z?H8IGD9(4LHTxv|k14I1GqSaqk$yajhikvUsvDJ+7)Door&qa;e05THH>SM!V~oa& zREax{BL1K6>#)#XWu|>k{^ak(|7bA6GH#Ohh|}(9`>$uuf8*&n_c@Ow7DkxBAl#c` zOa$o}rko`7V@8X^{SMJ}RIS-d~qTV{H*X1(*&2QBH%+Etv8f!v2*8({G zM*RATlnaWziU`Ndo_t1*mgMx1NY}gh(-ZHaRlm)cZ=Xo8*0A6k-!_-?_6Hvaiwl{| z{G(+TY*Le|N*Q`VUJ&ThedA!2TV9lun8Yp>;=T8HoSuY=t|TxdX1Jjz4fI`@Ra3@m zhzyJ`S(8x634}~)u@12FnA{uC$#md{T}z6y$blrQy>DE$J-v2@Ogbudkerj{zOGF3 zgh*5P{={M|nn+H;qFpd&O1>O~W9sZP-e}5MP2c%}*kDGKPNF2AJTb4wzXusN0c2YY zIG~T<3Mbk{zz-Mb7v90=s(a=>6eN!w>iIG6?HOd!F)nfh7D+q&k`;5fno*_YOyR;8 z66j)pw^#5?`mvuT#Wx-^xv?o!5sEH&h7_@@v3T z?ySwH((Q7ipCRwuUB{!K}v#(9NXtT&sE+|pk%ZW6WU zIM9HS9gj0%=7R@m91!X5rksW-XBPMzfH_#n0(U!^2s;KK%_Ge>+AnJSJY`z&fgSdi z>J9&L=cF;y(K>fL<(BLG5HU}cz7yS$s@bY|`gKJ4WdXWGrD$)Jp`xes^*9ePs4e$F z{#FuGf)1^diB5z_ij^9^o7Pz>MEVMC$J)n)4jt%TZV5relGl+R?VlIDIaUkAFD(#OU0aq3@3@eQa(D}Sp2~Ts<;n=L)cMl!2wE7_DS4Ud=2duKU zTN#z*->SjxZxDR+C@uLX9^o#8g{Tq8HXpzwZqzk8-_+O)e;y^1F_f0`+4oIhu9=YY zymQ*8hNP?ZC#oh6p>t7gc#0?Z^Qb9bA?q|4s==<`D<`}F74IW4{qC!_Ak`+1BQX4W zds=W5PcIAw6u}1xZJ=Ik{^i|rLK>F5(%50lc)5YPk!!uQ&zU>qL(5(cl=MV|sxjGp zRvR-M8oV;dFQ#Egp%*{e`$7f3i`rihzSlc}&$9cl7axnip&(1oxH(+_AR`2ifMkH` z+#!N@_t#Y{Y~iO+jMt~!54$>9IM-OXux(1~bh2vlxTo{OXCDdUstD%*4f5q>=U|yIsBN_XRkl@GY#R02VL&6 z3o+2mW7t(IM!2TyTVGW;fO@$3-H=zO%aO-rMIqIfZ^)D+@BZ2-op-qU&hxs?<(oaz zKqn59Ho1!et&0Un@Q=ggV zB18Bh%mdIr1-3{g0&gPWx)4x~edp-zeb2~JQJMR2`=r~7TVaq+nZX=v64VQb8)KI8GV*c%ZeF%P<0L(GKb!*&Nf(pmK0y+(M!cVU+V7gH5;;h!*g zf5qcvi?)C1uemlF^8U^DRQ5_@TldJjqMKp;_3HyF6#a!jtmLvTClt(KpJa7U#??mQuobv7fO^T zPY8{11qbG_iG&&#{>=6yOXoXV$vqeDK%|bwJJRC%n)A*#NlM79yDP6)mn`w#qO%J%cUI?wL4Y3 zV&e>33hzO5Sy%u5h>BLqyI3VP4g6f2==<@h=!#kLfOq-UCgDbM-{e(ishc*G!m8#* znWo23hNwiO?V&_c4Oh>1gK8|Z9@&IlH`B)~_T-%HqrQtG7QMy0pG1_HF{9}96~YiZ z?3a;^%)C7pJ~IfKU9PLnQ7L;5TEGT|RhXq|0oc4Tbw5QHHfbPU>&Ou~*Ji`R-`${_ z7q`bjDKWA0REHcnGF0Qi_j#ueNV2Gj`8=g;L98~1F3h- zrxfDtsk7UoR1IlTYx3g{r6|v9uW%UG57kdiTvGjVkE`c>qR!_ZQ_t(#gF)zOUuN#Y zb>eBi>oOy8I44)W=m_bkirTU>Cq{A)W*E-<3zgF!`7ic<-RhPqx2#0IH){$D%Zc4S zkr~%+DL>}md^KPE{Pq1lx2k>w9%TFM9;Y6FW4Fifh}4S9yR!8=s3b1Ac-Pf61pkmI z{NmHN6!mrRHD=#JEG`q_rF?xlf<;?>S3g~@Lb zw&F*;&VRH!`VRo<|M?X4=7NX<5IF}+(QWC2kZn|gH@X%lR_KhYDNfz0>L%YE=sZ?h zo?q5m-*2fP@r32X8TIWC-c2!!G-3+QpJ5}1&Vf4Att}8qjA-Yur}F^9uQ$dC2To4a znV)k^d1|cdv*Tb98>82MmC#QN!LuWLzJjd+)i`!drA%-0lh%5vv39JFg~aF|c?GoK z=DX6nn5umZVH?Wq7&>S{OPp@r>H&(y?9~Hy}7fP(uZ>6IfvoMcn!wB8g+|d|p&ihDo^^wE}sMX5B|m+~~%5Bl1ZuROY$u zXl+oa=XEqQ4Sl%HkBcrv>G4<4;`4T`?$=KliTEg8m||G)A&^sO_w$oRJ1^Ep@ERPy z@c13OlrSA)027!zgxSok>e4JsXjkdHl*yNsl2Nbj@BNA_f16&{__ptaRlP(&SGdxC zA?xGehNlZ@DS9Yv-lAa1S={4d3`JKFmI`?>>TFbhx1kEVZALwn;B$=eZ7s3+*7ej; z{Y3o%$-|LlJN;2&s4YYb`!w_eUoo(A6`MQf&B2f5ptDiCo%Opk#r!JSSr9#rlEs!7}T zGSIt{9pzZ*4OA^==OCl9%jN0m=y)%wt1jiyl)Qde{g4@3AdlQqJi_r4JS~(F3~2T= z&#@TyVfew>Cadm@$AhzXgBXOf-cs>jE?)R}h0Cms{=z0wRzv50^Ae^bT$U;9V8Oa< z8u}9szHKn40~})>6L?>5VK;Ha{#BZ%j^b*N2u8C%MHh)G7V&+h=`!2wSI!hvryh=4 zLPrj71&zI}5a&uu;ILppnCAi~9M?8iD6cikkKOOAfC^^I($oxua`#FUB4El$F&9HN;&trj(qoo7estbhpDDUq?+r)*NptlEb$BkDs?dL0Ne2!Ujxw7)VX?bhsh+Cob#h@>n4Ny$0nIS_?z8NI z1VUCCdWvlo1pk5n63jDVp3FQ`hF^OnPdU{jO7p}>kJ2=j5^rsF+S$&%;O#dlmBhps zncBuRd&#*=D8Vh*mtsye$>@iIXj;pYhpR zY!8IoVIuhrj&gD#07Ef@0{dmET4L5GD_%HQ@#K|u?)3J%XyGF65oobRt*of@c#5r! zzyCOo8;PO((qGFRG)CVbup*y_^A&@Lwyw;Vt!Ko+Z6aLL`Y$=6MJ_ixyJ8;aDpkko zUm~21O>5DXqn_PBA>1bcA>|ysIvmZ3XwMtT;0Wk~&KI8SCbdQ^l{(|Jt_r&MQikCN zOTV@}{jvZ;M%AGgHc;c?*wLNCu*5Aj49^Ns_JugakTAUs>`uq37$cB)1Hjmslb-s4 z0_Re_`iPN-g+ePgic8B=SCFn-u^Ca5-!wk5UOOdwRIuf&V^0m#E3&Ev8&DT&hYK+^ zEurdrQ>dJEYw})ion~)F-IID39(Uxn)+wzUs)iDM6%k6;<8fEowAlycC{sXG0aoYV z>=k>3D**ch$}T}JU_aF?f<aJ4^}++itv%Za17;kgm|(|E;fO&$Y!5MLo>+f~ax) zx4W`TZ`o(9nP~=oO}PBwc`wS))Cgu?j?7T1_Y*IXu55f$70*6S+G24jit<^hHa5X+ z7N>2hEsCzehNNzVRVg$*_#CC?WO19O#>l`?4~9y0*DtpQ2DAetqjuf7z?_<~XnaB$ zR;1~HbDsyWU|RZudc{Qq4O>N9O@0xS*W}WkME9YDtq-z@vC;Sp~m+j{Je)9M; zVdNoTza6j)O@l7Xc<#sk9mE#!q!ys!E4f+^5cB7jKPGudY~i8M7Oy*SDj9w?(oj;` z0iHHNI|80o0OBUtzM9d=GHPb|*2*JYo|lcM&g2(T_ZT&+o{aYK?JUZF-&=%S!g>l{ zkFBPb@zGZFP&XXzn&DqM6s@a7IP+yb&v4@6r@U;~Wtz4vEG%s^+@6MF+fu%BHIWgA zy|FN`6Cgx!VjYHA9ki5NhusX7H9Qj1?g%c)Pe^_V(diqrby4qW>pa6YX-?nij&LZ` zB*4n#%GE|&xREAx&vD&);iT2z`R>;TOG_=%FBh#kXw|yAx+WdU{^9(!?&(88z~SL2 zxh1gCHejPCV8EfQCm6{htC!%x94BG%u>RavIW8Wm#nH20^LMaZ?m76)An#|WS7YiE z38D@lAW$4v2)#8UJ@lQ!t7j>MxrY?|*-p><%++!piN$yGYuDm9v){N3H`_~uD1|-^ zoR9K5qZAJ~48TtB5fUVP0_u0aIh{9V^)xsxg}t!4V*kFvV+u5q*KH%GH}-M5tm8%b zOjDL8q}on$9ac|g41PX8zuzL6;Ra66A;<;;OTMP>j9Fzn=!ngSsu}xhtb$lQD|C3$ z--cjni<9(ENP<{e)8X`IHhk;1OR8F|tYn!4uV1S?$#{dkstEEAt39o3p^&3%s5&M} zGK%X~OO{po<*L`%8HL9jqvKQKx0IUiJL{qRKp>g~V(?^e2=XxWo!BwrLy!&WOE>H^ zx1_QrTU>mV>!BEbvbfP9uKvW%7Lx+8m-=xA*RoK%cc2IHg~LF#%KHrhXi{k8oL|=v z*L!Mol!#_x_58_ZhXWCL%}(lki5z#Hxkd%dc@uJgnh4R_+)j1k^@VWk`)v^En5N~3 z_XKR?oUZ$_uXjqxvd%F;{&n%w;=`A!wPW{A6fe5Rp$W)m;s;_CsID#}n+4I-{JyS> zsLMRTM^7;ySW<0~ifTHvT!s0?R2!2(EE1seMj#xZ2mB7wdKUwn z;uEmhnk6+R)YUr>J1us7$!A*pwUN7Ya^?ZB|XQ%<3K*N|;) zWHHr|4b;5DNO4#EFVBA*DWhqG7F2%l(ca;sRPX*lJG(;guu7}pr>(@}o5Fu~6}YWI zAkt1w#J=~c7n2$52`E>63j^p%%*Trr3eff$ZXdqP>f77BxM=iMJ3eA2NIY2@}#r z%r~gTKBUZAD;7~xpZkpD8sXw5+i7QdZWhq)n++Y~u<{I7UgC&8YI^xNIW{_Q@2#_! zK3HKoi8TuVgx62-+qcWGLgReu{DtfC2R^uLJ0Yr}a%kQLsR4guRI8U_!wAU8k z>#@1?;pgn<` zsu)2>Q*|4}Wf(aLSGo^WO3ZXLxQ!Z?>!@TU%E=y7PKp;zyn!lNB>KC@MxL9L2xdPU zBd>fNG8KvKe3|Oh=(pQf5cZ;~tPX+HW8#g;i`VT(Z4(VCIbN8LRh}#2eRY0pbmva` z)zOxIz>bqdno5{qDICT9HCHwCMbC>8XQ6Jh5_Nok+f91NSWRG^J@XYp?#vqR##yPP5xyCyu=o+~vZiF(BE?dx7I)S*_feV8g{ubAthO zvy`hD4y_J%P*VJDIc~4!d(R6pOT~GMoyhe_5cz?3q$hTGF2K@1ieCXd!(+j96VJzu zj_f48f&&dFqV>K-N9R5ITouTEba$ZKGKTC@Oewa4-GB_=W@{T$vl=5^QEJ56Rkp`W zy2o}@s`uNVA*p3+W@ANDrA0>uix6_3v`PIeNs&LSofob8>aX*qLCsyA^S9#qy#&>>) z2}*bhLDTJEh&o(Fq3gjm_ZTlSE*TxzcSt`*k#m<@VU;AywQ|g;*IeufJ_09*8SSYX zJU)hN_b9tAY!dnkOI0X-zWry?4!w3up;9}4^{L`Wh$ICwB9FYEemetYHkp0qjDHd= z)9hq_7as|D;wABetG}=18*iD~N(t{mvArX?E;dG%>^`UoV@*gzBAk5{Xs2Paf#4}wRv5N`AFsSu~`D|_asD~y0v1A z+s;(Q%J`YN8V}X2n>SL;I<3xR+ez}_p z-P;G1Cf{jg@MM_J8&IySPs9gCblDKihp=vXOI@Z#PtVWNa=9UEMU;>*R6d*HzFC z`nlG$z{v%@!kzU=7t8%mg#}Oa>fY3@8(_Jm9$iS?MA)|1ZEh3KLnc&gboEswj_NiwA%K25HGE18keMs#k8)RHZre$9UE z?W9vkjR`8`QPPvOMnhHJB#wYz3*ukBLFD-v9^W> zH*+&vUo3jG@O~ROJb(j6mr7}8^zo-G{EF%>OpTtFTW;`Jn zFG~b5>(_L9>apmQw=?daBnygLoin2EAGef|lHlM=*FHhwGP&7P>O?F+-i$yPZ;=t? z`3Ql_sgjRsOC0cob~$v%7bV<`9X^)h{-jMx`Z|-1`JF=za<-X^h|T=)POW2?Vjh}_ zvvn0bmuNraz}Hn2`mP#xzDub8y!SGdH*Wj)_gqZnJDSSt)zZ*{zg8z{Gs3ZU3w_z< zC9nvUt;TP6_wr?_K5y4w+7;3}?LL+p`euuG6cyM=?ztcE-hY=T-`eZBJ_6{kbFIQ` z^-dpqhu@k6;Q9>mOS^OB6B1<=Z68`+zi_SY&imbOY=Y{&35^Mi3f~JOEzHHyM@)kI zwaT*k4KilVMG5!+g0zI&{~##e6EdpD&~#4~7T(yel9XuJ7R%|r9zc`zy$$%;UNJ~T zT#I@mfe}EET2oPfm$Jm`4A{``v?e*Q42@4ukB zWYf3?F9p211RMlS+y+WV`DY<;s-E2Rt1ES4Dicn9Q#Av%i-a>2KL)r++T(weCP?;D zoSL+_eRseRg|jM;eiX(zuB!jC(@0%Us@<40b$d~4F<Ah&Pbh-%vkXec$# z&+?Vq-<{|uv~Sc>K5Q{{B2SX%Tp;epLC#JIv6*3<3{4>Nd2vj zQ{!c^@&3kJjahM;H~VMwp8g!BD%N0%wmzph%;3ceh#_s>11H<&&5bFaQqG5bm9rS! z)p9F~G|(Iyu}_K%oz~ET<@ZxSy46GH36!2X;i>A62YgL!UgKV+q(@or<6l%YU%LE` z<=(Z$dP5e`p5<*@)qCQ+W5}U3L_WG%E28XV=BgOxtHxEdDzL?Y&nEiTFaGs8ybr(m z*dCn^&$=dX{!~_(rBWf~`HzY$*y(gWIlu3P9DK{UHjnQ(W9GzU}Puh(w|xyxd13+h|EuadHPWw`EM&+jKg zq$FnAkefc{DEl~P8fiQoTe1niouR)Vu20SlH28kk{0oTv&OObfi*za=(%wtwlj^_> zpOpv9rac6;D`fN$V%W~aY4Q5GJ zQ_aZWVa`|&!K-=ssI-;dV@?%}IB<@#2@$38!!^W*2gM{uWgZmuJC)x2$n`ay(+j;e z^5kj|57W`ZdM596V-nD_t9O6Ci&UEBE``zRN_yA4$`=y*4% zwDaGYY4)FPGyFZV^v9o(f0*g@$N66WfB&8D4=tO;PMo#Fi;BQq>*2sAplmJg*RG?- z<^040^A7y2ms48+G;f~qvP%7}{qtWm1NPq>WA;xsaQ=~^|J{H8tJ04D>;N+oY+_yz zaq`fI=7i)cZd@5qj?0KKv~8MLG)FAKGT;>obP zys@>AFP>3s%V-2+Y-@+xHJ);?N>{*anwsl@*FX-}Fl-8X#Q6>T}x->W`N<#@g1GJM_pQnXU% zB#{*i7dV86r6D@(T@30P#sC1lc@{((gcd{QA*9}!a|3mf8;WP3pf{m(eNaNkLI5*h z(V&ZyTnf^pLT89n3<~pQV=ABNOBvfshT<>tcaoCj)5cX5DpOa(-M~lAAt_` zu>{0&8ML}Sfog34e%%J@CuI{!pkV_gnvO_hNPhcZTKZ>fC)n)iPmncZ9Zp0`>${^-!aS2i)H_0ytU=5QfI z;=)eI4>Lh?j)ll++`PQDI4jN6ug#j35l6J!K=C1M!#mBqT7CCiSr4A+fBp1O_+cjY z^6N|O#1s=a(gy5y&)jCu?QfgC-V9i*6@=Je~dcDBcKcerc>^2f4ino^1N*>d*C z&h{ULPHkHj9+MnIPL2pf7IlI8f6!e0Czd7*y2ATa>FE(;ct+UzFcRB@8(9MT!a)Hb z{$=2qPIL%v7>NlK=lDQXcSH_!&KG2>q}Code(w#7{pI3k3X|hK%Y&S*I=eC6c1r}% z0Rp~s$sG_?3>L-C+-ZZB-+^q?ElwBtbj)1Lgtn+a9X@OwOfZn&l~h%^SqZL?l`Ra5tr4YP;_)P!{K#l!Tcigm6uzCtWR<}T<| za`^bPp#HC zmjV)n2s09@J0wb%ub`)JBc3#NsBy*@^dG$^0I|@XHOpvqhQ|7_YX*yH9N$4f)`YLy zXs&8iDtA}8TTZCTUFFAX9bxqsn~I|{e0ioEpP{V~iva{J04QW3$)_TIfINoi{3_m- z-_uSl(T@6RV^$O#iR&lip`b6rvll_$gmGR|3`DVkU;>jp+(zdWNq6;P!h?qhUfj>5 z=TN3ni|ie4!&OaOJu7O?`!7``nKJrz9c;P88;c{)(~d?Wt*NY-m4gn%^$TK9`BqT( zwL1m3K8jywPvaK;m%!U$&t|g$owgqo|}%AdB<+a{lo!e9SRPXt`E%~ z{9c?WF1M638OJv8I#5-=&KT>{Awhh-@bDX5g^Febxon`8jm%*+UxN))TqptR)t?2k zR+LfD>xF;!9OeHe<0A-*Eubu@i`fvA7Lfpek#KjeQ+mGzOlU4kKDzz6rj1ABF1xAe z_-5#Wuzsd%&a_!+#j9V;O{E%Bd3w&&U;_vgL)fP!W*HQGnR4P7!iC5Z77fZWb=$DB z)dOIhl03?zIO$NIm64CionL|n#BQbUB__1&gh<1H*IGE>cvDK{Yn#!0_g90b9Tb{1 z`Gz?k-xQ-q0$_s2RQ9g{zS!v@69!8Ue3b4sHfA2l#?W6uGxo)SNw4?*tQ!8|=~oU# z&^rMSSYG;)NbU8`-+V6i2nJpe&U;-*skK%9#|9Uz%9Cgj0Se@dKWmf zR^T9?+Pp~}k(|gw(hH3^44Ci{_%}McoihAZ_4aR^)mih~9$7#@dt*1Xcj^DDsqh~W<_G_*=?(r3h6iJ4*8vjNsXsr{^xv>c zh)r7I;04d2|CtvgB9elevPC@cUyT`tBGQ5pEf=Gm^4(4;ZdeFyQnyc4=x|;!+HDFPI|{b@+-a;>oQc+HFi9ld0pp<7fS9_`GDL|% zxFxl6{9+wnh7xrSipm~(*Pq()eYvknA3js#>k4G=INx>t>bm6A3mu>ua+tU%2f z{@1T>pe|U+GSlA@N6{e7tGZS$367c?^iw3SNbQl31`R+sy7AbD`$9fRS{lI2E1^&QweFFtd2#`A8KwSeGQ;Z0! z5Rr3O%0-0u103OyQDSf0-!ed2w;6zpj+hYQ`=mMQXbeE#*+BJ!bx@+R$}0KY9@;JT zbsu^!^k`>E=-H!$DFPz71(5laX9FKVpS+|cwF@nbCv|9Ideq!Fkeq~=?| z?9g{>a|u9;9)d_}SZ8#~_}eUBb0y zO*cbPc)Cb_3!UGctc_h31$R7ggRpu!4gD1$MaaxGP~}#ezzXitPcf4Hwlki1$J7Cw z{BGRMM|E45KZfg8RKI6-#EfY6{2VE!3xNUv)*I7h)t8`7<6n36Y}KrhHULGT0Rci{ zBDxS@X@Q0oPKcxjIjXdp)GR#kY8>VEaPT6y-CH$e8Xm80eOY$<=mDY?5)Fj#K`?QF zcy)Av&abzDYE)ha{LK0l@V_D52OM0p*`K*kg#Kw1l#T;KAUX4}b&7o* zBi_`5RCGjvS^*7HXkj3#bu-v-^X0GH*-HFs`51NRRA%E56sX2R0W)h3KoS|%;|h}M zhquw0VXtv+ue$Yw$^@I$q(e1R-#-2_L_Tz(ah%+<91f9N2(a8iMDcIj+W=r2+31-V zvH&AS7>hj4go*}HPzq?>T0poRz^an{n(;+z)x;R$_JL}kCakwp74UC|8Td1?6N~b% z%Yksf6#N^XZSfX)15Vl6x z0Vjk9TnfDEdynx*OU8E>8*E8V_)RC75 z-Sn;E{I!737kKFzL^9A~!>m6qz%k^aiQl#iq}aF6B!8O25ZGe{XhIUtOq}~yn-~YL z#xUCGSOD1rdq98x1`hqDqGR57av$co=7EP5{zosx#s}dh%@yfF zK)kTpItT@M8?XST+_Nb)h+Rg}1pur20#u8EC}yq;-oOgmdK%_ru@@;GhdMg%?usj2M$xm%WRR zs$Sr)iKWHH4-sFV2hxHFk>d~U4rqw*+T9P0S_+sp(CIx3_<7(z?!8RCh}50F2+qoN z0;Q@>RP`h$rDfF!pZspth*`6^!=Vg!wz?wJr zRReG44j?@Nu#^azYucs`@6hQYQ5bqBWjpS$9earcnS4Dm8A&52xa>L6@uM_EXc<_3 zoH$kvtdOr1+GIW(&5#As)Tdby(J9(Mtu$hnd9?+yHvJE}bLQY|<@PQQIZ>aUqH`qi zA6us-ZG~1F35&~Ah%O8~30CXwa?7_)Dp4bf`+xlKiC2GIG9VRMgZ++#5Ieta+8p}{ zf3Jbk|D<|m85iu+(2HfDiJ%<~?=KEgzK9caoFBn{a8QySrz!6<^0_l7bw7P>OvdpU z&*6JLEBi9RZVmwQos3~~NF3tr3XRa}$^0eJ{bt8Cxk>Gps{MYghgJ8HmI>29xMApF zR#S|iAH?Nz=I{|;9T{ZjpgU%HdkJX9&IO^ILvO*|fVe+V)~qnu&O17|So^)CL!#c& z5VJ_dO$<|&aCuZ*EwtVHwb9_Jv(eeI(ui9%KJKT!mWV~8m1}L^$Zn}Cz}i3Ejq&M~ zU5^h((l)1$ZjA)DHS;|;GJxJW9W{c@cQk9etXFpS@vkQM{byuyWX}@kE9jH{FjD$@ ziLjOeO9RDYSB+g|%>(=G{hgU!b(2VL1#mpl6?3XQLp!R~-vj&hSp`$$1}X{2Qd1~o zU2H8F5&y0w)L9+zT-CU+Fv;Qb&qvi+y~pYeww77RZc)9BPjvbjN_|Zi17BvzrpQDp zyRtY1)j`f}1ey0A7nggVY7Y1a1Ddsc`u2J&@%2OPL+`ahh@`dEXEKPe(I}b`nM^6a@>8D@5}lOf=QwDUJXgi+SudxOr6SHdju_#=Cm)zGF>G z=k*hQ&5-0J-3k^qr<;$iUnh@W?^*QX-Ftzp)0|fA_oWKm*H8tl#24;>5!(&3nCbOn zKrVnuTscj(KQOiUb$n^hL(`(v z!-sje@A{xtGC{Ln_0lS^E8=_Md|<1&5P+?A+nn~|-?IFFsvljWrk-G4B242)V(7y6 z0ACxxg%mI`Hhshl47IZh%h}!OtU0K!7vESCc`~I_`__O3c#rj-C1tkt&JC2eIV}wV zp2wmsjsqyTv+daBICa4M%EbRL^8*(AlU0ZPduxu^W%f^=ssHG0BY(Xw;E%09wUaDk zsDG3A%8)9tK!Ai1M4Is;n&&8-qnLO`*$DT^v6Doa##4!<`Df{K*O=sDNmb2Ic_H-B zfX>f}oQp=Pn`Cb(HC^=&}S4gY#sXaq3&8F?F^?V6#SC**7XS|=Hx zeETLCuREM4P{nPqn#6ed?%hqgeCDN#7fs|dxZPyqW^{I4(SP>HB68h)G?cEF4JhdW z&Qgjs*hkRwHg*w5xwDsz&g({?dciTW=ky60bmWptMIl>MktFB4~*iW!=^)Y|GL~Jbn?JKJP^0ZZX zMhuY*2Ak|{nQx|Up!9*E%jA??F%|AP@ibSWq{FuO;RD_RLaF{y!3$D$xqjoaRdvuK z0$rH81d;3!03z!%gJ0KGh6tBYkQSceVL*T3I*Mm_v;+IU6xdID^yileE3-g&yu?m$ zLc73AzDfcozqRM*HdWV5l;1Q&Dg?*$DOmL|kPcoOs0>v;r1l>Lu6>7TmVV%Tb3oS! z{#GEox{umEdC!E?+wtf`WpY1%>IKW|7D+dqntZ>gn~$6U&)p>ElNP#E=^DZ@wSf{| z2Z7J?SZd~Q6lP_@7Az&t9Hw3m05MTX;HNu+JrcuyztIEwtg&$0pRV-$RQZ7gx9QOz zLm&nSOg_gI5$XJ|R|UWeRQolh0%JeW*{|+yQRZ8^37mW5NbZ&%)AwK$Joqhrtq?=7-uyJA+m+6xAiM!c%ain^3(xmxQQ16lJ!D*l4GV>O1pjRFA! zdx=^924hr~0;&L+;E*&{vVqDHh1M@{Ls6+?+L$?!&1n564CNHAAMhzLlxT+uT>tUj za}?TT+`Ya1sSD zXto7Y*hXdMjc6$P1qNPgM-5-YLrwFvmaA7q7ujtbG9>LCM1n;-?`aP_6C6I^g9SK;%c5#mA68ZdDVU!1c2U$bcszg{K&zf1bovDbmD`K#YAS!EozR zlq@jK=xg9k@1f7iT1TN?{F@Eb_e2792lkdZ4VaW93ur=^C<64531Hpj6PurSY(p*M zgK*o*FH<{4K`R57_ZF#?OH|`@*UVl6xv;i(3K_ihLM_`kc^kp-W=)9P9s?gRGXxzW zuG-*^wU`lHEQo4CKMmKahQVq~7zcnuvU7_+w#i@>XpA5aVEbkg^N0-?4!=GK<8fNN zgM{HMbi(bN&F`NvH92xe_-3}-t+v{9k7!BVLIWu(uKNH$ABE_=L#&I}sS4#nsB@r4 z)2B50=p+%yE^<~VhGJ|IErVn{ciKQ1juWV;5}T<{*a0js|9jX*0D$!#K*}d@33VrE z{30*#I>pWOGsac;AMTAmEO}bvf%X^UUDp?#*5%F$>GG@~g2Q<`p?Viz*%k)3x*hUZ z8Y(Hr&Ln-8-EtB{uC+`UWyJ9&KRKxPO*S-rAOR6rgQzQtw0RJAY{j}D*lR7}4M^tS zoRRX;AEw9vq>dV-*aa)ch#Em9BO(6L+gV#JAK3SbweT?@G^}BQu+RAKp{i8G0&C=< z=I|J-+4t3`Z;v z(UciKynBgc65S?FQXi*4X zb9ZZJtOdx+4e=;xFh#WntV6@RsY3PyT&;rf=&I8)$2@_UE-Qd z{557KB(q+$E)1k0vELmr{#TP9x4`iaKG7DJMz43RhXd=^wGWS+Zm7bnDmdWQPw;_k z6L!{oMvgH;yaD}G#h{?xl1yk?>O9kYCkO%YAp-bf0zI{%4o?&Dgcx#h#6d2K5^ltY zb`NTtqg!HDRAzt>G?zyCzWCj|e&JV*S+s?IwXT7l1m-n>#_YxR>;|0<5cu6gu~!%0 zOBlg2!mY4KoPjpRT&+K|LX9wWZq%GE2K*`@DNa+_tYF(Q46l=$V=4D|x2l79ywb~N z=3TdP&|eO#_bi`NU4OMn1+JI@%Ghc@oP(XQ-;|**zB4nFfg>r812_W=uKs^f?%2P- z08F{T20B>#_is?k2I`$Zw%Z7i=mM;!4z)HDAaW{1P5=7~ghl7y{yOTnuQD^Piy%DU zv4QHIKme!U#*j2%u0=t|@vM~!-l}6$)iG#aN7zd${=2oz`%u%oxbug~bku(w|KxKn zsGUf1b=%!T^MXjSYTzDDzS}V?GSm2VX?x~n(PaX4AGlZzQoofThTOXV%*AP4bDC4C z6a6HXoaYNz^sJv2h&io53|U7MeHct8FN&e_&LXDwD3VBRE-+h8*bt}+o7Y3>yyF12 z@7E71e*GW~-G299(tv;ED5ih5;QWiXE}}iW+V9E`?CR(;xdqme@q0-Q`kxBz|8BAi zjRgF`Hh(mJ16a2RH=yO?de{c3Qg1=xd)6CyfG$sQg*8NwGbLkq8rOH+`eiXFE(*%^ zqKid=K;b273nRosk)aSxY}Ef-uo|1z0xJ(7=L{&tNQvWb7OpMy2u~ND-$3n4hc9lR zE)|+E_9PQ~KKNttCa}6Ba;BJSf|NQy>zBEqN9lLg|I#QrmV-#1!2LYFRv)vz^Z_WR zUOBpVj1F#214xMd8>r6$l}DhS=SfH+2{%>?T3gn@biLJUa2@R$-K7m{2l?N1oI&KM zoQ+~p=}n@Xk_IA%eQ{^AK33|Q%_Nm z{Saz&XmH39>mv~<5&!j``N}w!JWYfpK8ey#mx8Zrz1V~tAHtGH#j)f_yg0DMw57o7 zWy#TRnhXMeXgfF@{t)RZ@d_T6GnfkiBP(QL=F~>}%Fj{FNAJf?SBke!zAyP;l=3)1 z=DJfvSi3A$^yck3Ho;lkCs+FMIuJU@1W{l+2VfILpuwB*fj0vh|FfMIiyabl8;HRb zr13Em;)J0XxWcLcc;W?379tr|@&i;Sk#aWYOz(>|R*}U5YbW0HeQis>Xe20cbV6Ym z0J&wGHGCQQWZf9Wkp< z;2NMc{D;}-kXsgFWOsc!|Hc#oym+Up#|LNFu1?G6hykD1u`hT z9nKr5v_hyL3gruoYu5jVyEl)As{jASM?!Xrl%=UC3b_iYEQc0Lk~S&IR4Ut5+FX|8 zh!DkumMfL1R9a|SNJ);dx1wZ3HDktBwi#ItbLRAWjjru;U7zoLf4}$Rb3Y!x$NfiL zU1w&_d7tj3`l&sBMyO7r2brSI>L=*2-d+9CJp^6FzaE8+{0#amg1q% z^B_-Xc6@Ncpt0+@NAL1R#-{qB}Kz?%Ceo_YC39sljJfi8Y!rV$xSCu^X$tU~uRf zkc=d}c@*pzj=;Gug>m1o0{TNK7th%0(?uEx+6Lp|o(6c6vfa9CF-D!KVx*W9{P9|n!q6@vdMLbz?`f~o&=Euedh4^%c`*w zP>jJe<`iVepo=!@1m6_b5RoqA2)P^k2)_3daHf2m4`;eMA1vDg5Ga5Iq^fd8AEVu0 zx)b>@sVY+o-z-{#OY_{5{T4PkCNMXB{=D+`n{`umd;ZdNIX&7u+MM<^159f+p}uR4 z8DY{~EQ_?NO33CrWo9H#;wIY4#FIkF3HKofVqqT~{%FE?$`nv*7vWV*s3W%uP}~r< zxSH8N3qp=Tx(F{tXpeiARrUld91(|VE&WhGQ20$;bpuPFxOr=cOO-IvdwHh=*kQFgG z#ebbYsqLj1qNFfpCn^a1MJ)(P#c1e)t87|z-@+GNg zeGM;I`93&GyUAoB`;_Ypvhm)c<*A!fYm8%kiF`hY*Dk(q~KYj5!HoCR1 z{OR?KCdVXui^JdImu@(Dd|$Yxs7$EDFZru`ei_8LLE%AcF|LM8mWU-UAh_Va5!_TD zIEg)Ng4Ve`{X?`9i4VXi5&wdXu`BC4CF4B!cw&ygJx|twdu27snv8Vb_ynxPSOZ&Z zNiK&t*J!U3NYYhsm&xs^+v&VrAN*!G5iWx&*SP3QZSv@V>F%=f2vf#!i|lWvX518> z7E34V%(5%2)?Lm_f$>|cbq+|qt#G(YHOX%b8KgBCroELsqK>-jQ6e@lzBWTZdm0RZ zsTo;pb{rX607+S%rf8ADfAI+=?DTT^4F&d&LE`OpU^~_z0!L6Igq9%T7(l{Atu>s0 z2ve9|?dy!e^yqxzTPqz67@HLAeZm;gI}vv|4BEbw#2U zca4B)mV!zKUsdI0$ELuu!cX4S4(sYxUa95wFPOk(<68Fa7a4t!qt-htnkNNdxS4Av zq5zeRu0Ac*+mUHZm?6Qm5nILyL-NKa5TPqunZwJNf+A>Z-#TPx0zBFbtoCKgX{r8W z)UH#I-f1kZMYpRUJFsJrO)*E};c(GI;|8c*>%}d!-Z)7r+!?|UJio4R5O1A3rN2JM z-djN$k#7_S(sWQwtosKJ?-IY+{hhLbvy28V)F_Bl_e5@lpJl_=zfHKTdPmlZx_y!&c%V(2&I-%&sfjQ!ntZFo$Y@wrtQBW>MqO#&eKGX%a%t zmWJy zqJp3T;s(7XSbX3;G7um5o#OL?LLNc5o%E5VoC;+03m~vBhb+mZ$<1GAU2I_O<>-sa zw7H zYSEuA_;so<6oJs>3@HPp-61PUYxr+eRC$zHuqLq{Kfj|eI3e$B{@KQ@TXJsAUuN>+ zCgW*x%BCSE|0Zw+lOx!iUPTyB0CeLE8&JVcL0L}|5W!|F8W>a;p6TMp9CG!L{$fvU zdky^454T@Veu8d?v}VH+Ot1!N4lSM2;vUCPHuGL5l+7tv^YKDZXd-u``tpkA;WVSt z*aqxAf+xw56JU>4EOjTewxUAKZmMAa^eRF#99cvfXn%078xi3dWBu^|I8ee``89$8Ip^s zqi3^!xQi1)N5IVhxGG_&2-ebpvMgaf8ubHOh25ujzvdUPqh>_9*O}QYZs}R{p=Ls1 zeK7y0=8E^eRUOUeRW0@D+A@6(9h!I;O_2HD!0|@sPm4bDL0tpXX zHx}nF-3G1btG&q?k?^QjfXx5+0YH!r$ymR{V<dOIN-FI^@j_sdB88ufq=~E8ETb_H{ZeVa)+iEd~oG!fKh0pb0@-Ju(3U> zL-Dz0;5tT9cA~ZPTo47i`6TslGGyR$diWp=bWql;DD$WahNj)5NIN`bIHNDo#_ZOY z1~nbU)8FRJGtFG|ZvPs9d1?EMi3hzGB74iO>_C@9&%gZ0alXAWWX1m8szSPfl)^7K z3TTzzDHHIgbQt=fR_F3ZSPdc&0tdntFzUXc%x+M-sM1pigM6 zMsq5jgZYq$!=}jT!A2QG{S>)`O->3^Qsc1DM$OL>H}3Moru# zNl;01FJs~hMWg~paE_-ECtP1eQ_;_>v$4GC9hh;fl>AWi)Z1la0K-4>(#>Ps2+!fQ za4vGzC`-r0bmsj+oH6sI!rVpVo5GgU6sM1anj#O8lM(WelLFxS${z_TgR#);mwf61 z*R%n;`SeU8RY2j#ED1Z|^-w^b5n8qu7H3+Atryv~?oHb`At3xK9LbKv4a-6EdKE4>bGnE^+%IY(&-@BsiGswJRtNce|FL35Q&3)B%#g)^?Q-7JKrhh5R$_HA=<-9Ohr3y!VAQ7F-ZW_f0tq8Qg1pC^I+X$s1pL%M?smv z#0qKPAK0&}R}4wTyOQPwJR>TuZzk51hgoa@+R8mgj|a+gqtue5sqTA#6vAjoWgioHMmKQ zc+wy8AJ|>37$gGP`6~dhSREQU4`VYWt_C2jIS9AW3^yrSo0k^()5*EFcA$BthgFBW z1jbwI@>NxWLNlkmPk;IJUZ!h!`r>ATRck8`b7N_Jw{D#bV9Sx`5fZ9CrWRZ2LLaPf zSlH()0H7AEk-PMz-^5C=ikR!16BQj3Bn~W|g0fzNSHxX~uoU#h-X>Ou_2wLU33r9< z=t(nLJLSlma!?lr8NWdEu&V(G{vfCd^)shIgVjbQklTkPdNLiUqD>5#wgCUcr)q^o zxi2a6Xc$y{88AOtP;=1Wc%#k9JJL!0wH-u8Kx=#SM946Z-v*oWJK{A3Q27W~B}=qX zGW|Mw%;0~KdLH_3K<^o{Gvpr*?r6I;vEo3qNP>gmVIvEeV?a27`ga_>U!?hejDwLu zi$HXA&45(Dp2pg-S@sG`V7tJDfdkVJSu}2fIB^stHjst76aeHfa1nzJhT~BM42`S; z@osESy*xjM(5m3#m#75{|D0EONEuR!7dWVP{s6bfsBts)gf9bFwH-q|TF0Bc`GTYNgonrz|- zR#7zxDe*-M8t!c4ar0}$D#7OPI)`HGpzCSti+3*{+2A++8?D0+_!G1*YquzJRxmO3 z3mj(@)@VjK2r0k(jz2ym{`ioN$Ai4Eg0UT`n7}U^@XRQ*Xof)tL?p(W%D_!J#1p}A zVr7B=tNMbDUE+u*I{tpCOB@Szmfuh~J!m=rt`ARaZlT-?&QriuS)i*0?(`hs+apoq8^;6f=8f$Wik@L@x>Gh zo`WB7K*9n*?UFh`7?++8y`gn@oD%mKM4i1^=$>eC?a{k#k`=CXVfy35YL3C&*tT`y zGqG3IC$(2Y@?*U!u8&l4;%E9=DV@^l= zwncKe=nZ&`M0cEgQi4Cia1-#iRiN|~w^}@(^OuJ6-|H{n#UF zQ>KA%T7v2IxVr-YxX(Lp?e%b4eT_X5iI$tgQffvG0HMX&BH)u^i)4ijNuY6z&_+Hv z0W_A3kvJX=WF~MP8TMET28(P3!!0+S`thi0Uw4nbK_3J(EJ*ab@FIR=>d?&ee2w%* z&tX8E>SJewcIIZ2ik7P6eA+JbPe-rQZ;oDg_hSfeHmGQ;;1dCGbg!3ziwNAy<*UZ< zq;b&i*F5pzwUDfGNx&pkCc#BFA^=al8aWH!FSHF&K3wjZ+PKME*>1$3a(R*O|l8NPS^?dU`>O20>2ttJ6p+o zn|(g!YVXfdY5O4oUVyS5(gd5YF0h;UNL9Ri0E8Pt%b?y<*B|++8M>TzM}%;YegOVu7z7wM+c3cXN5XD#q$>U)#v z_1(pj9+n-{cGW=x8kMr5U+mjmGo8(2mvTntM1ehX4?NH*AU4CW1hn6Mz)U391bIp@ zm!==k$no7M_aQpq0*GEV0=Ae6c~-z5F>gZgm46cWE}-wFq%sJIK$QEy%k!kgrPaeN z)&>iQAN5yrnj*qW4U#qqk!{A}{Y6yYgV%9a*LT65I~#^ht(e)dXlL?Nw-Zv8nQ*D*eAsyHDCc_9N2ZoSvpWyDr8+dkWY}$)Hjk( zRsZ}nRq|=A-21mNzT0G7WU&{jLnUzBQ1*3~GYJU{h7RC1Qw8~V2Xg&r#YK4+M2;1a z)w|E1*9lrRp;|viiaCfvh^eaP2lc!j{*!tPf9&E#*u}q4)E>z$qJQ7B-(<=gj{G0= z$Q)Ao%Z=Ztj$F$@dkukvBtU8df*Uf+sM^G2aRSV;nvEy>4JjWkE7rMx4}0{H+2*d~ zu+?hD+!d=T&F}qX_dwwO|3U-C1S0# zXTZ)A(Yc@g+Ng4a)R{l~QI3p|l+H!E55vYSskO6=sv~E@sKzPzZ!N9EgL{|#+Tg-9);@KV{wnMXt+N(l`UtKc zkF*waey0rg@*!cy6K@(P=e3Hy5){c!3_pH?oB$eb1qLV=3#Af!>HoZhRld`IN05hh zMxYExGyGsi&YTJ7vk0vVuEY91Re`e-;!c0R#1!UL1M4D}rcPsYe7meV1g6yS<6ziR z2fPPR(eQrAI=F@KW}4yp3F66O9a8)e6&Aw1R?*$Rjnt)nAt8+r)>!lBf`EM`YYa^P z-#=~OF04?}D1o&L<7Qy_C^7LRPZikcgNT=Tf#FlsSkSpK_o@YD4s{w(Gy za6VP>-nw!XO_$aoW<8L%Y&Sf4rvnS8=AfLH!Q|qto@07v&J}$AcM9zsHoD9fz1KrQ zlZLAhe5NO%AsX6?ug#6zi@z6KSYKOY*3!A6@(OBR;v2JoCH<49q0X-=auLL>=}EB5 z0G<|sue;ry4i@Vxd1T-=EK+CT;w`muIg#=(t#$p-X*@S8QXMzE=E@AqDxWmPLNq}k zarUV>2X1dVI=mqhCO~>1g1u~Fvb`=^da42GYXcpkdlTV)0(n!`PL7v?I4@LEr3J$C zzQWN=R!uU z2ljzRs%X>@i(m2+ZS%rZ$<+=t{<{4Tth@%ZgAt()zpwn07Pq6+c7dB;dFG5K-fni4 z6Qyg+#r5mTl`e)(_v``*2&X5L+9z1HiP%{H*y&6=O|bMEl$)Xw$@35GFIZ2X|eHwyeK%K56Ykqhx7cPj3L#d<&W&rmd8#8S10vGSWMKec2w zR@C={VCy6I)!irCB;3Jt24Vybd;$GWq-%(EY+LlR7rZ~u)1RLK8xcS~1_Nt@@mY*2 zMq&m6f)9kE)}dU#@(6<0Q2N5)qD}uJ=M|=&ybKG3axe)!`MA{VvjL*;&)1Y z-;Z`Ffv|H%QI#}-BORaP)Ey_jH9dPKtJyU? zeBVg3hT5%gUB02h<@6*;q4DwKP(C58Pin;{S5Xg)$#4!@Kqt;SL9VDqJ}Ndy!%E4Y z!;`BHf=Tv{%fZbU#6%~a+=)Xq<8W90!qmgJyR_zi_|SZHi|pBeO*5yz3VS(UIzt@4 z(Q{2#?MYyNMXb^OmN<{p7>Be6y>%zuoM^o$7<<$1f2R~ei{Y@)9v6tOeBg6LP(D7o zdXQ90hl$JRFdYi?4#Tvl*5*i1N!yfx5?wxHrIXRYH)s>&r2j81hOVJq{Gs2Bi8H?& z46jxmiKB#d^yt9zM)*vT)}Wbq7C5E9Q#L_sj9j+(7kKhgOHx413nngfq+s2&dTikA zN|3!`Uvg;yZLrB*JfT)DT;NW+S{*~XEw{t1)!Cvq0gQfSVEDW6CYRxzWsw-Pf3u_~ zz;;QwX9_g?01b6&4<^=$N#!(A`;EP1hKsgQWQ%`wf<_Kc$oL19)N=HQFG z$qB-2pV!$wk$dXi28C-k$MTUHG703ZEdBV;NQ)ht zB26yUqK#VKlolWFhO(nzT)#5wu*#(XVS!WlU4u>hHRq&FIs)l~N1j(iz25BO9k_7c z*U5Vg;ty?BPjK8gRZb|2u@qb2&Pk$Wi@sBS{oSKZPXPHcqmwb%vggOrPn_pHDmZ+; zIneR?ucK1Wkuy=2unx&=X?0Ko@9*ZBuIKyEmEWJP3AWfZPu@j;$KCAVBM##|yQ#+? z*P7AAKfyUWSduEJ?&>HDyo52T0+hENTsh1*4xwuWzMMl(^3OIN;0YA+V zn9bvB5YZkr9c4+O+`B&3!rmu*x&krfJ=5Zu-Gmy~el530;c1tM67F z!4$F$z=-WO7Hiu`gGLzv>F9{-lr)*eKw2cjIq$v2VZ|Vw>ZyjGG%Jc+HE!;oyx{IN z-3ZU#)pr=(PcNc1hCfUytIATO`$Q80HQ+%k1M$DlZjdw7E7v(ht)2{y|+FSR6Z4EI~Nqc%T87w$oMo0}K-z`VWoCte9x!cVXGF#dpLe~s2YgIy>^{XLarc5==?(IFY zA7(pFkRl7-vT8*)^u29holm))kGz6@uU1)-xj4-rv&+L~T zb;RNuK*WVxCMSjMNi|q@y!ypG{a9M-F&*h?r$3GYn*X;L_#8PC#KI4N3yrl;&`C=g z-){_&F0e&vIzY^x1DBfQe2~t9PRapy%LIf&w>z~*dqA=fWT@P|1SB! zb?gI**zhkAVai{vEsv*vP4oF5js^HncA`}6=pa?Li!+LskPQrh>yaQX6o-9jxxmp1 z%Q%8MKcTz7<1ctq=dd_H*Qi5bu_d{WCTIbe*dbPPpNw)rO2nagY94wrrWEu<2eHVP zQ~(*Ud=fOK1MX^Z`o|SokKBs?XTN%mO{fKv@*vw~48#{P?rnxb)O;{Kmhq^{xSwOu zleFPA&+`3S;(XrjT=yzs+$*_y_Gk>m8C6Sz|DAEnn<|SwN`O88c50V6A&3Zrra0(`n-+`l=T*6t6yiEC^5;k$oFQ^RStfS# z2|e5@YfD&cd0~6XsZ9Yx{yQ_nWy)83XdGDm-cZ2o|9}V_puI}n6k2qqF@$yu`2)XYYrvlzrmZz;(nqXQ6Ct}gNy`BiBP>|+xwKK=kw|EgZl;k2FG12 zl1p!edeq!v4oH2cBnaV!W<;Q+A_0SxGsM}NAmS&YFQ$+-Xrlrmo*R!<7DEOL_|+6( z(K6G-pi+Z5jfy;{?gWuuq)QzP>T?n1ANwW4Fk#?AT*lDQ&c{g`oqcuht5?3J+XoS1 zI-wRsDh!mh;qp9;7#GonazE2XR-q;DpeaHk_&rl1Kt4J^jw|fj_(s^7Z6pV|Pwjax zcsZ0SzD;Tc70cnF+^vCn>wVby*1O~1l)kY^$OuqX$kI1pk7_~BMMGku2KxH20MKkf zE7bDV!(H0Jv@wVR!?E#B2+@F5WI%}zom2q0F0(`&gYM~TTo1$MvP5%;wY9{4;iM`r zlcwFVR{}y6E!)Jc$28)nRGG9&AEkA`d#h-}4WDHqEe`lYeexoMFh0HDL0u(%KHkfhP3b} zf^Q|LPUQSsU0$4HgWkqE&SvV@GsAN)=_hO1Je*H~6&oC848DP!9|s~Ze~!VRh16h2 zH&C_VGpGixyaXy7Qp`VomZX@2PNCkDBQl3e>7qX(9bxELq~i_kkyTveGCt!2!gG(c zF$|bBbn7-Av_3EI(eb_EyU#W)Pu2)EDVL(P>KjS~ibDUJil}jT&bRu_k_|#2q|vG+ znufB#naaK6BP+fWr5pCL0+S}(w_pDfrT=d4-QHafL!7rKwM=dG{ovsMR6Fb)^GJ9{ zNi9oXaO>t}Rk<;9#pbW(%0w_%5HMG;Pklqzz+n6NufY#67ublX3N&g?B&FdHQ<(@6 z(2fs&c-v4}w)NfJd9x43r5N4y4gJM7=B5E#G|6zxC{?<7JIWS!k4D-CjR{)l!i614 z5|5iNHH9JAQiB^*9gmX7G5b2ZCAb%_U$g=%h%tD!^sS5O)2nmo>tF-;*zH1>TU%VG zzR0`lpvN+xzkLXcUy8fl+zg6XqsTHR@PNs}hK$_P@0_a?QmdAj&$o@1o{crOmywDr zpss}Y$_WfUJ5JgpaDpgs-@Zj%G`@noq;5c*YG;bF4gp@u;FK~a`8Uz3f)!qegkJkx zrW6zrm+3M+#6!A1jlvW@uiw10ICJ^pjj7dkmou_fy6BIumD?^D8fC8%wkO`=IM0mv zOg3%Sd#rG4Zkqc&X-l%ASROdB3q+lMUZ2bt?IV3vbOk4i$s2{$HwL9PZmL&1!s20- zC&tP|hG;@2p#|+?^={%;o)8JP30voq{orx3hR+p~*E$r+;B2hQuyS}Kn>2sl*Zt8& zX}DmbKGShyYEo*qL-k#+lNpq+k*XoVzI@Y}q_BJ+>u|9C6YkucBp1ZtHF_U|T-m3H zz_|xuHMu2sjICth#?G(on+gvqFUR((YLeyyYve?Wx)_439NfU;&57_}xhkfDdC`Ko zM%Ladd8cm;`rm*#q8HhkOoEOV;XDdBVL2lwobsjUeA1pIJWv2g!} zXYrnkJW^fkqNnAP5Kn@7u|h=P;-=L?i$8Q^J-;Asdtu(&oGTd%pV>d3X`l4MF(PJy zC26%SwU!5>=0nF?*gA+E&Ybu*~BjGEq5$) zKi@fQv%qWyKP`0Ewj&d!Cw=?6&ZWzKVksrWw=oGK3_y5-Eeza%9bY^iH!dset8!QL z_G$F@3VBw(z~^#UTT+w7q3xLp0Q2#$t-w`1vS6I~nBfN2x0d6>Evr(Gk{OpIHKgX~ zNpRAEdvI<-Id;Ow2v6kFV(bj-2euyed{%!*RBbwy_c*`sQd|6n=?9oP#?l|P3FicH zpm+sZWYt{*R(u;*`a5M^J0Rw3sSt!2UID}j)_qLTwWQYqfJ!z06eo=ORyjoD>&CBBrfyS0nA2|`~TBO25KGsylb3aMw z3y=z}QJ*`!t*d+Awi{ddx5ay(+Zm!CGu3$}#x9lMe8}$F2U_GCdE5{>Qv#8e(*J$| zYf|}t^qU-fADhtZ9Oqu*4qEO9cFjp3Cdsu_C6gdZS$4&$k@NA?8?UbgHEG2idTA{( zj{9ITsbX)EuyAPC_88lsz8?|8-?KdbbC~`=+5}w_9}zjisM&LN(_DqVL&t=+^kYR< z z7UnX0#G&nn#n-DU&>3cgcL*f<^3NUzBLGCXs0)1vr}ZI5-0 zTG^jUIx>LtYjBGkAbyz?0gE~y1z8rESc5aPXPt)kh#Vp(i+?3`KSgfE#XMItg|NUE z&Q}?0>kaxZC%11)y%oIC_h=tY4lNUapvDGhHu})tm2??l74{e-LR>9KOPJ&kjkBsy6Q;q zaNH|`XuT73rMW#k%EsTr%NFsWgF@fYGPWlJ$~8KeLmnDZk_nqKN1?L6?<6Kz1%D|P zPyXVm-ElkwQ~Bt=Ani&_EZWTy((x&v(85#-B9^nF(Nq#Mr_^ z0Fz!vWl`=8RJM3d(oByU7mI$ zdQos9I>|;xaImD1QC^dzom-rE*}wL}iRP{wAC%OkV|qU&SdP5@Hw1bGp4xI7`T&OV zf8Bm6(h`vqL!G{_%nxqtdJZo~fv>>VSa#6Ul-`)mV|2HOtclGMrYs8KqsoEWc5zp~ zuxENd6^U+oEi&b#W(i(FS@1m$4p#bBn3Bh#>d-N5uU%b z(JWTYrJ$$4T!2oR@<-s~@!~Vc9u8Rk=Xx{+7s~C_1cos-uGwAMLj^aRr{tQLnC|w> z|W= zR+V0UWgH6ev!FOPx64XCV3%KTnD*xOQ!V-qDVyozFqa3)=<##VDlq_d;!LC(1%1_W z1MO^~^)$+=HbmP2t;A%ZFBTPpoM`+b>)(>(as(TKN1U{(5Rv*ICe}ZTm9>x6-P>h~ zlZw{2^huL!s&bJpulaGe#WVAAuOqMDzpDCq{|eo&Z@LdtWZlqv2B-j-0ylV2c{|E3 z%vnv2C?1n4N1XNiUyK5#TKjKaJnNrA7ftG2FNrPhI;;T48EA&Xtoeza}Z2efYG~=F)GYEe>tf@(@mD23_{wXZgBq;Cb+PqXlnW^<|c?{rFH# zChGEUJVE=DCt7MgoJtir5lEF-DZ`UIQ$g#KJ(@J9X1hI=cPR&DLA&ic)2MIf<@W?} z-IIE9y03RlOdV0?;1GU5IP1tAap_598Uya_u@00~3 zSijdEc;fb5a^&oDzr`_RcR|{+R^ID{&|$1Tci_m}*_mH^H^fh=-IeJwW$w*s*G|bq zT{R@s<`rr`kwgR^3NqaHLBR?a1k>V%Y(3EYE=uhu>S|m*|D|Hy-rV<_S5L6xvPB%iVSJdPcLyY+BcpTf z5ZZP#7&tS(Ml@k7+Szbf#p~_ccJiyWvBd&c^YJTua$fDD3>&b&nlcG>eNv(7@E*Ok z?2_QgE*jN$Pd@Tf%2_a>hVK64XhufVIG% z0Ma1_L}8sYj2MxkLI_@0!pGn>xGD!jL&hO|T8~p~mrFc|I-xg;&y*uHuUZ7`1Q$x} z?FB9mZRPm1a}k*rj${RTYCGPVpP1ZQuq4HR~y^E_fVQsl9l|qh!4)zsxO( z^~sOzvY&EUPLfRH+3uMHMNH82iY6151{ivpG0t!CR(4;N>FRwaAb4ES`sEc{r})O? zOt2jFcXWiRSS;8rYY6&HRuQiYG4jlFC{loM+CT6J{cntKNg6-#;hoIR9w9pafGEKL zYJNuw{}rhD;(8j09k~R6Lu(9hn2rGs+TQ?&`k#OUM9-Fgea`jaDXU&E8T*=11-%98ptXswick}9Jv5YEmHdCvfG`76@^Twa54FHoh0Rm9q{Vg!m<{;v-7-~as|bEN-A+roF8BUM6iDIs&{ zJW!ZN7pfRV!F|^VLyzAO%F_2wW85g+{n-D?#q+w;TjO-oztGz6K}{bb@Hh(1ofRul zd|oDrv;2|yzY09N1bTj zuuzpmCrgXDw@pwq4KdU{OyXd4}{{J554OFxZzA>2y>E@^R?UCe`=Z7M#u1nT`|_y{riIV~90e0L{^j`k>6HCxXW+?)o?K`sS! zL3kv{mY7{pz?={J%0!$bOs7~q>_RE2^Dap-*LBTZ2*%V^v5> zy!0e$J7!+wdI+p%(8zPkz;g$ZbOUZJaOG9?$O~m_THINxs-N+!@`&RHgR%pJQ}51F zzZKdPn-2AgO~rs*|6HY+WR)PAS$Cfx0xu-)$#+Vo7n3~olGasA0Vw(ko;4mm{t0^N zBXF$T9IAZr9Ksj3<z+mR!L91vB{?jOr_jbC+V*wi9Eq0{p5)O%6=0Fzs)(J^ z)rR4V*Z(brYr$$c3n{V@5sbhnEt*(MLUfe~Xc#I;gYkDj!-%*w#FQ%1k~BOYxS$@s zi~je&dj}P4h2t|LCROo6q4^BosuVlMm_nHGsOnwXHv2vVG*~@1E$=^aV|KH)$|mk4 zaJ+6vJ}G>g{JP78r*`}aKiRw`yUgq) zsmZn!@7^BlyFCEaP5-#^(?kooS;0@dN&;GnV%&Rb(qD#7oY{UGxE|Ir`3V_B<9&E?Us5P~362SapLihLi;T&T>4#2<(jhGHx|XfMKJN$QArg~U z>@quZ-HW9&W7Ad}6Td3YbI+YcInn`d^F&w^OHPGh0L2Q#QvB#m);0eOs}loHpLR88 ze^4$`-(qZ&Hox<>tP67xL{h?Qlr=XIq6V>uV3#^mXf3Ckv{kY_UEhG|AzBra9>qfQ zI=Zcdw4rdw>oW}tsvxa|jQ9{`!)0|AFY7ah$kDHpo%i(ce0*G{wAlC2+7AOjkw}vq zv~HY`K`LH@VrwvMVr&n>Jza5W#^_%H43H5{Yt$#@FTiIMG5ozpH6EE3WlhW!TE*Gu zK;H6)z=${ZZ+@^@cy{40{jz7nTWt*dvrOC_XkQ==4@weNUOixEhnwqRyj^-B9Kl9_ z?UOwD;nV>mD3UXP+Kh?K{?g)jWG{lxsREm4C{~;jWfZB1&k~|#(~N`(Cy4pCzG%-& z&E`0nei|J7^sqvvNImvkn$x!1S!r(QuODFTW0Dc_6vu?lOH#7~qN@kc7CuKB>HZ_> z$_ApQNx;EBzKwJq4Fgx9%v&_hiqx%ue{BNp`60DPO|o&ruU;T#CL)ti)(jwy8>Et; zXZS?qW`T7@Q`GqzrNdgD<^laC`zoe;NMEc8xKHN293pZLP8$YGI;Bm-R1_TYYjGvK_(&<=A3rSjUSY8%)B@z6&L)LZPI z>TM3%@`@_(ppQ=X0lH=#Vt|cyGse~r90=dAaAB&nJpdE`^6y~({eg2WzLoZ&uRbIOA?BPUi7%x#n_o4MY z4?XC5Us;cEli}Tr?|itMOO6YB&OhrEvOU`+=2|PY+_RkaFf|zWN5+8#%DMsjzafMa zXd^_-9<(b7?VK5nR6yL2H3cjuY0_WoBzU2{4e@no8I*C|p$^y=`N2v*Af;Hd9Az*| z5%xj${^PL z40RHiU?sk1W(NfcR8)&kMrXigy3n{aw!*n>AGu%aD6wNzN}`T=?jj@W%bayLryniyt2fS*(<+ujX`L^X1-PFL}dU zN=VfB$B!NeZ_tl~Pfs!m&i(YMaBj}+v1rD_b1@kA0YGnKGubbbLy>7kdb+8by|eBF z&X38te{=e`dcYWyb@|ulwSxj4sz@Le&0smXnI17zcaAaWW_Q7pL`z^zIx~TsyX8R( ze6ctr)tSK{Vj&eS9Sp_U7%Lt2@hqx^HUemYNZ?NB+7A{=s}Ahs7RFc_ml8tF8DNemc%$qM5b=#ei#k~D zBlQB~PbOY8rcF#Z+`f+B_mrL=j#OSCzwhZd zm3wF01m;_L`}vX)Ty|0KjsXC2iF5$gP4@uSeF75iG}iJB5v;^U&Bk=!NtEwI*!hw2 zu+|k49WCHKw-t6YBb%w5YtgF-fXP;~{ zNG{p6%a%1>y@$qikhZ#b5xmAfa-KSf3&GK#GwgrQdHUF@1pBT>Z-$;6%0Splg!G<+ zLbHmf$+!-5kg!@OT$p_@=%5Ni-^pUZ$Li|NGO(|vk9vwU)@=ji#`p)bfL2u54*Qbh zOFECS)K)_Xp3+(ZpwQhNBoQj`*#9zeoG=sV)`L$vkVIlW6E`o{D%KEsTzjN__L6z? zC+}5HcP4Fm!**I<))LE!xgeTHkVX78fFD z=LijqM~$=S(+kt#(oa~r>39~T^uFl-%=$csdwyBf%%8Y-=2Y9Vm~U90p;a;|zfX>^ zPT+3-PPs*04u`SmrZjO9X?aEaQ&#@WXj%CWc?#yjPkE1}!L6JmM=VYt74B*wg3CXI zg|Qe!;x;BeC!t$Lw3w`<0p=qP;ZIr^3Sloxee@o@v%ExI_sn@-cd#i>(PWb2$2(a{ z3o_4$>YZNGctD0S+n{mZ%3{dqNl=D2l+4%5h4$M2N)HlG3BXa-T;PCW9E7^;d`Uaf z#CETf`4se#7GW1e%6nXtupWHtG|H!4{wyu*h$Rhi#0`%Xh}IP}Sh($&(3`{Dv&qJs zHMiGAe>-dAx(Rk?NI6Po_-}obI!Ha}B#I2o`mLrVz3%@;P0N3>`~NseQcrv6ShLpC ziy$eYtk8-)?KxsO?FQE1`R^`ABiER?MF+I{K$I$_86SNfQsh-)t z0osy}YQm4UU~T3!fVN%nAjGbvd5%K!HgXw95OhugedE!=s&()S7h`c4^iC&_VBAiy zzB>NeI>{+_%;Zr82mKZtZnD(^6VHpss+~dLe(b}oRCxtu?lClhr)mOKz1S?M!A`sO z=d1N`l~-=QIy)phZ_>6wRsU4hK{Z+T*;xH!0OlC?f-e~al3lPL%$_P^!ogL^SEJUx z2T=D2m?7(di}!%@>VN=t&J@MlsDeQRpUn|#naMyacD7`0FMc#lb+sg{zZ)9kAoJ8(;Ir<^9<54z3z1-UYQPf(q9mVy*+SWI(g~WWG9AGdv>DAKMXnE$h z-5~uKkg+dT!*rk_Zx>{pGao@lRS{a&S*!>R-=OsZwR-}l5*aS^tDKCeMokUSiWquw>q*Mn(#qKxJU_~PdF^qpmd1qX$s7T# zp>hj$lk4Fx`8_&v`T1+`-P0v~JEx29p}XC2qwG?I zPRm2e+|_Y2mXDTjyjD~3lbnEWUYi=5r|)^U#eMFvWUZ(3E6{8vVe3xH7l8bK$)e$l z@+E~Le41d(M;Nu0P<17$W_c4}6SStVz8MLs;ZoJeGGsb$l4r z{qg(!Qg1B?C?i0}Q~5Mv(kEyiRg1EwBW-XA zRPP^6iXKty@}x)`D4l?o2t)703?!I&jH*+-h%gme9V4K1y=Fk++50(PGY$GB1%5%dEo5MESj?p2DMBQPaBX=iMpnK(^l9a1UjNWn|D-BN#5o(T zE4Y0*oz|8Lhj|-zFa7is{EznuCgrWF2#o@6_<%6}SG0ctTlCApOn4g!ze5#$*ANe4 zH9toh;VdEAf+R&+w$PODJ1gt+_js>;H_x=kB&X*tSEB`)A=M7OnSbL7Ay)VjfF(+;aO8BggNoe zj#-+nUk?U?sp+71!0e0wmHHG5&spaaOQds;6i)R{g`vI@o3e_q3FZD0P8D46$0V&w z3nU>~C&$6HFoLX=)7(d{I>##zCp9r7;5{BRRTl@qJGAgagRzkXv?R`uG1y{IVKL?1 zL374Ifc0Pd8qG*|zZNTTa-kD>ri#{kR8m5s+D2(M0Q>oH6K>!Zd9EQ$7#RtXbb(uX>!FNh?c?Qtx^T&6@+^f%% zSC0jFG=pOpf>kxRk!Ni2eB8aFcrI?nS4LL79Xz4)^y#gE3#XGN%(yho#o)lN8&=5t znn{_;9K8Mx<=)~5+&V2fr-NA%bB3SnakRnozyyj~RMcg5JuPpyQ}+n}F4{g;3F00o zDFxXkb^9ZqJdBJ@1Sr@p#W_bgU}m@Sjy7)BH#^WktqFLer00aUleQ5;g!tK|15ZKn0AKwD_S4 zzDi22#A-EQ2K*+$helX~;0r%9#SZJ}hwujFF_85p=&QWPwDoBrPUSoE5?!A@q8}P} z`()bcHx%6rd?D}iVb?^{d8Z_W0Po`!9$Zv&x@k$`h}r`inOp#|uk1QNC|aViPUU7J zR7nhH3#_W)vJxF!Ak{2is9jKp3otv^nu`*tG&xYklK@4%_3t8dBdxzv^kwMcNGIro z1#%^Bo!kY)-*NM%f!s1(mmvHs< zY(h*m$Hxb9CaiXlgZA!T>gkTE%n{%;V)@Sqbng(Xvp(FpcQ&rW z_khN&iFLjAD2okU4wp|^*(o_^Nl1; z7yS^kGrP7q$emgdmG<>Q{P_b89WPMHbm9d>5GB%A=r4UE9|ZuS3hLU6Bk(Z8XE)DT zD}2e?tWWKq0!jm>#-Ax(0<;7Gk$ai6#GColDYz@Q>ZGNf@kU7tCgZ0^X16>06=b%% z%sgu)AMfO-bZwu*>!{5u@(S;$@UYLSL%U~T;1WydK)#uNk&F_6Byx!c1jC56lJfjd z!6}Hpbt7=eyzBe3z6ze@f4W9js&iH-I^ekS!9AOpYidruw7!={cHY~DkdEtUY)6m9 z2XR(Wl&y^mh~O_<@}zkCgmdqAPN?ai^~?}0g1f`9aW6#}mZ-}3H(`}Evdvq)I?v8` z)5v)CplI)@>ASXVQFGwNe#R>9EnIZ1dH}3W0@ev@V*!PBJ_$1hPMnMp2k3YN13c{H ziN63Hd&p)S693P>E<*9SbYjx}+RJzwHz$=oq9kG35E8^pejhO@YeZss-zSInH_>9kJa-lFMje*KD7x93Ay@5QK+C&+!P z(oD2{BXmT)1i9Y9ATr+51nc(6BOu*iw?Vk+H(`zENNBN6SI71LKib|r9_sb|A0OEz z`;v8pQX!`zWt*fzk~V~xHboOk3uW>Wp@fNwQjC(yIxUtG#;(Y*hR8NrteKJJm6>^+ z?=?E-bl&Il`F;L+|Nc0S6K1(z_kG>h^}M#{HN%e@^-nC`*ZAmo5iu+;i1ga-=IRE4 z=B}+Y)hi#fbQg%2FS?iksWYe-Q7B!)vgXLWz(jxaQ>FU?3Dtc{86&r;yz%Z)%ONfSpWKs9AD1EMg< zC~pW}i0aguoUMt=_V$o&8K>!e8}xb<>NTF`9J~31yr)K}q!lrBy+z*P-VlZVKL(ur zH>V{?@H2F(9U_22j{F;Di00&W2Kw2=wCI$y~D5f*Ievq#xfgb!oOJ zAozn!aEY?%=y)&y)gFl9-39f)R)WONawBY~93KWSf$8sDL%U4K|H_e>t-?cy(G!s! zMx=|~&t?q=NDY;$6kXdxk6*6Sqnnfe!YqB)`9pFcCJcH53VUu&@O}548B{|+mzMB4 zkJzA2g>F*NkA4x7+>K|cTpc6qxfM9fiIN7m8^MiGk3 z(Jm}XxO3WPfnkF0lSj7Fdlj^CqHm3`v?+*HHp0*lQD!RxUkw$^2GPZ`%)-s3EQx39 zXh2AwknFd#p)xz*%I{9=!&Zw4+14(g*I%XgryzyC$R|o*El5KlopYT2u`UW^XeH?1 zb@ekFxLKGyEZ^NiEwGj0T#3kdDVt}TP?Ce5cqcl|u)gJAU({l&{N!yvV7SLouFLs< zfJqAju=05!Rqh|7%tWE)wuH_+LjdO7|CoNJ)gEH%fa7dneXVwob3?TT9>oOFMiqmK zujyL|zkGSaORxJ-j7uy|yZ@x_Fdhw{j(_OxpN$5%5CrkQtZ7|r(P z4_aHlAytyu0J`fu){~4${{3i_`FBH6(XV5`zMR=9{9`FnE{uQD%2kPtqFA6lb_{VC} zF>e)n!xovBdt>;HT9nBWKLNg%nBu-hhW;s+^P^eCXbQnqNC7^i*f?NNQiZ&Vwc|#* zH|vhC->Rvsmj}Xyo#L&dS6&*iCzp;E(7T+mg(;^%vSZO%D7((-z%J)w`ogKH0+6!j zpMpLiJ|M}qM&T|Wu7T4}UqKu%Br<>jAV zNCAUdb_d5yYeudIQq7U&ACb1>^$=m`NJh%{(LXCwRfV~>b5!igaZv9vLQN1jy$6X{ ze1XH*9hUQq68aj(ov>Dr!`9vEC zTPMM2M_vf-c}zw3^seTdacc(NP7)0k?#4?$T4O>;C+h(+eB>(~hfsd79{tq8Z6nw0o z-#p9Sjl3-g{FTko{?j2G;_sp#CH?>u775|6HQvNIXL+PpkN3g4Jl6c94nH;I%BI7h%wsLd2 z@8h;ywO~oKR>)Zk(=IGA#?N^Yzga_*V#Klet33YeD)0Syhx_KVdRN1njV7S=i#EF! z=zuEMVrrl^kY;rj1rzl{(D!)e1fjd|r&Rt_qCcf_0Scik0WSB|32e!bu`xk}eBi1= z+VO7r0Zk6%hAdaW?FJD%`TzAdqWV&{2#Rr#t2SPP>TB@gZxNO160ZpO5 zhgk%PA~3p+xg;RI=CizW{JTR3z_~KQw1Z7Rel7-o?65Fy_tnRw(DK2c(F7-DqhSFh zH-X^M`?OinGTxfQtbV?5rt5ujb!~;@Vr_536Ysi=`f)-6@#I_>$5n}l68s3A)J>@4 zK(eGOnm?doS8u+oPP9L9-wPMbg!R(yYTHA2IL|C}HSru8hHW(No1QS>5xc5dz`4C8 zu_6dV#!ryp?I6D&5hjz;x%cT7w6dS!_Q)tp4-j(K++_Pm*Ic`4_XdT1M|TblS?TJE zM+ls>RE0k24UceVRmgwf8uvol;5#Bf{}df)JwHFe9_A-VJW8MHzU*NeQ#N;<#IXtH zC0)Vd`+|dY&0|IB3v>(P$CA7IN&|ZgNi|MR$!|2ST+s+Cm&bY_UW0mp#u1q27B#APBjdIIa}rnSnJ~n6ym|?+yy0NFDdD);po3rarROY4;itjZl$MkF`Hf ze7VwkT8<$F0$Cj_p<6z=UPMuj+xcVXl^X<@L$egsxR-TBzg>ygEn%su{6p`oJT^ps z>fJcDi+LBT*4F~JQVVn<_-QR@3-Jx3iz<#xEbH`rQbLPMugjaJ zMuEEBHlH?GX=Ri<2^!kUxb8D@W`@tvY=tFrU%U4woj>&Hlz8)05`jEO=wfE_antG)=HAT+vLyfl4LP0nFRcBjp( zVLo~lFhG+fJ2D?ppA3IDDrv~Cp?Rx5t&1dSrEODkaS)Kc9!Sfi>>>_8P*~80A5*tv zwZa^n)4;U96#S@7@NAcZE0Tq4@WM#{$h^`S4dh|)i-I-3&l*Lpl>~0E>_J+e$owA z4}Gdv%D5;zDNmQQ+^A@pMG*GxoStwEcYF1j$i1OL$Jh4W<@IBg3 zG8d}j8Ud)MrLV1vRt@K8X8fXLzrmtW$g<4t$`^rjh63%_{b8qWa(DfY_+EjJ8{J!M z9Jkt*y`}Nv3k}8&AYZajJD%f?aL&<}!T^bED8s$W@ZLk|9fpXeJQBy3;A?Rj^5b(L zXXTZ2Rf|QzEs#IJU+EJ;jy1{;TNtxE9?qT0Yo5}5<2UhGx_9%$=kD&;Z`eB(POK;s z==c$V?>zRehv&j=3iEIQuQIfmtmmvvF>r(!b4T=htQgx%?7=6(C4*74G;C2Xx`dTD zKXHJ@F-7U@O@Yo%OhUL}Xa16aY*N8#@0s6}PJG@Bob~cr?!Rx!-FM`=F4wenyq>)n zfJJpDlR}&mhuvg#agunR#NmA)pV#PeA8^9&T7K|jRG5UJ>YZNp<1Ubv&DzfI$p!L0 z=$nzy1I=Z?F*X=G2q7>cG_kCI4z5-{rr>dc-bbJUVK!wQae~?iP#MaC!r;Uv(OIU| z4A|+=T^wiBlBrTXM~LP)XLY&i$5I}6R9?L3|2AOZ(}-T7^Qz&3JRlBhTD6BQ9^3^c zBz|5v=Fu^C4Xl-P_WsV^p>S2aG@{ zouk1DpaiR-lr1eet0t6#>U-9F5z!K=JFxJr_CaUuOK(Ssqo%ul?E1Q7BWFebfkS`0 z-4ds(gMVW_zGOx?%V<;{fq__wNA7=HHL-NxfvdJS#W>)-)bPL1(P%aEg&Qr&d5?%f zCJLT8wxb-+Jo)G|eE9~>sg)r6$OolnnJXVxL1XNFFdmK&Yam>L^OrJQR+en75~c>B@9Fmd?e?zc?_84 z=07oz-Q)wTpF^K(JHHD#8BF9JhlQ$8(4~@Nr`ay39VX_&G_ESuL60#m5_UXerQ9&8 zZm78xAh&C4qL@oh?{Ar@{pXBr6RT#g z4|ko7wl`{MsL$S6tt|fH^|N_2hojvi47@@POU|vte(jzRY)%;qEc{Cp#ZpAx^Wbh4 zXB+1OQj8F=5*+dzn%!k*YPs>8UGS=Lv=JNw@^)qkm;3ed^zzig!uLMf2`;$FYMj(X^bQUS;^EFpfM+@=ET&)2Gi_DKYSHl~(|X zzXl}!jL*?p`$mW^_nzc<1A8%l80&C_6*f4pSSqUB{C4DhH+=Z@6r?z~9aF?^-5DF! zGMdi-gt$2N%#!Kpv1^M;ik0&p#@B1zTdAq{@P%v{WO7;#hMoCBuY_r%r?q3?im9Ti z3xROX=K+|Q$54kw*N@O|%YURHU zcia`fFD1mE#M!qD7?pka*Tyla)$~A!YpH+lz+O4CA3l#5wV%BI^K|(zo?(|9b7K>| zkOD$Q&#`@|J0r`@4iaf=8VrlQ}C9eC*lf9q5nmyjv2N~J%EVf;6H{lsjjhdl@J6f?7=x*+quBw8AagUwkux=N}`aAr)bzd0J?j+8fdY#wG8^N>S zxZ>nPDNBYLlu%s{-@64|s?}z&HgjNUHX|15j2+Sfp;<_*Fc;?30Eh)F7yjVYu1j`b z>bc*4zyt_5%~l7Vww3&m`AaAt!rb(+1OL#YNI3Pb!EEDKQ1WRFiR`buwE$dvgzvsP zW1tqS7YGnlSb@knmx!q(&H%i9K!ZL_RTqTS1z&IQm!7r$>q9_S+Y@;9ivafPulHUL z{@C~I(lE%~FI^q{zv$|3X_Dh~w(`gA2}}TCci%1%%PgwC1G0;gnY5)n9!Su=k`TDq zWW9Ly1_5*dK9s$9Z^nVb1+Rg!`mS7>@8}^& z|JYiDmU<-43~*yKZr#7jD7+lkbnC#`Yrk2lN+CgAC4CIO+K}lNf8cf~{_K(oUSe;+ z(J5klGdr#b?&EM4WLhbe>{G!tH|m~RvfL_szirCQeRdQM7We%^(pM#J39>h-P&k+M!Jln)q|M$$0SasWWBA2k;?^6RN+2}irHzw#c*_XW z#4FJc8FXv12Zf_1PGIur^LUF6eBglz_F+=p@`IozqO{fVsvgy zz!In3n&kqjp}O2xcLMg1pU{g41{f6Hgg@r|Ry%$lVsO;zK2)a!>@&EoU{3KftfM?6}(s&aom|LHGCoADqEGasyRRFyuN`xSwDS? zq3jA{Rmy`dVp@0Ypkx(3 zk}7Ftojc^(acw%Nhp_W$>#AIR-<(H9af3-phhySxdTczQxy@g9a-+j<0)Bq)I@rNT z`Km)P^!|Hzex(o{Oe#j|(4 zZrLm^VBBNy*Iw`X>)7}?`e2I@AKV7?p8(pk5ncKvJO?FWqJ7L`mA%?8?mEz4FjJD{ ztMe-i{i`)AQq$6Kj+aJ68ZLGnjHVW?o_d@q*ndV2$6=KEvV_ND*vkSP+={^Q#g?Ln zceI^2;^nKKd)5uIfm_xeIpfW*%MVP@&Mk{t=v9SK4WWGvERZ#d-luS4=jg~>AkX?A zqNBckos~3>KILzm)6tCKSqfo{28Hgbl*J{HyILm>yX1WHN%oA|_tDc`&34|O0(-jP zpkXTh4&CGr+zC3W)$YK7^XRuV=O{kJb8f|k5N!XN=Eox=*8QjZ+OrK88L1s6m2CZDx{v== zMMk{Smbd0Y{-j!D;5q#WVQyuL6o_08fMj(V)LNQmQJ@3A*m^N*T{y%Lfc{!I=mL+l z<{SY%LHi(!MLoVuUtDVosV*K!AY$RR_{2w z!7*8Hio2tI7nFiD|<{mf_oVY$0fXBGf|kubLC5jP3(YK){a_Yot4SF$ET$kq?M&#^{= zeLiD}0X_mj+35?5Tn_J5TF*%N&cX@&`!;%g>haiv`Twd-S3_JIyjccXX0Qjq`%r;;2|kc9_zpf z1}AD*d>X5~$s+{kYYco`9jwIM8V;!72OK)c2>|aDs?wZU zkpq)fe zp>akLZhQPsVHxt{sI~{BIQVgp&J6`_DCME>=?-*<=UlY*M&Jz9mEFc(edJ+ zHpWDf83aiPD~jbXO{-G$=!Gp)qXbzuTZq}OtQ5DAf8MBdJySTt2&=?1KMuX8OoH$@y zQev;t-3CzaIr8tnT65MpkAWW8vZn(2_MU>Yy~kvOy4#xJJvijWF^;^!OdK~Qcp2!6 z4W&jLbJTCJQ34I~XeZ+{UA^aglQUg=+QgXEPI-HpKTIjJNmPzffh9tW+b~6VlNT`~UYo@<`s1N4@?1fV*|~y(H!Tf@*)r>!x4PW99Rh)W zaKpY*{;fZ=7S6DM?rJ#-sNqloZN4fNJ+S@TkrU5Gg4jHrOKR-)nGGBQ)?c4$G5fq9e`R; zou1w>-FD7tS6=J(av8`++==DY9NG5swUZGUnNNQD=mA6lt`79tE_G!0-B|wsHQ_U~7rrYC%=0DDtMPfYRX! z0T=rv1cxZ-VTZ0+h;jZpP|(tUAE+|2h;W$J^`(!o?vfe!9pwF2l5SI3`K$atN)lxwHva|E&$ElBZ3a=P^^YCuj+`H8)0)kY+ z9e3JlS9RhfY>hDWU~_IPLb}R1o#P-`(Wq!*&aKy7B!h;EXIlzvIt@(G_MOQg0Ilnp z!VpW?Z#7sV<6WwNzLNJFzpDSB4~M&DYgXAs|G2g+&Q>_o(Cg^#+=O)_b(I3rGNDj? za^{Pn`QG5YbGXArHQ}8h!rCbz&c7$(3lzr^y-&^*66MHC<({p!wr_u7h`D4J=cV-m!n+y4G>1(;0nSh;^Q)h7u>Y z@+NE^aDy>3u_5LrAjKjdX6{K>lUV_5BKEfHVp{T^0#uv^G-6x`8h`fizyFTsTqwQVYGh7uc^0|1a2N6ye#k-@8)gKNNwA`I}X3&hfv{?bZ|X|snqOG9I>LCTP$Lm!ACkoq_L^Gy&6z@y88 zDcOM#>KSG3*~AjeE--EnUf6s;rFQ`60pj$v#ABoRbyqCb2(cU``u*_cHt#4?TaMFF zBYo|0o_WDd$Xn#})v5zq8t7Cv^Se_-(D~_W%RfhL=HY0Nxc*Q+`6)FdvV$Y9)QOZ^ z2XF94J_WQxu?bLax%8 zyXY~96fg!Ztn$!kI!jR;Nj#^VaQBgWk4ul+Thldm0<6u=Z&f55Z(Y%(s}&Yzz$_Bb z&KosBapVT`F$te+fZq0w7eB`rwhSSaNpf8OLvbwUY5hF&(-m-G?13gtouz+o?jo0Z z+3@zUf>$y3lXTqnRcgUlpfaW+zzsjrN5D4iw z<0mrQ;hH`ecw_S^!G{KB=}!Ieo3<)F72Xr|2Tl)<9-~vJQ(9;kY^mg@54a-WV8MNjkF;!4rn8S;E-DRdlom|F55sVnvSD zoFoVBh7uU?NW};P_$K6QO&`X6j6#tOWbezfh$0qxs5Q0_-=E`G9jy3%s($&D&q<@| zo}0|7^z3^$kIjw(DZ*wQb9wDPrId8n?K=4dQO0UY(tEK3NAsTWOIBIvF#9f_scA2E ze&Xhx;I%2g`eTOp#%E=MAHxMw?tS+-4-q<1rX&_gkp!&vI|xXuGqicDA*WAy!`>WX z<;^@d++ma~?z?F(sGX4j)s7!)s8XX48>5WI!n+}Q1fx19_~CPenX*{qMfjJ{IVh97 zfho)O)FI{ycZYBUWN7iE|a{?-)Pq3|@6;BY9Qm$cXpO+DuQ6 zts0=_>pSl`&yB?j;tr3NYJsN|p7+*>CQEc1Za+DAvAQio-0sGuM1Q#>thY4y{)jMC z#02fz2&=p`zshPq)MNqa8WGhJuHKOI(CedJ`oPUwT3i!#2K`Mc8cAFXL9Lf?&@>Ey z;$)Iejb9kH?7e2QnK7dTzq8~X!!(FAdfwAtD0r{6UZ4O|w-b6!2_3j!fvY2W+@5n3$H{Quhh1B53D);B(p%o@(&zU z7hz9P2Y_lc98QJ-nE69&9_K1G=M2oc`Z-_^j#}CbN5>VN3a}7BF%0QHn;k`!KI;clntCf}#AXsME22BqOTaVTl64$A=>0c|s#6yw+DYB7DGNV7+_^DEJGLr!!Gq9u7ei>Q0yw!K6vS{7 z;R;L{JPRJ+DEITyhz)JEYa*Q%udLLdq=S1^K}dz7gPZu-aik)IQ2yEp)nZXMwtBW3 z^@)r+IVFsix_o{5GS+ZcPjVVg(x3EB?A9o89AsFqzWEFZLA9y1cQo?CmB{-3RzN*! zns|Q!y@v0u#!kU;U7Pp$@5f^E(86V0_jfdp{IQt)~o1mbY0;lZZ zqdibH@gX&ps_PL>xX^V3)eiO+q=IlCX}m?}vWZLpNhjM%g*c?<>}BuIBEqA9fQODG zc;~Ti4{laXIOW7eW-{n6?y+gS2(Lv7 zi)2Q=oD-G%ZA76_>)B9d))H;%M%}hc4aikvj!fA8_AemWH_1{_q_-d3s^<=9ys47@ zD)E6f(~0A+y@1Pla(tb#{mr6;46}3XO)JhXy%nmta?<6_6&zpNjq3BqIEFf9ggJFH z0<6uHw>D#t{hbfv4Nx89+7*WmMVAVYF&ICh_GF4njDrxCBCqn=Lar{=X%(RT7>!H!jW6%Gh zSL3lhylfwskZFGUt4`qq_@a7Ke zVqX`%;Sp*$;L=IqjqyZc6{?Oh&_tu9A zE{3BwH*a|r{bH@*0muSr_rvy`;Ofj=LdBEZpBu+a;WpcdnB1bxT5}CEiFr=p53Jb) z^oz|{zmq%vZjCE#Qd_c9-edAb^eg$S%pT=1QNXKpyh-O$1*mV6Ryz8FNN4 zV2Pvi?tET_+JYUnMxzHp*XVLog2m8CcO<%0+PUPV7W;T*b~f^P0PgUBAxu689PsDl za}Q9meDSubHb-uSW`^3D;e>bFpV!5j2z(DttEQ}LlE2v>mf;#IGKQRY*Qxqup@N1- za{0lnnX+?6w~)88gi0-zkjjYrkvCgMVRf8ozY931$AF-hmqQ3L1$N<28=Z(LOZIDw z4#xatwCUw&;58zCk_)OCA#%JdH2-0*;WGm9zd~TMdj;zuU+rsNOBi%h5 z9W2Qi_2iHeHehQ43m8HYlpNyo=hrQT7@I`2o>=5pfjYMW6Wusn%jF`h?1V$LIvJmu z1bfzALl-m6+Mu#*2rr0Pa6}7H(}DFmtlZGpB%{#h1?k5{G<*02l-7vQn9mnoit13; z?6~RU)*T|eaVf$A-zNmr^<`8TD!pnUV8c8H{zWa2T)>m5r>t)x{QQM32agOafhYJn zH;7?$w1px~`t_J_{@*cprft7LimGoT#7Zsa^(VufZA}(KdoqKYN=@6Q8ihs^6*|T)>2D2+fJK{i<|du7SMd8#UnWz3 zjDKz5X*=!PZSnRGU9=fc_?Lw#O$>hJip)WHa~@@$H>%FBemLQCCPp>xnbTH*HBb9P zB{jm@oY`Zxsa~7bY1Z|@9ZawV8WI_NUX9{RG;8YkZ;DkHa#usElYf)&m2jQ(3k~!# zgV@58OW)H%^M?;z7qYfD8NPMtlho!wji||&k`vdFJVG+^T4#)&I^-YxyE=c|Sk7A$ zi4>RQgR7`2!Qv<~V}=s%kM^!=K0!OdF-W&E zFRA9{K{ms+MoCSna^95^Duw;KjBFbB^d*ePBjHFSVSFHG6fkkY&}xKr1BI#@ z-nIzz{uz$stD^h}7ks($lA~U2&j^RFY2WjA;OPqm%D)xx|M&&HcOfRt6~@Tm;ZJlS z;dmxxi12;T52nGiazGEmg#kH0nkWoOj)MFAK*YmPOrlc_r`sz5Ur!=SG7GP9c0KfV zee|<#&a}Q$BSmy~kCnCFZj&MtAoUh-9&rX&Ns+Xi!21p)io%f}-S>VB5cLzhdd`V& zkD|0$1-bIl9~`Z_R6_W0lIaj$HRc%N%dA>!=7cfv9B4Ja5RsheT~vM3H6_k0z^U~~ z;G^)%aYZS-U{P2n6sUTbS<(8T=HgFZ1Y!)pp*r>);F>@=DzBIALM>0SJha1>;7Q3a zDCAuU%YI|QVEwDnouapt7D&hk!U)#wb!Th zhscZfM!H$Icf{m+F1^#ULyxMuT9e*x5<<9VfSPnbWeXM6a)`+`yyoru$mf`q=K7tl zr406%Nd(d+$$k-t^l6ywt8euOjt^GIAaTbgrg}^r3)WqDDE+abbLrBJBd1!ITvBxs z=4f~bffES4D*Rg@(c2`98XWf!VkH2tNkmq3{CrQBZ_NBnfIrxb>0vcYUtj<^QjfU9>0b-$R= zsFtUz{z%z&-^%C@+k|X2`~zvzRd|_T_1O-vwj9xGFj zXda4Q3ln&y-GwY)TzD=hPixnhox=Zx%rAGdAP{M);?i_t?UCFNG>*x}PpZNOmE8t$$mDDTOddlfqz0bG zLnjKp~gbfbi`FFS-)C zr7CM@#v?HSYuJh1VCvtX7YJWWc@nxBVL4*ru)Y*@2zhs$mxx&436BdIQwvt}Xb$XE z|8_dZ5r>p4qQsJwGJ76T(?{9^WZxz~t2Xm(xRioBaqle5D~s;dJANLj`504Wko zLXD4Q91jSPD(M+I5`SLJVNUMr^?hLt2(5E?quj)_ivFq=yYX#D3L};z^1!{ht(n;8 z$SP`NEp|8ok4od=5)k8IT_^KDf1$#G`JfZ$59*d1UQm_~?95uh-m4}8Md~JzwWS;Q zwOr}2TXzsN6PAC#09}u-wormbO2A67VFM*wyRezBn_~_Y7T(fM*vj6kBNAn+!=EZd zf3ujg7QIUS!FL>wE`rd2Eb$qg>4#i3vtk&adc?9Ge3jy?V*8EJ-FnyeZ=0+c9DAX< zX2Z&@=RY~K!FW^%_WRAj@V98KM zMH%7`-}6ik*|~Ov#6@bWD3N7<;NG08#80#4AC=O_Bu+>K27x(Z+7M!jO2ZENc?Jl@ ziN*#6#;=g#=1Mp=2?^f-Vxk$mdf?XrUN`aICk*d-ZlO;75esYCPUJZ1aP+8=^eMGg z_`@;-Y$d(?F(x~qCXw{_b=Zwm1P#)F2mSBkeGZh0gEi{1EJ=<`j<)7=t29LYPkYOBbB-pY(XRHZp=d z6O_K3*E+lM`NaNq=KOnSr_2Xv`lO^@@|Rxntp z%ufxfO%-u(VoEkRI||@Wk~qhCqTI(70(i?@M#NTz5W0XYxOI>yJPO8|9$U2TT4tAf zPfwtMQ|$7CchqgwaykVaciU=DK|DgbREe{xOKiyXQX|*&EhxQ?Aw0~?tWH@0d)fb( zz&F+4yytaM0kx)Rlbu|vXDXo{c~bmOZK97bw;_?eX3xOL$3f>_akJl{ zD1xGQ_U}xaOx$gM+xEVYh+Q^ zACtBYTVSyv{nF%8L}+{2g@fgfp70O{p>@bhH%(EIOr2m(PFoRS2(B6Fn6 z{&Ri~J$mZ=(xiQA($cY0}G+WsyWvz44~5&R13=v`Px-tqM1Lz}a4f`j3N6 zNPXVC1_;|umngpIBfH<-EAcH5iZu9XK%HECv9oG@>=xwZ#CS(PMX(#Irptc~vwm*v zAUg8d#b;1J+IyLZDT%WoS9A@Wsn;Jk6WmC{0(t_&})s(sBUo}orw&sFJ}EMs@b4f*5&EZqu@OYRvfueK>ygf9i=dToasK<^*O|1H&Xu&-Zt*6 z#JXFqE*925du>@H9H_lAC&DeTT)?fAcb@wB1T}QvtZqnA19Jf&yTq$dR^0_IaTLVO z85pyYUXall~2?wssCsX&d7;1yLn zuY9*J;c)zU?WP61au>~IUlxY?pTteL-$qHSMG>EyXm8BIX3m=+eMXX%Ws;OUBj3oK z`HftzD?}QANC=fJo9K6!zq|FBcBh-a4`}riJFEeRTrTA3I$SxN&_n|>M>`KAJO(|l zKJbqf(dZ9rlO82b9HzlrD>s#BZF#!5@CN4o02_v0MS2Bk@Iofpi$(UrIkr#Za6`gf zM}{}5yMsHh;E6zg#|iqPQjIsEHhHEqPM@UxmI%PY>W7g@S%_KZX8hy|2`~u71Yumb zFnQUeSG(Shm;<)!pbyKum74_cVBwQeW%4Eor953s1kQxFwE6hWhX=-yGyY70-J#i~ zsx9ThPc&Qf!aTy^HsCL@z1j)rA~-2o_idV6L)Vu%?g|mc@pbOxKdQaC>Vh)tHKeTF z2Iy6ybT2yIP9ie6u2<{5h-U955o7;J5!kG=+ddP^dPU>j#lw=``P&v=iFY_X336v# z4p&kt&LvM5nn($sdmV_gByJV`DDjGgzzc42@gnbloPD)=uJWNphO7uDh2iU#8etya z4MI2)nkT&4%_TXnd04yyC1I_l>saZuaizi6Bp;8N^@AOL?;U-$#ZvcM;P{^6K=3pP zfS(6r0vveQl8%CmNKbU9m01|KL9DJZDkja1*b!6m<-`nC{YGSSr`z{wZGM!=0FiN8-1|+nU=# z-HIHD`!SXx7UBwi0|+s1%#2tg!+?v+m*fcD;0>yo1r2w3VOiglAP@F>(RmYJ7BG#u zd?&z68eq#Hy^u`?sFrf(7wVwADoko|Dtu|B3IzLw0@Cwh6$bkvu64-JO(GIxY6^su zUio@vHflM+hZuEz<9%uJu8fe(-*9ZJldvIiyb?+ONuaAh`fmplnEF6IlGPuc)BeI! zH4fF;;!-tv1U+MA@~XV5X~N>{O~)W}ys7C~i>*RJawll5J4( z2CSzQpoS&^TUk_3dL8se=3C2_gcU-LO`+M6C#@ILm=^G$v$dO5NB_Ro#BtaHfhQ?J zG<4Z)x$fP^-bWA-@9^ZeM|IwM*qYtgh)>l!xt_)X{1f_`LtwSv2a9Tfem{RWt+zou zW!?Ka6H6^!{*#>=;pIZ~t_QEOsDh};WI$1I%Zbkkn_QCnWA0oXv0Z5kj)-^|v|Vq< zqt{3N0=(k{`0}v4BG~uP!yNJFr7;hZfHrg`R9-)Htt}V6FF!jENooks{yd8!QV=P~ zW_MS+QLN!&$BpuPB|`YhQbnJ@R`M!M|GU+u|IL|CJg;5yOBN1)yg7la5 zx@E+^PK#4r{(GmQ+PNn>^4PA(fmw<_!m#FHGCE+gl1I<(5A`m2ZsJ?wEja!{YP+*^ z_7j_-Z7;Pb>Hby&1{GziK})oQ2#4&PclVs=%;OH1&0HQLLvY>yj|;spNL0En*YeHm{0W+s4(<& zriQ&2j>nj6_q-JfNRK7jcdOd`Udvr&aS7&L+vqddwIA_g=D!^HaiG(*?a;ex-#&i7 zxRC4{1w*-?!9-(?!QuUhoR!4zi^Xfxk$yQmXA2GqbDs?(yQg>M&pb}|aN9_|V0$1= z>e%}`8|8nOj@9M*A^l$sB2#cpfMH_@a%c4wH?@wh+HG%e)jnv^M}NWD&Z>?MiCUq%Kc@C~ryfs{ z+2d}f`I~BV)@os@E8{Kj{w;+n2z(Kq>662axC0mKHfkJ{;hsQcBEe{q%SHA3inRI@%qvTZFUdoHgTX^=^N% z-LDnXoH?qxg9gGQz+pcT+int^bIh#8%S!>U=p9N|>d^hGz|oLXR(j2ey9zjx74`wU9UI`4(4@zQ;XbAd;Zi`+<+lQ>NBtG!w~Az1 z)Eh62g|ffpcquc1^U6Ko@mKSVJgPaIl$-vbw>HCDQKsu-Re<)>*IP#}c)HkVF^xV@ zuTkaK0^Qas*w;%hz_mKVhm{hTCt+M3{s~Y{W5yr6=ko7;w@s_yEf|6l>E+Ekr)O*OGrlcPyW_`AucLnO?N&yne2xY5cz1X&KW_`=x?!b@LW2$$3{)hzT{q zBSK6UxPwT=jsq8=1H+wFXwz(@>sJ4YvtdoOfUI`BsoxHH`D?3Za2;qnIPRq|w-YxF z)+do?ect<_nza~{jZ##-?TMF#uw09nYGTP5gV=IPTsyWjieTT9nmmz{2ByO-zgSmzQ6E_~|xjvk=8D_hy zy~Rw{biWeM&Q!7)vxGjHlm?a`BlS)#N4to&2Rx(~bCBnUj=Vzl>A=?VLU7~~tKC%8 zJnFn3F&|~e)g(&4SZnk!WMxPdq--VZkt_H5B#SS&Q)blf16-@mD}BsOnrod=K@(Xj zPTc31{BY7Edd7aY{efAwvHQ|r-)U~nh4!iJPx4x!k|4RBR?NEt2R9-E$}L5bKm5ah zxfQ^f7B_+;J7`Xrd5gLs$U~2E0gn|ecRt1UZrQTyth(LmojCq@hXwTRg6>+3U!j>Y z{#vq#@cD~@!)(^4iSD2QT4j#Tj47IyF?>PjuFq-V@O+@W(6!ms+}^%P$;DYC?2X2! zb2OGw5n9OF>24!@^(?WpZ<0-*v8SL?pTMM|lE1X7HRLJzA!_?eahbw(`-lRtw_26; zmv%z_$8Ug$voDVtTP%B)ZHmscckg=^Z);|nxpL*l%`SIfhyzYV`$iK`wKONf9ieQG z-WvJjEV#c0Ac-E+Y-93n^iB}l3YDn)Ugi4mX+1-2wYqU}@~vCBY2ptR%b}K_Kk5(M z$=jGLS#PofTalxi3ZBDq0vZMW#G<|r9c31#l$j9b+S04Ra-=zL?QXje!8S6t3t+!J z9fWq$SBJ1t{i{MQ zc_b!o0*jr-WRxF+nN&~S@hL`j=X^mJteWTOVl$JQ<1=h=o(J}Zt`RYhIf*bcN)?c% z{9k4rJg^CMFr1R`#xie&2dRm`9}{u`P&l}L5X#{tegCD)kbnPz50iMz+cY@^9~ll` z+g;uVefr?CI7?i*oHmP~cbI7d7s%^b{IvrmD{NYQ-)Po}hc5A@>O(fE%5g7xDffXm z-xGJ7@Z+R5zS_)$16gm)b!m7>GI`^{`y&GQxpO=f{17E)V&={AAsQnJWnpjNN%OH$ zq9kv_kp*6{m~zWilZ%CJ>p?N?Et!GqdYnUG3R5&$y;ml^5=YW@7HHq~-iX^)%-i$R z5QGYxEqjKlY>Ao9!qp!xko7` zbkk;KP<>|Mz^#t*KGB~2fv2~}+ZvLDjs~rFxf4?ip3{*VAHg*Q^;=QGnlVNWjw}W* zdVcF&To$K=E>;Eqy@TC$91$&5SY$Flhdcf9DlnbZNq(cw=yH~q8a=RPC00MHy6<>4 zi;3&UtvFrXg@Np{tIU83udbdD&*p4Cy!SN?ZZe^Z#thw) z_l>sE;Tl0ME#{De%<=ns5AX$9;MC@KfM+>e`V3$=OUdGNZm zM%YJZRHvFnV%&+d=4m=@q<-GeAVtPwt$H`G--G#*j;@**XwbiN`%}D+rZ(?M3wW^y zxhj#{LCRno@m)b>!5dz3(>Ui1mG#+Kq^?T>Zg!y-O`Q1~CG#p*!Xca}t`ql&T6w|k zb-YW`?+d`wG=-$7nsw!b`d4h@FBuQODlj9aCuQs)GqY~i`FVs_@rs9OHM7!LTKW$U zWGyHI-KcI!d*o0-3P9eG5IeYTzHnjZaqScE^x(6QDHT9Hf4bYk2h7mgu)va-j98Y6yo2#@bNf9l!&oJ}1 z_S4eJcu8LKbhn84722Icpkenk>__i9nNfaRq;_z@zMLR*?irw0mJnywqV5^TlZ28_ zpCUw@>4!mPatfNYXMRhPk-oAx| zA=ef=$Hf#NV#Jzbulzsa@JhGK!?6Quc zB&0-CG9)BPsVLI4M=C8ug;NwIj0mMGQy~>Km93CTwl-O^Ma)&mZ^myMM3ekFG13G0YeLQDGfJ!l z2yDyevKDx_`Cs6(3n1xCLbW}<_1p!l`vxTjJ=i!YSH-$LuibubVYaHDcJ?xD+9^-r zaTka~1+nN~fBs(8b{X0ma0|ELF{e6oM(;>w8or=eVV8VLYg|PCs&+4$(d+lHUut0M z^u*zxrE>9~|4pqniTK^_Eq1-_@2_dHvSU+qGISIipzBW<7erqp{YW&j0%4y?dHERH z{%20okL)HDrBrkTI^2Db*lv5}wW+w)Dd`jEp(yAw?ZM$q))s$@FUC)Jt2llwc1*7~ zda2wJlNG=fH%SvHLfvPu>PaMYtPFH&iIROScs>$zyhY>5Mpq@H_ZP384&8nu280i? zN?WGmp69cl zF!}vAXt>tT8#O*I^vcrmpyn7e8A$rA0h;CWC8z!sO;ooIoOAqouf1_t`M??lZRhn9 zuU9_aLg}VK@KqLQgN8i;sJ=f~rLUm9@Ikx#JE`;n-QoM!R2Ad=T=o$=_4r32DP+O? zUuh;8O@+8rICS~`Lk(@7r6MaQx!-AFG_xqMeE-YekfL;@yvTx z%J8Rv8b4r5COO;c={`%*;|DTWUDXj}qAxUL`&1lrvaRtQaJZ3Wu)$$Y*0Q%RUxfr_ zJ(}57)AU1Zv*-(hLxr(c-l`5-zS~4-Z~Y!CvYD4H0U6Au@*vOFLCFO$e~g=SAF6;m zox;u?&pxK`Ehj(eLg#J8bw_tPHSCZ^fa)7O|U<0XAPQFX_bdO>YB!AiW3(v^2^v#{sy92y^D%uo&F{f=U2xoCbz z#6u?V&l9*liEUl|!$Ijalczin+>HjKB{+m^2A#^lkA0O&WdJ=rX=>vHWGnGjgzfs1 z@WZCEk_qy?RQ|$q5F%|Lq|DsiX(hkGqqzgvoOJFJdU9GzQ|5`2_Pm6E#0giJguqZ% z<-LHEA@ok5*wgxX4wn~lp2Wr3!KBvK6L-R0M5Kde(zz+{tOQJDmu~C)wzlAb=OP_y zP(ntb*CzuU2>BmB!4Sd-x{V>h_`dVE4*}!<_a8yz$r|L7?+qv-5Ex#V^gk1CBOh#9 zt9-lb;Px4#z%;UluqwWe(F&pO-)vFvIizG`usTMr0YV0bs>x&G+Rs^Z19PRU529s? z&+PO!<(*RS^#kRiQRoE(O*lXqr~k)+uUYu&-iA&yy5^z%AtI6X7GE3VUhaqP>U>O2 zu9I5!LEt`UC3<=zGDl8Z8%7%+3}_!cL3krZLhW+Dsir-9A+d_43%30EE}=R`n=m&8-Oes1e)&*&x)hMVy0PkLmOe{F z<~SDioKY%)OgzRpFpNEfWb4}SQv(UZXD)QE(U1Rb(WL0jZ=KJP3Z1`;>E0`hyJdhh zmYj|8)AiYwt_y7 zaLQqv1JsFdDdd0bKN36kJ86vn`@;lh;CLYTy}LB{&+BP$K>oLXfwev3bOoPZgLJr~ z7iSLy-ZmN(>dBxicr}2{$F-z`=Xs}{l1laaNl100=azYa zNxZy{MDo7qU%vzy_jZX7gm=H+UgVew!AN^`HBR0oynMCNX(%R6-kF0yD(gQc`Db2W zj@p=${>Jfd)i8UGMVGMFFEV}0z+>#;;^5`zi^tI=q*gF)qwk<+xJRJm5HKp#uXh3Z zBxfHut(~G$ck@P^eRfEklXLI}SZYx?zINu;KM58}q{%+m#?*iQ$biiSXqFCWmNt!M zFQ@*Gp8?%^MwBMw6ARz;ju3ww5V*YI44$!fff|e;hs;kL6+OuANt|i4#7gnn0S)h( z1K@}=Klq^ceN!UvU3aFzC^Tq!8Aej%jv}5kjW0U`s36cCX!A(>WP11eN16VMB<@%c zgrT&!S)7-Qys^H6RDMwEy6Mq z53StiSa8a|JDD)K!pzwlGa>sJ2z$hH8_}nnFT0Friv8$(of49F{I$_Z`lS97*{%+s zQq`C3{R5@RyPm%-zuo>x@zDNv+SlLdyo9CO?KB4snRxo*mxrNhACt>DH6trSLy0>+PX}>a4boLK^!Lgws^3jpV#>UIfY)Fw4e(RvvH2dXJf|}6aBH?cV z!Mz?Lehi6g1|lvi9|(^%6P!%&W*U3FqrXiAKI<%7 zR(C|xjw0gOasBO`Fz&&(ntyO8oxV~l6~2DKoY15UH@m$VXvEggncK8aR{9&=Tl8JQ zI)oq~7Z```XV~*e0|;-5*5!6`W4!J2vXX?Cl10#rd&j~6a{&N0>n}L&hu1OL>m6FF zKvj;)HsWo**_%QU>(d`yU*2}3r7{kx)fV?d$adfVuT=Q{n~Gw@9z9&@TZ{wn!h{#c z>Yj$XToHca4j@eu^w{e0mOHB-`}UJiOazg;j)3eE3u(@9OuGi>Q<7I!-|+jPW9Mf$ z%{sW!MOX8nwnl`SYa&v*82@+}_v!_-&UW>JLAdY2SP_|3*Sh)zne}Y9^4WX7ltF|2 ziO%BKdA=ThpWTL0>MTqQUBFtiH_)1e%=9c+Go@?u^7*sSS!?qpg)1&R4bB}P>>H6tay@+TpP2oVd6&n9z_tLA`oMf^DQ%SCdJJo;2Fx>a0$c>3V- z5~IxadryhVE`%838a*tN+>uH4f^%#lSH?h1pvuyfBZLF;XP;63md0e;!0=+4b80@f z@zU!(KyH!&b-DpbwR5gnD7~wGrPYVxHH+94}TS1#D|NVEQ>jxfFk#Go5I?3|SO7&6%mgC|ghpi2gnmRrnf z+{aN0^HY7SvpLMdqw0@pk%!m9a%;ks)znlb=;)E^7Jt&{7IO#aX?17dD?wfx{EoQQ z?k`nvzZ5h>ZMoL6cn8)}9h`iRKB~Gm48%F)U_M7Z@Yubo4u*0-wR=O(4^EH!Zpv4u z*+VN;W>smWkmY~gl3+>O6FM#fsb9|PLR7QeB3~Tt)V_=7^%~O znIOr89Nw+F`#3s7b2wcUtBl*#IQ~%+ zRa>225*&{CKiK6sJH$%p+Si9A%H6IpHR8hzOx3NAgefAd(zk0boIM{G04G*=fPYCW zH6teb_ikV>XEP&<>{(TZNHXYJyvw29;L5UHEgNd8l(k5DcLiO)yBT2PcJelLk=W1v zwyeXfB(Aer%*tyw#)U4J9AUhhjnAp#-?cCgsK#d#*+^-MA?NY z?wYN8`73qa)=;4yMk9ARy4H%@(cui6RNbBx?B~vN*aRc5mm&Or2KqEScx*ywe3=H+ zi$ckPCpe@_^e4t`YnyEB!h0HR!#!JvHb!2(X@|>I}RSQ6ACfgb5^*H(P$q0o|!$b8gs9nGXdz& z10oVta7J9I(+(K11ta9TTM%Sd7ttn5Aj+W(R;~BTPXh5HzyxhTkMxkY43s>~*-Ttw z2tnHnF(dk51=5oe01#313=IPaSs4#avxoh~F@^Ii-_K>=uXe6K^)XFol~*M^ zH-8MX1FJ~or1I>X*693Njt&Tm zV81&dwd+7!aL#c6c0(j~11*OQ^7U6b!2rMm;Ix4M^CY5+6^8y`us8X{EVD6MZW?NG zhp@+f;ip{$Jq(~=wjb$7$FKdh1s?V-W{yrJO)Z#76Dyk6MON`n&WnGDR@&tiJ};_Y zvM~QexciHSu0)+vPhv7)rc5i0G6-*z%;YVj{m4jBh4t{DPwy{IFzge`4hnpQ=vM_LR&vS!N85X@+sf%>ANS#$h z+|c056z`v0k(78YsK`{>Q9Uav&St*m0Y6R9zyVSd&X*VbM+~prn++E8(Cec6#4*R? zd}ILTb8?ZPF=W5apc7}i)=g#Vr3rHaux?Ev0O9BKjkno-LdMTpaBwjQ1;0}V$voTO z8V!(1#gzkm%5c-BTy+axS+cLGV$-VVk58*qKi!R6-@Z00OA46QSG)4`YE@<2$j2p3a7G}1d@s0UDhQ+G<-a+*Ak=0%=TNNuVoOrrNuDe3dijICPLq$EaZ*_^ z%?rL@qwx1}u6E1jE#$Yb(_aG;-nuN$JYtUB9gj!f2o7b52zCHEHn`aAf35_*K$ z5`a`qZTTi8_%a9+MnK_i z-fxAw-Cd`7TY~G7@#hGa$^d7vKl&0Y8)9k`ul5wbzO})ugHG_&k2? z2xUGN$iImkHqc^Gq9#1_UI%OB739`T%-Wl5u;PoxO4$}$7Z;h(-%DAZxsCq_@p_S~ zcb(cvxw}s&Md_^Y?I(J)x3+gzc98xbv#UI3Yy##NrX|q12xQ zb8RpNj{XcW&G3OP;>DpS^5&sZCVf)-Gjh=9OTzfT86=`u^<6Z% zN6DLC3$&mBxO9;0T%pEUVPmKi5h(PdIXYZ7ezF0sHZW$*=9NYcWOyj-O>Y58 zrKeV-qk7mXi+Po57UZV+mqoYNFR>)-AuKpbSR@=vNDH zzak%El*@k#M59J@F~(QwYs&NL!D{k_oSo_D`HtsRdym?$@_49yBZ-j&*8e;_WR1j8 z7`he57Qv@6dfO?}pN147)25#MH&ahV;l|?eEC0Zdi_KTxhi0Hy@Mo#ud`L0?ppby` zuDk+Fx?W$2m48v#aW1(z8F3+v4Ut}*JU23!&0Ah_Z?~gTDRWNvdTq_o?%v(kzU z{3SG(8-T+*IRl)(9795fslTgflk@KO)egoVJ=)$KI$J4DpEi|^`(-#o1Z+wILw%_yFHN19*nE%7+TiWx-nP(VSOc1Hx8MoAaU;g$p)Bn@`3jL%+MU*pTR+WQCp_L{y?9Dv2*it=EM z0G|+~M+uH*#Q^5c@FKQQxRGFrJV~maEv8g+D=#mv$4nc^rTwnDYA4f%6=Ym2p%5z^ z0#miGfi4wJ`ww*5u{l5#@GsmK3qT}`pAmu8Pn)X09)%13xe`C3a|L00`@=ARr!c?e zV0})P+y!$%@Br@i2;SCc`aHM=QPeQ=NGRHw8M`)aCU4hmvnR5KEt%X~Uh<}8WLZ9o z+^y% Lf>OM}Z9*%_~i^7vbZWRZO*M4YR_Q<0A7#hpDpy_@n9z0%_vSDe*81C#wA zw;6ZVSO{N0`5S6V|K|bl$BZ%t0luyqD@9PfT!A%Xuwk`y_;UZkRNybxBZvC6R)+pM zg-CuSCfnymMs-N9-*$`c2q3M4)_}3dSM~UNTDGzX@}`K*Bu`8mN57$h-3j-~Xhtq=-~M{DIDsHa@RYld{Gf@uxZLdkG;i#UMONiEGRkSV#6jLh zyf&GH$uIBPO-&6LD^pBNY&UM)A3oPl?|O`}`^y=R7QdbGKn*Mf|6a`9y@mY1KfQ@8 z$n1Sf=mw=opXkxj+0XN@P8iK}wh}u;Pp%vS{QXBfY|~fZI_5!Kvl@azh-coTqU`0B z2-~STtk{S{5A$`TxYifxWqkE$Z*O+5K50&uO^n-lVC|>Dv-)Lm=V4WfkV8i(V9vL+-64`hc`jUDj}!nYSv{lS0eI` z=;l>~WtJ-2{%;6u{5K`{{;aAJds=a+*D>}8>uS5%I?-Fa!)sZRG?c#q6swqnkO2A7 z1=!D zcQA-Y*xo9O?atrtJCoNm^ZD-iFLv&W-XW$YO`Om8u=C~0JK~jc?_1|PS3V(V;|cqD z7ASQ>7Ga7p{K>4A>IhsEy$y)cAN6A~uc(+LGSa@5KDe2cJ5%dre(d9{Oa6BApRXXo zg;M%+$a|>G&}3Vs$p$TYeFTMxjBqEe9~5>t@i=OX9Hv0`X{_{`D@qzdp(Y?_vJ?%fWB|kI(%7|NF0xV5$_K zHFQkTpY<_c=J}$cSMN@2BaZ79CPTitT2RBU-u5eI8M_PXJjhQe77ngrvNup8Y0BU^ z#5_~PIlvsHI#6^~!fEu&blTXVWDDDxLBhd<(lrhmS4k z+6)0%mgW-Vly=L4P$no?)`ojjX4d6^9u6n^gEN4o!cUdPH9^Sm^`c$a5Mc= zi-dB1Tgo6;`_mg*KMDgb)a@^2#$o`7vDo`by1$Ci=@ArZ0e zU#~_=h4E76_sIDaN~4qDKxxiQ7LROqV(7mkWZ%0zKL`&kI6PwLhODx~%+K@!58WIWc~`c9jA@5}PlIY)NhxM8oI zV!sCX02bM_+YF4sZz9ShNCAGYI4>4XW~F-8bYb zLJhV^EzZ_jZ^@N?Gc~)V15@raHN;vg6zNIM)LI83I)W%q<4wqdg^#ZC$UpjF{)1fc zlBwZnzzFrb$=Vl_sH86_x~WQA4O-DT3Y^}dKHl+GwAU>^taCEznk@cpyKO!&7Q4JA z6$9Zb|CPC2&_VcC{v^cSYq}xZ@EP(Va-349S7a4)%H>^Ptw%O@>q4vIg7e42c;|v! z^HtjZki&L!;fU}v|Wb76-ggm&bjxinw{!07? z(ntpysM6q%P>?=u0oVTQG{DTd^$&zw7Lv72R}i6lBaZ)I&_|UU$LZT~U}e(;v4cFu zp1_ComzM|95uOBIe(v#jOG*Yi>5Y<-snh}BQ*&SFe`clr$Gs49X~Q9-M6cqRmjGrx zlKSLxSjo5v-rLI+=SOfrmGUp3)`UL8zp89XxPvz{oQ>0E$~q0SyH zrQqXRP%X8%#m`aejUtM|q?c~My0ZsqKJSmgnu?Kl7n%hD14JK0RF^p9$J>VK@?Qln z!B(LAIa+1y8f@;B&giwDH+?p4bM9}ft7~7mLwmEeeB`x?)Jxk5LUkeJFFr$DWUPS< zxf5SuGc1u0pGIdmG%MM_o#&q8VCNuT@AL89`a@xv#_pQL#m zO18_3%6>cSuw3?(Jvg)})j#=~1poi9aW?%og%|&f3kClt_Eh6im4XILl!=$RDdCCf zNL;EPeew-}$Wm$qbX^yjLO8had!oj?I-LV&^>g(Ch99@Z4m;Ow$Vz+IEo)}NvgS(g zFQ5EL(2n>MTWk};cfuC+rjgh?>EXyksWTV>dp&-Dxbjrf7;HarNlXGY;4MQ+^vn@X z1otF6k;*bu?2Fqm+tIs}yV)zBJ!{E+JDIPo`llWoT;MKgGzYn<7u?Rs1D^7$xKS_d z#~i`q1q4%GzRM9(;9+Q?cpThL_>%+bTCu;5Ex4vq=JI3E(XL?FNA-;DSy}m zS3=nB%zSaa0+>e-UJxKhxWLl$!L5AU5QT&H_4sczdAC>ofGfHinf_fd{0{kgO7N>K zP(c{soj_^@!Ois75s-of3vG}~y_jkK(5qH(A`$!wLDWMMe3uZEWJuG z8R(CfBdA#tzWE)8iNA(4UHaxGwAq!n2q`I}jfOi>NnBwk{^D>(MR5q(6GE2u6G#V# zAx!1l0BoNQo5U4x(ROJL?P@Ryjch8%MeB!ij=8{-!I{ zZoaMZ`U$b(_!%KHPTJZ81B@>@gl*~c5rMxL1{(*^OIj{{N_g0ep7Amm9$Mge=t}7l zW$m@fPHO-{a7p?3L$A$9N*KU|u)-01jU{Oi9Em8v>g&4#!B{`j+5roBn9f<);?0HG z%n07j;Lk8=NM_w-idoFXwxSW8POh>sQqNBovSng4ZJ@ zYXvk$U)(z)1`Rm=ora~)O8#&O{$a3N9DU&j~DwFc~l-v|5ksz3?{ zQqnJ`-Dm3bxBdP7a3M#lTjFuY>`DfgP)`GfEh+lhiFd>VQ$T~v|EcE@p#HBBItpAi&nybBNf9MY zEcIecZC6K%#HdiIx<#4lq)pAdGe7{0FwAPyebuG|v zc$-Ih>NDuDAj-&|eYNCcl}g;EEBl_Zgxe^j1g!na#O2l8a@H9gmE|EiMjO z$RaEi7i@grqMrEq(>uS3%Y?ni05>xYeJBEt7$!iA4EEVS!J?n_ZKH1FvknARj!2kc z#iwASfSgbwwuuhRzIO+{8iR-)_iNJTP~Bj|fOiRLmR5};mo=~bBsgaB<1FML1@J5} zmz`hg_KD6;ePinCe{TEr&yGu?4~&BLrPk^~<8?6H10^J&KC%v(A{M3No_&mXGe7AP5trLPN~Y z$lQ!2Zo!~ciG1i3x#N#OAttW-N)LL)=hNB#;^HO~j!8qcJTI@j&3y&_<@C8$KF6=B zZK-^``_X(&q3zo?5(tvhb7^g+^Z&9Bq_G|50xSIitTb51IRiW&*mV>@4!`r11q(X_ z#8T%agImdp$l$RCSez#t4*wgXXc!%)XZ9y^(=D!;;lJhtqt_O(wtZ{J#Q zBgI~4VaQ?#Ae?w>1<{2JiE{~=MdfdBBjeX9DILD53KJR$yc77BWW}34_r5a}iafjy zd#RL5N)MP`0h<0d_vXK!@hAq?0%C=XD*Rz_IHQ!r<% zgD$8gv4;&WFu2;Dv?)6UB;qwaW1QUT%`InFM0pnDk25r1<$K++gS-Z$@a$KPGik`PuC?=bK-K(a5C5;&`^65Ddl4v(wc zJ&3^jFaH6J7kB^6^#>VtjLZo@-mFrmhX?8%6uWi5x#DSSOH0KwYkAM557j(pJi73j zut$Q>dN1-XktRZxp&as-aURqqwDIkQe~0@IgctwC#Hw)N;H2Dla_nRDoCM)z+mUl& z1@+?sRkc6=2>n;Z1jV9yll_pD2-}xl4&0zjVT2X$)=LeR3ZQOb=jD?w`+B0Mn>@mpOB7DUW6@Y2| z0PSJ`1OGQ(exdq^(Az2`sS;C_;T^-9$*N@5lVBoSynN&ezL2vb;$)h)v%_AX&@^2! z%-LBu&ok{*TIQ))YH%T&Pb^(#Nvih2)Rmzga1gx(&5gcHzJ3hj?a9I*h&LFZ#SX~+ zra=F!=@>v7m?IfgKhE4huw0$&yt0$wOJ=o>gp;L;As=7j5v+oWcRq>O-?>I=bj|we zy$@a|DUKd}o^UTNN^4uCp(M5x`rqP2C*wUD3=1s zPjbNwL?+o1{|_v@3K!U_hux(EWp>q2-wiM`A4$GXW%eejIAWIQ|p z*H?86{(+P%LXI+9K5(zVBfo+zM0Gh`olcjr{T)OZ?`LjDl9$jQeD!EOG#1E{*>gndNAEHPJsS-OzVksiu{Q(h=r|a?a#7Sx6Zzf-6gg zFSCY`qWtOCyZ(B;?SJ@s^TcZT{+|RjGP-aCleY$r$)q6&$_~Ft(L+l+`F0!Y1NW!4 zwz+*hGU;q>Y~&$8@^#YhOEp86+91!qL7ypXLmPO?C$W?8ly%$$VWu8N9!6k=C#%Om zEt$mLM)_e!swDAb{pftHI7TVni(630s$)eSr zdOPS@hJg+l)+c>dMa1lA_mzxElin6H6F5|o=fgu3Je7^WgbHUjmfSOY`hW^!tJ$8w z;vo0GomA?-A82^}(*N-F%LV{fE~5->;X!y>V#?T~uF4JKOP7Kgn-oXm>T)~k$9X$_bj^q9N% zde}nXN^B~N8PZDH56E&&72%x!Yv@`q*iD4GWjowj`m!col^0`FDj!-Qy|7D2m`(}t zhk*seynp*WZFG*{_x9K&f=9Jky2eB~X8Ynr4vIW!YbmtTb_@ED`fy%C;zG^PRnq5N z(F~vnk??>mIxC0sDc9SFL$A1^=0CaUkxAz38CP}?#BbIJf&_2mvN<~2oiyG7w{tI< zQF5zOQ@CD4h^x+b@>qFTlLl&;IxtafRbU$eVe6Nx7FpM8>X z{#JyMJykK56jz1%bJK-_+3dkcy)1miaU=`Q^bUvDvf7x)nzj?#V{ zp-oBCCzDa#%983?L`MPtt$i0g38D&CGKuZ9UjIRrt1@d%i$7mnnma66R{;s2rtRq0?nsNJ-(lX2vV{d*)yVq z5!pUh&9}1sOQymz9(kS%YWvnuLG{x9Ty2u$qy2of?atd@6A$TV!z3ZL`PqP#a4zy9 z>eq3Kt0)VUVLO4@Tx;7=sBBj~K-*`WHO0S3Map!^B z$I5LjBKjXIjO=ydpfDEsWX@a@yc1y$GN4e+m-&|~0oNh*L4ZO4tHD*_vyMVnZyShW z9Qx1SvBpv4oGb4#L_#9dz~*u@0OMT_E46~iWR%H>-UDX zi_TKpk~XLyV@udtak|7YiaJm|$z(bb-?G6=B_Hus35Q^i?~6*^;(ia8Y#bO!ypU zG`oEFD)`F@Oo0;-Z8AIBeS3Y8R{AcFc8|Ap)yDLrQPo=cA3v6L$2W$M*Zw(wz6fsV zzo!~ri{pY16#Q8%TwgyS)bBYX>-L;|yBfa={#zqb3|1@Fl%EG!*4D&#;$wg{Zpq}f z`0(d8d~R?$A+hLQ+MgMGVh;(I8UPP!>J8kZ%RMPmo4qmG0DljeHO%;7eTD&n*l$Jf zclh)ii;`EW7$bKVkQ~*MfD6XS}0X4gG`VP*t8D2nKJ^lfU97ZC8-x4so1SBhyd=fo%k30fa&ub^vXQK9T8SB^LhSY zA8aPTy`e_~Xxub@xVdUNW`JHv>p!>sG3|2TnwdF_P}Rcp$bh=--TK$!vQ~#tUJJp? z-@+5yDvmYzC?Tx#$Rz@jo{y<3Rbw(qonDppe}nczBT=AHLT_yawkZYX8)h^`(cGrf zM~azXsuIL+br88U*JyZalcW%o%4Z4W$-e91bI!n0)nW$V1l%(4*ec4v7fQDd+(;qv z-ct~wHW_UFNmze~J~4PlY*@Y@vxFQ$q{Z?;7d|){RyVqJV>RL!p0vza#Zi31%$5GvGQIn>Sr#kY$a$> zcCmt;19&jhjrQZNg$#_1i`*fOCc!_n%)NHj~obfY;?>05?mvAS@c?8F?@s z-o{5WdFz9#$dWlk5p?X0iu_dg)hAvqnWEpK>tymgwbe4C*xpzp(lJV5Z>yeK(hj2# zues123Vz&$jxIs4d#k&c{K$Orlq?h{Vl5RWE6bcf>>87|>tq6fdN@-qcqw|zy~Z!V z@zqdtTbKFrvbHTvFU;r{fI%}D9#aXC{7s)Cj0_%#0L=_I6JApwazjMt^ZPgUaaYmh4o2ur}OPeO0-hY}5G6qZ9& z1=R(*(-vQq;;UIG!y)I@^Oqj%YLV9&!Vt)vPR|x&^jQ`kTaqduz1R2xoh1Lt-|cW3wa+zI7>ad zt-mCMXf}P{0#ERr;B@auB&%b)z}aN{UU?7bd(**EJ-v? zyvJu(kk~F{RM(J-={-P66$};gtq=QDa*Gm^664#mZH$c`x<)stWeSJtjdgSzHobgz zhxi29L;SI3?7a`$AhJktF6KGL_=m3ONYe;6DZqMi21 zyNA{a5M%^w+5zg7l*Q~}#PiR^tk|aI!7^q35df zy!_OSTxeEIV>>FBR%#zGGD<1mF+*?s1X6MUVLc4l6L^N>Pa4OW9ZO8z7cFCYFUJUx))T2`}Iqsz@E~&CrKBqHx+UOrQ zdO}^Lr>kBf0!=c!4A?Acu#$Jh94ZIA?oNlbA$E^*E0tiCkn$2qaSEy1p}HAITg2Sivh98=t$aoLV<4d!Cp zp$T0XX!ny4cZrTP+YU_E0eS&uio$shy~~6E<~*mm{CTM$fT83yy3Ey#Va4c;g%9_4 z;{EOel|~NBvkOfZ+>Eie09Cn@)$(ck3aG;F?(JxkU3Q_B_SW64shOvg-E2fi`}FuU zbFhFS=isAvx1)yZiwaTY4&4RG@7vmWW2s>Wow@B3G`I$cPSjoIPhp|)e;_Uql!=cR4E$Imsk%+~W9 z9NKIrzd=U97Ifwb;&w*me zQXM`AuMEgjJ%4QX)ZA3}9*N8G1&=QLLS!vO<6L~vDg;wpjZ4HE3@&3-Qk10YG|7EK zpUE4bSB`Pfe=?c=hAMd^IV#$u{8Riv{pZU*tS*}Qv1}?1D8h0ePE<{5CuarPlM5(yuNWqi_Z5_@$cSa;688wdaz<|8Vq|IOo$iSWs^I5Y!i2? zl+0fJQvajVL+yA=Qm3nutga4FK(HCviz)}1Y>YIeI8NtlM;3#B)}msvU$#9(KG-nw?RLGdJ#mI7lv?!AC*~0rV<&C zz78aRtU7)6tX{SGr&DzfT8jSrzh=uiy&^b=@y|^YJQVg07DC>ujYZ}WykPunQvxo1 z0--xiMvT!X1NliS`L_baq1rNs4Cl2jWX;z|^S@UD!r8{Sv++ z;8CUH=OV{P?ryE+fgjYi*qxq#Z&9U?l*R5A#%35cGeVu z0`xSD+ziIR)It+40WE{mU6fE@1t7Gi zMxYFuEsn{Q;)Lt1SwZ$>;L0q;8zJS8svD67u(CO;ygRYuldL_Te zWEVK#cSy-r^7q03_#)v}Ob@^|D}~J7L~M58Ci25AVC0`f3RVa>Uk)iwYPd1q-Muka z&n9^opXyOK`$Q*pv~GP_A|yo+DqGzKfn2mqUj(*_$Fd}iFX)3)x?ukU#xM9zW8eFQ z1Soej#Mp*mxH3xK#1;=R{%#ztv_5pTyWZ*pUnHf=W~6z(O4tUaj)6ZTmH%!C^xqVq z`_Drla#}TN+>39uVIjO31B|J)6L>BZwi%+qs{?oQ{|F57;cR6czU*Ue*r{jM9j?Ci zjkWyuxF_#sJRN+0!h6UE4Fv2?!mm=n+OSQa0Z)U>8iZb%5ACLve6buuz5y7v1=f_~ z7@W}1SiFs#;}f#PLIFF4rqTmaOY=|J>9+*&-^GasxCb76e=gy;U3%a;Pa(k?<6&f0 zlg|%F(i`T~w?{~yWCQ*x0bdI%ND=_agGyl_S1% z{*rLU+a0mq{J!9&py0BTXB8ke)^WJvw%F&g6!m%u)!PE1Tb?O))21Ga>A@L0sZv5+TMFdO}Jjbx2V>SKmV)M`GYG{7JZC;yV7M* z+2NYxpM;f%;JV%;CCJg9^@l091}#!J7TGwG;K5+)zd9pv99AE#U*W%pJx z1WkXDMG9R&PTsO9A3DqzI=Bz0K1^H*qb-o}4;OzxAAb+#bp=@bC9^8AMNEnPc#*5% zR+7jpswD}+y4Pw*%?J?4t!?5RX=1@JBpILe3TkT~nNO?BsV%cvN^jbP-=wTy`-Gd! zakp;tL~$iEo`A2KUzC7JC=iDn@+G;nF(6QSRP?CbiS z5)Hp1v$rrWx69TCig!_?f)~_}2kjQmEy_t5O4hv_ySKKf;99+FZzd^8`HonWf9f}S zHiLCrgUGHdGi?39n!FI`wO=tK?FepIXJBZcqnesB>q)9>bfw=xZJDpGQ{YNVlNqc; z*hJQWf1PQa9pdS_bCu~6;7>y%p`{g^+ASue&-XB+G|bqP5{6`7L2-PCy4&!l;n^lT z_onFWR9X4|Q1|BXQ10*lxFS1~J?m7IY-u4a7%C)3(nhJKO|~SJQps&evPYs+hA5RZ zDRL6-WJ{X}6Jo|tWXX&yH*?Q@zSpR8-sgSJ`TQQ=_v8Ei{PFwaoa(;sx$oC&PSI1$&64d*zG!zV_&Ch!8YGrUFJcP+YgQJNhvzAs1q6ij zJ~D10f?!6kGRo~I^Rgyfdi)|OtVAZqks^kLy{@vHg&B9o7};&Oy3qH*+{YWa4(f*m z=A5e1eRf%TfK$D==I7zs zX#x=_TWoVi{{aENVxJnWA7=3OBNb@^i|3EcGayXaj>mie~Id8{{vj)ys8~bYBxBY>);)Z7BW`*6)`4}Efy|(FT zVocGdqfW=HWTszaKchr0ODT>_7+EFND8I%s(^ox9`usM){uB)6V8V1bw3@Kr;~nDl z^uvUB;5Pe%`9&x9A9!k5uc!09QBW13N{Rdx^00YpYWN~X=!iFi_XfAD!)Nax^TP+x zkxh&*JuxUN0U8_ns=r7}&%cGsa|%?JrCf59jyV8G9$g1*#mtpv%TYh?g>*m6P#0J1 zx8?4>o8?7QH{U!X?^ZXbLcU|smUuJ$dlq4W&PIhYN`x-LnV@3&6ut^s3AW0|`A?k8 z|EF9i^tfPqT#7{jWCgy6z+61q)#Vf`#7h7gSEMPwPeCDVllYO*zXonv-;SyU{heXyF_Hc!F+iKZvMph9JY+AbGG*7k83*(3JD5$?<7lN1z68i4&A4m=YwUpa;L6 zT1n=qno{`&DQJiMKi>#3UkNGV5_>@1pbe3FsNcqUMxez%fkp+-)er(OHP7jg`#9?n7n@FQ=mrqZlK zpKDi+Nbl}jnLKnXz~3k>HO0&A*fWh`*Xsg{%|T-N)*dVZDHa>hLb@c&6!sGlMzDzX z+H@LNRBve@KKx-=+sYHw$Q<=6cCd=*Ct;?57+YC2a z&umS+89)6FOW+ed+-iVt5E!f|XG^R`&=0)_dEkW0GiuTo3yQEshA>mEFk3WpSzexx zmcoaHbmN8U#~;6)_h{*}q7(bmRPc8lR zSP-_R6BQ3ktJ_xJ8M*0$ud@2Nc%MRh#pSx^H(Ara+a7+JAp%<=JSh!2K+Lfvq1L{2i(>Mop5z^}T0?@#^kn&wPBjd{*>_ zv)x5)lz#9Vl1}-4Z&85kxnt*r_%^{oDRtJ-LV~sl z;a;H>&JN@7<$x&hn#rG&j($9JoX*(}gq+JR8u-pt-+(=XU{?vJCaU1P$K%HPI>QfU zFT6D?YiyotLMHt^_0N;n61YmgjziYX%pO9cmfH`<=A{H>(( zrsq^ci(HxWf)Yzp3vy;VN%uQn@GEHRuTJV`ApnVi)_6AqEjsXR4bbVQjxmPF_J0&%J!uXb93@csA! z{*_UP6kHYx29{cW1UhOcFg4JD9oHb~bUpYlCIRH1ge@dfIQGnV>X^}6h{NAPqWOS_ zhLn@b!NZjhVg-a&{(BXPz@}y;H=BJudd(R$f z16;8S-{RM`J=TLvfBQOIg-l2eY(P3Q%{^#5aWf41mIL=H--CWuW|Hmr_G&0`$rOpQ zKvu(HbgIMj0Xxii463p%m4n~X3Xm#*)2e&EdpXzWd>p-Z$BmY>x{VKulkF@DD-Z1L zN|cb!0Sh4(=u;CZpPPWUF~cdG-BbO&qgh{)?`ptscS5wz7?)!JP#gyx6T_ZyNuA@* z>m^g6RX2Wa?c8M>RGnrsCN#BN0QM9-h;76s2$T>taSCAgYSYI> z=p0bZ!`uY>riIO-^aa*aMjow%?A!H17=@6G5#;D$U0}>@D1;Mv)CEI6LGD5&Z3vFY^>8DQG%|jyjb$F0HF+m&*v3$Z0nFhsz?bt@7?|0psJuAxYA)RBu z$%!$yV^QT@v5FTeNncxUUtPJgb(dA!f%Bd(4;;6+y6&avBq+J0Y8c!zS+PM0D4d%m zSZtoNGnM`9_d7`a3qgUks0^WqRB%z&1Lnw)^rf&J;esG-p^_3|fO2p@%%F&`p$m)3 z*~9F#!zt>3`uCfV-M?O{6x@gm`<7XdNB>$JF?lKvUgrb$k(N&VgAxmRP!KnwJ#NHl zMx{J8-(LHolKb9xYcWyOxTQz~n!gPKkPdzwsrU{q0IgKW0rGd@y{IAbg2T|~;cJ^m zon-pP(ul}atxVT9smkANk8ocrh=RtSD`fX8{R0=a0m5Ek#i60R)q$^tiJLI9X+-03 z7wiGp8h@D~`L9!h=twR%7jmPFgpfDNQ%Ox7XnDdGh(X+_;|CyoaYla+p#B&c<4PXY zX6mNh#4wq-|SrY_}P;be%gwRboY4IPx~()- zm|!)ZlG}deYk$(ZTFRNH!1~?)Gf8m;a|1`z|3^{9gin_fHa8WQKUq;mYARWiiBIR8 z%^QulKCb+|X3stDYsEvoeLrF(K6qKlURbk-Fuq5gT1HJKM(SPTu}1d?9`uBX???9} zR%yiEiC+91>N6o`GLXd;QD7Mzl64w8TSr8D1LgQZEd|AgHa7Dq*!+3aI~{N^!al;5 zZx0w!{FS;(KOUth5aMTG2jile*Fj*TPE>TYxVF=7 z@fD+{lQFlyHm|h2;g{{@Z>ZdypS`F$Trv^5ri$$m652SUb-~RL?aLvyV$jmwGWbIz zr68da7H9MPT%hWs4Bm?+hw*&s6*UGf=RinUur2ltRjTzoHx_H(&HQCtMZtd0t?ra< zhTm#V&byLywpSS6bh;ri8u&9r0>T8$?S(x0m>E8;yoYCjVD~^S)wPP*H9*Q0wB-|x zn!cmOwZAp4v}AFtj=#+@NVghLvYH-ZesjkBEtEll+CR#Jz!#|fQh%uZ^7fBVrvfL; z$OBgjuQ+W%j~pQaXd3feWQw_5!{ba<07e}G0>AorWl ztZ~*mS)^MF?V@&{gflc>;Ci1$Fv%)qTDawh%mx!j^(wW9ytvDc2BO8aFNfKwjcwJK zW<{PUs1V4?q)$a}%&!A28wZOZA5+0&Wf(xrtAIQ2VT;XUkw4uyK5f$Jf29QZzu8mg ze^R)ah-Y0(49j@%80Vc|IJJNlF@RG$W1`qkFKs|KaSGlq>-`jB{VLtgbYcj+*CH_c z<+O>SAt{6zHC3xXLvXdt;7W}H`z<7Z#@96^9<9lszZ+_4BP3S9@%+k9Dx2Si@8@5) zlC;vu? z!>LYV#gBc(Yn3PCVrHZ*?|HU1(Rz=q>IP_w>`UPQbP{F=vHEKvTqiyO)-3>nPAVeh z!03h}nChTe`GC68&`2NNV#dBq!lt_z%evqI%jBr3V*6_pJZ`8Iu94+^tlvu zXfQMnJWgjh>+p(X~ELLjTFvSN+fLt zK?ywZp~_OZNxN~!kYnkl_pkO9q^$ikP3-d?@t#8D_9ED@=}5T-rI#s4!wNny)vpuZ zzkoh63AcuzR)@m6iaZ8~eg&Ieoe)YBGa<dxb}@5#qp0^^RatklW{2 zM1(;i)K{hLCcL3)LNevaQf)0N7P*^KmhQQ0VU$cszubgHO}CzArek*;iV#k5Y= z=c2j~PwL_hJ@(=C%V)h@t~3!sVTI5*b~C{jAEP!tEnWZe+|y~vvGE4@jcbSN!2kmv zYM43r_%{(ka2S3%-+f%L+4xc{u`3~*u&gH*;7c1n*O#ox*i~~jhiT<+R9b0Nykh3X zQ=4DX8!O;|@ZlvjJ;Vm#D)RUwI@IAU?uPEIx+mB2Q8EP+E1js`p5e`A+DEIAjdqcU?*VE5|4R}V#8 z`mQ165bkE>w{(q#a^dhjsRtdT7$)pW5tu1i3I9rU>YVD*NR~&+kgN125-d;o5d>Ny zCB+?vX?+mR1;ymm)CO!BYv6kT0!uO1RI=&qxN-%l*kgk06PLNopNkv4~CUbdTD-G8-7oP`7=vlx>@ZGuApql|a*Q#Tvn zcOJS-9)98EG!6>HfdzC3k!2#Ra}>_H1Swpcn;K+-n1h~qtU6c;lk3dYbXxb}8~xC?xe*l={vwV#oAj9DiH&mK z4DbfJATO2Un(J2U>kC{ebAMdR7Ktnt03bC@g0W-jOzpbH0mtkIpra6mr9S3i(^O`G zrJr#t-SJ8_%PXsu3-!!;6Dwz~GbJipgA$CAmr&+G0?xN}iqmQZUa}-#Hkl_1hqE`8 z&N(V5@SP{M8(2UbgNTFEd?s#nHDk-8YmR8F!;DH_#S5pGET5Cio9RzYGDA7mBuu+G z>(sK$vmB`x!%nLTl2)AmSQ6kl<+E{%&Nw@2ec5#EV8?KEv(JTj>kddKRYugOj^EfCG8cWjvyOhCf_1~1>$aAL&1&`>%l3cQcaL)NJv2KcjV-;2 zR_})`3Bv_LoZmS6JT-w>px-cC`(46C(9UlM5YQsU)MCvJY*%NV$F|G2i=Q>z|B`0= zIo*D??TO`GI^b|W`Dd3orGNp(W+mwlm$}+cm-%0AgqW|6&KI(E-}bEn-0(AqIvQs+ z^g^_Ywww{Tsxo~xCewK>x1X=}ip};ozq-L*NAgV#sVPba_IL~=_t(El?plH5MmnDZ zGgkObXxU$QamxonLkxH^u)M6_M zdq(L^3BZDWa6G9JxoqHg2ynd9yCXAq*zDiyA(DTARTR3~PAx3*^Hw2QPf+^>9ghCi zO=cVzz^Czv_(k#+niums5*M_Ma7W{(8`s((FI+=Q$g*k3)Vl%D^f z->iPsP9}9{wU*as_dQb2G}0l?lXzB%c!zRWkTjRTS09f+hbo~$4y|n?Bm*FbE=Zhb zWs+4bRf3j&m{VUu$p|dkod029&#d^44%L?ZabmiXqGBe~I2pY@`~rlPKpl=@hLXqT zd66d_G?AzyaQQJX-UL|MNNNy7aq^9UC>vaRg-1Y(`0sba{BJgvnGORKO)0NhiAul} zx=5{P7h?bfugGf#u_E|w>qxpf*XDNKZ8w{<^0~z|=A8#NYRG+;UFkCMk&LY{Af%Sk zCJY&1&67ZhsO}Z!eFBec{szFNO$Tl4;kyVBwzbsmTToP1NWzsP2}wKN!60%eM&j88 z*+1jzMg&M?{+eFxx!2~D(IrY%8c>^w`=jH0p#=N9c>*J$KMiUhq{XYSJHgs;5}{Qn9R zq`XBSA7+dxDD3@s6PTbTf<*~N4|qag-2r&yAd0no8=MF424)?0gRU6u9=VbR~j+Ra6`qHG-$Gpyq)gIa@&!59RrIvU!y zAb4hYmp$ZSOF8iuhM_gyD6m;iKWrg5Q-{XxV%GPA#33lK^7}5%gOV7f(b$X6bM>?C zM@y^~*_*5-6a34icb>xI9)!$2F@kKvp%SPuG7^0n(pc)LtU;c6a6MJ!Hz9YYm`S^u zgpkR~G*Wbd$|*y?tgR=ave&53)KFTnO*^pzb^YfNk~v4|c;=Zd3jY+0>+2yUV%6y6 zTOP0wflC?^PYS8!{QWGmnuI7u+IU1j_u8`jq*+StuAkOrBwJNTXK;LmJ+EJXKV4u+ zXDHYMRdH;nA*dwcHH7Jnzo3MY)eEv2=0T~s{zS&b(+sX0ur*L_$`6X+&41n{ zAVxq<0U8T!P?$jPhw7e zPV}s0%s4w#zQ<9jo{Xu4CU}RO=Hb5KBpL6iZuf+g1Y%4CE z9dE#QbHdgAVU!2!@WeO@pt>zY1h@t955)53A^9%&nyH)rwuM>5oL^y1OOg6nA7Fv! z_WZD(`-364Rz>B?KqsC*FHiz@zKall3xKHmouFhSh9>CO)Rh<7PaygfHJAwP?+@7h z(|5r|be>@N1*b}++ZwAgm_zKAxwm*MG}`vk#icK zUO`y^bts=rmF*>rIiuqdL2JaHK5X2R)XJX!PD^h~?%itAY&NPu!qulpTvwO%CxWQi zmMYA^iLFEx<^OIqx=-;|AhqR~3Q;ZK!t1l><018slb`^^ymPalwir`_M0y!6EV~_U zjQP`X2aef`Rq8T66|DiyIXR#1kI%>~E{M8)-b(tMe0;{RI2)}?`%cJO!7pk*w!il9 zEQ%y^gpge5eaAm;xl99B=X!TNh=GHUJ?w%l=|hJVpF)in9YRuS`~i+G zuCWTv$!YF*+=$=wKstMLv^h18>gKxJ>b+a2QR$mp#XIX_XR6K$4^Q{uOAG=c1jr}t zT}}60Hsh)vTDomcP&}V@KdhznTR`?$Oquw8LdEVJNbf7Z7azazLVgw(12c%m6}dyW z`fJ$4^Bl3zLb@g-O3sM5oqN-DC$mMciAq%kj)ohiT@2gWs1*paJUB$if?Fu;)sCID zK4R5woWxag6CSr6QDl)ugOjHbm5MG{KLCu{Q#spdlYbJQzX@GP0`LRM$aa`4M+?R+ z@HgQ>@Uld)oghzt_r51w*iRS?0?%w*4KK1S3~mqfA!HZ9H!HBz%F{@sDJ|CQ`}RJS z#cocjMMV`g2Im6U=EK<_`BwYMBIU(YjvY-NJ6%zEx~{xUf^I71`M~31oquY|7TE#h zRC|;q3=?dWxLI0g&oF&V4@L$sFR{!KTN7ZS;;)f^W16x2>&mGh_dfo_L>it;!j|y}afa`boohHAPA+lbdf&eV&hAEDujd!moA^>H ziY;!3X4}pw$Una-P@7z#v0h0fJy1we=o%2-Mc*3!QACi4vn1#+m}!v0&y%5oH2LK+ z@%=On4+l7_OJK&#N6RmDoy1cR738D0fJd!AkU^p}pxt_)sqBIm0bbL0+KfQVtaA<^ zkAT--Hy$nDr?*>TnW^Xq#-ItMOB)Y98)SD6 zhQG6L@;SolrelWXBIH;G{+rN;Qo%3im2~;Qm~Gs{=vj7L@Mep#TmD8(Z*qi69xpz* zPfR0!e#nvxIFjdRZ1Ov_y?<<19lrEr2s$tgyPsf73~j zzf-s{KUz;1I+)cQHpW)y5>ksA^l6~`=pe6uOm_4#KOxA6O^qTyLMxc$z*pTJeXZ|8 z_n$hoN=W*WqvYd!VW9&_ff=N%gq;iwfdpR4XC1O8G7!js_$%1aN3t(;P+k z3%n;$V(xk%8nC$P55nu7pYen8@^;841S$}mlQ7@lHrgY<3KSYBUBi65*vJK{yotK; zOxWf$uOx?d+p&2m>=Rp#WIRh+2UHK$6mn30?E%`vPv|3YG2&dQHVEPU^b~}x4n-~$ zzT6+8GPn}g#Wu4jpo(e+Gj@l6sZ5w?rHX$Go?D*r&-|GMI z%@npe8PoRT%khkPd=TwQp!z%m1Q!y%W&TS)b&3-y!Zd<;XA-E z40QUl99;d(bvo!g zuU`{$VMD@dWzkv2$WSE(cN9V%0^atj9zqth&#>APSos+Uo&XuLy(nxj8Zg6DzOodP z$0b3n131IMH2)LqDA^U93jjN+L;qSgwisT}1XW8E-=Wru&YppAo&#?4l&f1Y6s0zr z8i}vD^;*$0H7#J|ron~n$)eNhFT7kq|ArEC9>Wip<=5bP{c;JD0wk^fi44GK`p?MV zpLR9{vJ(EDLi2?-_7H|aM#`5`4j8wj)-Vf=2Y~;!GOW0D744#+BB{yVQhcYJ!q@SV#P8hR@6e0*nBv&2b5aus--$$lTv1a5@a%sr~v+C z68w_JIl{!1P`25i6Q}>V)*_64>Qr|BZtV|;=(9cdO<$+`bZw%9RQi@gYQL_6vXzs$ zq)^fnJ05mQDAR7j$s9@*VZg!5Y{sr-b$n`jxkAry<1=j%R65#yK+0=rob5_S#o*I9 zgex$G%J*VCS@3&9a#6Q{h7yoi6?B|UCg!JsUhy}o>Oc8E)B}Sk-<1t+`M6+4U^pQw zpfD86u`cDjd&)gL>|lGj8jWnGp6yboF-v1LdXMJH6R4dC5&Nv=VyXp(BOhC^`;HWfz|hbj`xo- z;ZTIfL)v=q?1qISnr`gR-A{@h*7_}(wYS1IKJ|!mAeVj;WUXscBGF+=FETb$;I_}D4lU*Ih_yx}qI^tHsmlc>E>i|8xXLT_ zecv{!`27Z-Q_8L%9nGZAEux%hbonMrbN9Xdu_9-?ZmLYNq5qv;B_T-)^rKRls@1#y z*87z2@iVzsHbB#}L_ArI0)2b|poT};5AAps~&7-%8$XjRNHHPA~Mt{RMK0y7k-oI&{oe%#=Z^CEKH z74Jt0HQch-e{!pC+_M;HPr=`t}p2;qE5WP!iH(wIY!?40TPwYchYVab|{i8f31jafp9 z)4CM}4R5x*zvHp-l&TW(i1r9T_u%x4Bw#VTg+VJJhDa!GZ|^Ivi!^C#t9{cDZuq@T z`(=$x)72<&5O~G#z4kdF|-h5Ob5K*zZ4QW zmx(rnvyL=jQ}f)ICvSrA=dTyyYyOO$3;q7*e+^gje*-w5^o2pl0)I(f7}!HptW+UV z?Wo;Z)c#$-;P)f1RLqY7FS{WHiJw{YVdo4nedVwA1(S@y99*5rn}dt;pHf)2%T)O3 z6fcW8g>{nNI}YX^^Y?sge|U$3&$cA(FKb_Q`ABn4(MP~Qzu*Ro7(*R#=z&cs6xgMj z86*~PiKBpNb&yaEhps)Q#(bS5lyUY9--p{ZwiR|%1{x$&_A%p%se>{+^FY~MDpjYK zyn2%AduP|XnZKMj<9~Xztpm{iVIdMgjIqMev+XR5e?xt$)BxkV2dBX!92wa;jg<2m%*-NmwH?1=Np>%!9rr`bP{VOZgWltU?;c*%;IF7~IRE zezY5V&duJfgSm4ln-kV@m9L~IKgkPlUo@{vVU60$tFw>Zmq88yu^xdp8X@%yX~0w& zAS9tn(@P((YR}ns%jqMaLWSK%%AbKIx1QVr*7kR1W%D@AQAS^AM_Vs< z-&}DyS@h%Z<-?`o^zVS^RL{h8#PC~WmI~PH6e9T+@}wP!(#7S4iA@O{DagGhgA->H z+IBcZ%~e+Kc~2S!7=sqvV>O5nx`!DCdiD8&QOyKseeQ}o$ z$-R$tnK(tlRyq(;>z4!iL8c!J@)}9d_Nu?e2D#XS+sp%<@fH|AQ@3JBgsI&>T!Z3( zE_lwi249uXl{Sfy;V3k$@WCygq!BlfgT|OKRFg}MHS%wk$nms;-xC&*+d1UQyu+_6 zf@eQ;T-hM{^h1DNO2?NuhR%_zxzA)?Swoe-55kp#LVq73>Nx0W%TVyHHA3Y!!g0tz zI=g7X=)j43WGcPfkGDytAaDQ*B0E!GpxCnSgoFxbKYK$2*ROM^BI^DqQ-*1M=i`Qe zRSdaLfE2wRCq$7pRs#4oArB7Sr$P2hh%Z}RDvu1|1JX!6+@aKcM17>LjPhZDhm@@Mce)&T5I z_=F2aCCZWEGZYDQz_)=i@&IJeadAO8DWWfw5>;-%Dp-|xQ%1p}SYNB2qs8|d&Nh;^LeJWGh$twkCAAHeGgiIp4fp1SFMVzN4|V^>x7T@$Ae zs2+A-ZhpijnD-Us94F(-e)b>oMxK$Q-Wx8f&vvGdIYzz6kj$FDFi>$CQKslJA&XRQ zqH=anC-s$K%NKwh)V1XwE=?m!j{gS_KMkI{oWmdIpLq?A>oj2I#_b)G8xGX}&b#=Wn)^^?ubO(4vYrZWex>`)5ZkreyS1A5?;z$X1M44Vfv@8n zOynv+1nS_^row)Mv$kV_wQIJ3Gf+PF4z({OjFPQ6JNhP*yS9{C8?}!(m=Dj` zzWTve&3GXp4{)B=Zd01?uEj<>NMjxpZv1W`^5{Y@>ZH9=@In%13WfVm#I-wvaHMg^ z)Jiahaw1Ahvjfd4b|B}1DCA3fZ?)NY_-Z*EIxiv~b1CUahIE=x%hkw~*8@KI96{mo z3JMwdh-?JD6G`LL302Lpwmr3wMVMk6;t-vP?Fpb@hEhUXTjUO|;cyg*~2 z$Ofs3wH%L=s$3ZtcYR*W#1ZQwU)8?tvwsn?H(XXwAU0myWMQ^_M{q^41R6ngp(gIoH#Wa zk!*oHee<)P6QH=Dppzb5L)?yo0j}2yv9#G@xI}O9(0WMGy-j}TfvcYa$sCJ);3);^ zLrsrDXCB@fhLHYqv{ywwQkBeM;||EAlb~D{a^HToa+1_- z5UdsU(6EesR%}=aC0SdktjcrRcydSj6l5d-PEP$v`LANsPmjhytcIUMeQB*?gv10a zE2|0hq_@yEg|Fv2-Y&Y;V&{MCQ0lrl#aFV!U#A)*t<-A5C6P)O;0@b>8ng?A zNVyrXpzM`z>Cu*oSbD?wCiKGyZ(TucN0-C7_8pHtwavJ^`c=k7k_AbL5A&&9K;_uL z;N$`jT(bk=9-oFgz+t%!+23hoFxNh+fQ`_gOCQapSCc<732}9p_eXwM*{YAh^}h){ zJ{b`pV_bFGy|k*Zw5s~nu>iH20M%&c&+Dq}^__HY1;2~!N)y^4*bswZ2!_yjJA!-! zI$2t}IX@fmc!nLXQ|fivW3+JLY=?YV%k`e9mzn0a77oVD_K zpo+;^k=xC1O*x<;H!KI-I40;6ii|U0=ks)7HIBi9TaiO_3^?+jS7#kIAAp*5cRuEt zy`X8_dXrX0+v_fS!~Ax&a&frt+ei1Kp13w%*YXceSA z#OFCv`+b3hWFqA%5OcGcx(COx3T8hO{50jWC_gR+2rj}8bn2^L!b$~J z0iee&Y$^cVslPa#a)XV^w7EI#_qWUq@K-ZQUA(p>+E%~CMLP|Y57Bo^tAv(Jxybm7 zVLIl)Zvmge2ZgG>7qpg|u88Z!kr6?g?^!&^^b^SwF4-q>V4C}C)nIpxaOzi4l;cgt zNMczDY+;w;MXGsbUI0Qd1cEa#6VTI=z}{4@%y z-58r!G?2ip0B%s=R8AK6-lnC;seJ37Q@1QPV5LrMoT`y}{u}v2 z0E%J=loJhd*aqNbRDMq>hy9z%DR1r>fX$lm9u^n`V>#$~UI}8s<+Q?@s1d$LXHg|@ zq#l~yc-?Mmd24z-w(_ z#6n1=fvRjE@xNt{gEGUpM92UIhuH_oVp*L-v4||La?j1DL#5GZ)WXuGd13dKr%(KK z4mn>rKP~x4peY}Mc?;;tf=)!yyBA?|cu+ZvRd6N`JYfVa?u>KV^e|UcjnY5?`eCJxFF`B|ow`Xeyw!U~7 zy}k%wzq^3YSpAU7h9BAjY>dT1DwFC!V+a8)KP0%FIqZ%qp#xi*(2*1%8Yv!%O{Vh4 zV0NN7u=#L!l0`=pgT(IRX556h<5x%WPQUVzdilY&z}8cDjYi0lJNoPMyIsH=cU1>6 zlI5MctfX(2QdxEp#}N(On?nmXU!$5*v9;;`;6gdqyPmhNIzQk0Qb*w>ebf!=wSkVq z7OgZOPxr<{e-m2#td7tJ{cc$_pvyYNF*!bV;2bWeh~m4tjD*Y83Qo3eh(S&rdV9ZP z(e{j@q_sNA<~+G0wZ%Z+ulU1venE>t3(pj=56;AX_R zT~yFsp+&?l^1ol1Dt%7c3Xd|mk^;u}r$<5YiGbI`FEdqK;EcBqO~rq@FGCybqQBnj zfry*^UwOQM|8M-k|3PK^JO2kC#J_V;f$${kO0g8zomDSbBoS0>Z^s^~^aTi`q5DC&m3|P=$Z91s4la87~ zr{f?sP0^G(F#~K`2C(uU&9mwz49g(O_^=ry*DVrt$ll!E+*mqXpLKUq%V46JL18rw zVcL;Vg|C@%eS}z3`q0L7hpSh7TtZamA04og7OK^aZ=6QhE&5FeDq`v5=lEAC-EC!L zP!x&)&q!x0wv9O@gd(-IW;9{H3EdH(nX6Fib(12Ey@2($ST~>%s1lhv*gI}^PGL>-TKJLiP2@BX>=zDtD zt-n^m^_h*=?S0=?8-!%!o69Z`#v^LhP`^n5$rXW{)$d2*IH+fkI6hne^mhm0*veE8 z3;v*GgiI~)z%@WknPJY447SAT##ZptgSOxPnl6G-@9z%uuqybqA@8tz_=XSje9m}w zcqx{|=4Z_Km{bGxGsrzhC4hM4DuHK886hI`V9p{b2yR3E`z0YHFRIS~UyDMK&TtYv zo;o@w-mXbXwqSEz{`|Yz?+?zb%CmxfNJ5u6;cAgE zx;-`cg)|iUl~J%_J{QyG9DB}u%zR-i3!-KJ;jwg0&TX3L)f{V`ID;DB!mOelh6?pKP^u?=d( z7==|v<@nVBo;e}Gh4mb42DOFM(pTdeUioC@?HgVS9lNixhrb>CIN=!cZlwl;v$5S% zcoMYv@D0{v&JGF`;{LA8-=}k4fEMnH#32Ag(W?9B@*njaLzn;}oKZ6LCI|lR(_j`3anYc2ESOH$e zI(P~$XV4-M$Q{ByL+)d#65&kdLh`Os9T64`VYzZRT!Iw4e|WV35J+qd4Ql{nbMosg zNb@>wO014hw~#?RsSo{drQESXSf_sCL$MgHuxf1J=?A){Y9RM8d26Ko6P{+9x_DZj z=B4w#zb2X1iOof`{6P_mgHO&ydsW!RxGGIhf(JDcYJJP#DnRM?l&qvr0qnmi?Z?Xb zB`E6>`H>4gGiz)w19-q~%EQYKvOKtE&uzs=o)-FCI^}tj*jjKcNPUahx9&s7%D})3@XC@^fR-aRm3TdR;wJh^n-JvdhCKx~>{J2D02x0Dku@4230JKtvhab2 zf|?hi2W5S>gvx?H15kGDeNG=h$KWu9y$o(4bKFR+$0{fsh&Z<$>~|~yNA?nX-@+>e z&!~@f6`ZKIcoV#{{Mz2{b9Zu%8cePub3DNk*Cj%O=fXsR>kIPeH$I2N@?JmKBd|Hb zVQl~$D&H7a2xn6U7jtYY+#)LpwI@$fYQNFn)Qcv4S$^&4A=QbUenl%_Rx&^KKzjoQC`U)vfsz!l(6K4zQ_Yu3odxwbY7-L+ z0bYL+{4GxbkK@MZzb5{hkPBqk-K4M~588mn7r|G-Y=QF#T0dQYRbxFsMiijGrqu6i zDhlZdkRYG~i2zJY&u$RZ_!)ATJI^d=FIqp3!-x)$@TDhpJh#)cDN3;}9KWvla+S-Z zVZsg}9M&N0O9IxIP($;RY)4G#T+fdk=g|)Bv5R~fK@M&r3kNy!EP}SidqINKYu$&a z!^z=p&jZz~%VbHK{?xhze6<;zKlo$@^#aN{hzPovB+pj8fCZp6V36J=yMUxQLFX=@ zkKe#XIWFbY0>5&a0xpilC)H+PMUFHF)Rp@BV%w)oJ8^t)xw(eR_h+Eg!~xdAj}xIr z7MzG4_>RbR5MU_+jUvASh^+nvq!X^7tS173#bHi#D#wA|0&-#~(sCJK0R;4pqEgI? zJ0A;LqXH{A%eoEATwm55VvKANT_B{Zc(crv|Er~8;A58Ez$q@J%Pv&Sq`AbPGtOa4 zzE^6{K6Z2a=h#Qx2M#+!NyVFAk@8Ofu(W6~Kj$YafnWr1m>D2QlaB@gq5jcx%j$4q zS-ma-V_iyZPI&HzEeG(`6}%Hc-W>YCer_wz=2|(TSm;2N_DO7Yz2M;^v%#k5S2_EV zfn(~|m&wW?o9kgV*<=nVoZGV-AfxE{elP@_pFeC_3?XrYrxUc0D>i7&Nf7L=U5KGBqaf$kM(Kxn?a`{4OXin9 zJ$dIPbPCi<0M>96SVPwSS&jIr2=KdH+%pI^BBa(XrudO3mL$Yb#`L^Fb+ey`M2s-c zmfc3PouZ)s8L&$yc1^|GW1R^xrkc|V(3!4vIqerlSP%C8x<)<17uURh#Al-DM2Cac z?GT~t*ao@q=<$RWpH8UMA`FOz-6EJfyXwRTQ~rqxr((G$j76U|B$hSzht zcTbUWqcV{|Z5OPGWns44(eK5HW8Pyj%Wm#oeC*(P&zJ`=`vouS=AxH?#fJYXIK5uPX)0$ltgzow|AWbyQm2ZJtxcb1mYF=4Wh8E44UL!*w*j^Fa6p2T zS8?eV^S0b|2ZN+#>eTXU*q*`^HW|OLqlX{<4ISAGxkk7(q~P;F7(a}t`%(XU!^QDQ zv&rkY`QPWO-$_@g5%ambIef$vf8R&rcmV)|gKOir0H3$UmGgb>?)Ip@ZS`rum38Xu z5-2&yHeU(iR$@j1?BP0~et{7~6TU#15kyXy{Ar^gD_ocyF91$(Soi4Kw#NiLDX zS0x&2)h`|<57#xi3~y)Cv6YVrNjm3Qm0D^AMcR8edDqg4qRn0>_Bom?yrX0G1!&^;-u?#~#(${1+}*Q0$7Nz|4z5`%&}_UE5ijiI9jK?+GYaWhmx)_2v(5Jc zyI4KvaGwWCX2oT3`_CNNxc8SL<*(YT6Y5u38^vzRD;{_+>!oq=T>K04wM&;Bt_kO8 z#vTr(+8ElquWc-`G%ScaIQi<6WtxYlCA-=Sj*~b!;`+(E9%tMMLRA&W^xjuwX>GU z7*}o=)+XkAKyC$N_%atNHx9z7x=OsFD4MU~EeTqHzVG7HR1DhoR7|el>BY9+R=43| zo0ZPlu45wBm*O3Tkh@oxt7ZU|+u8|uCvbwDtu~wQVkeU*UuJ zNS_|$#P!kk2J|fVvnTfNx^pmH>7;%M>i}ngFf(BXB+G+!qi+R+xrjsfIll{L&`=8? zW&ZDcCf;Fn0&WIhFe+PxyK!}^=~`aJ-}`s#Hy$>1_!8r+r}}GgV)n(G1FLMmEC<+K zJU*nh3c}4z{lpxQmti2B-W&W>*U>lhF*dfjlM+gOqFXr>Mu==zumPChLZgLM2YrX7 z&bUR!FM1n#snFI@D5zj1DEo(puz6xo?~D8kt$(fc^D4v!6+u|N2sD~`PC6H?f$4If zgya3@wVcv(D+7|==0sjxGd;rbZOH-FQGbZ=U$Q}x|3UBm2ZehY@t~zg@gVw2mzetn zhX3>;!uI$fZWp2rbV8yB`UWx!kO3Rcvo3n&`KVE*a^Nwi#-TKk;jX2htdiDkJ@x$4 z22oGpfmc5REmHxP7hE$uW*i-T;q=+9@J|7pkR|v3h|{<#Q~fQ&CnTAX85vSHnse?z z*yfKyIzcfV`M3Wj)INl4Ho*)hiFT?6!ny%H3ivK#$$Xkt14Vv&{|@(9-!tOr#`@oO zCAq7|c4g8#jfuH%p2G?O6sHFgWZX1`5GrV9Cvx!2nkw>`W<5G`6Ro{7YRb#v1t8@w zkfL#1(QGWZlz)xlseVJvNrt(zt^L)L6ZX<&?vMWub?+Vz)%yMqJ4?lckmFPmIi#Xe z$tWR7qJvOPr4o|b5=xdsD94bL$|#i-CgspU##s_dqMT+>Mbwav0-t^2;O>v~_O_canHwXVv8^)feAYlbUdi~5y-F6>}Y{2uB19ArLlh&@b(h|7nT z>Dg<&9w4tAnCVH#D z&?OYxr)U(s%5Qw`y=$PSvmo6t3sk@Dn>c{bZknhapiy|yCA7Y3z9=PPTEgfV_Xbe=Tb~z}Os~-ilNP4*U@S0zk8S$pDvF_EW&*kA1YnHY2Iqu|huWx0UhR+m)l6(#!jPd4t7A)wdQ4JiW;u z_ATiC)&Kg9jwx8;c8l0BJ;{RtT%39lY0w}5;?UQFtE#d`fTEMalkmWtY&V$_pAi4S zUn%XI^x&l#`$L4>8=OFKR_R?Eq%1k=;?Yy{tcWRT!7pt^yvZL8kkU>d9=`XRzVSM^ z{o5bHynR~K(NsWIZxBW{GB(}SfXP>bIva#d@PPMjkU|&*>jhvLm2|K#Fh7mNI_tpS z^{e1r$OVX^F}`U9q1^!B-8g$Vr5RNT$`@`X5nZJAJoQ_RCpPh?mJ6~VZlH|YRk9l( zHsuAU={uD*O`QVa2}wuZ85P5`e2L43Sc6JFPW;7**@xS&V{O2zJr%$M=%G#tfM$q0 z@l&9K_>NZU%b-?LfT-Wu@WIPgb|itmHQuVVcGqZSi+v|y~f&zdcI ztyy8S-ul&u_E|0NzTd!yF&wUJ&mGutLhu?S%`YD2&O&Mi0^o?{Qn&p=zaKo2o>tP7 zAhB?^PA1 zs+0wZW7aNEP0$ZhKoQ{6n*u()U){QpUmK+GddQ#qf)NIE^Jt4#)0EsuA^|?WX=|do z+g_ZFf@wrbTRlfq%(N_sJ_Wg@Lmr%awx}U!tYr(^PX<^vkVa(_Gy57rLQeg&gjoMw zDPRw^s1wKGq?nNy36!xOelVv7pHKUDZ31!o?RMk=7YL|H!g_@4^mFOzrF9=zRa>ER zCc~}J!A>^wt!csn#};*!=vZ^)di`CQO^D%2R;IcZM>yD2?vIM#Arm#^ErgBXXl!nS zHq>mSsG)phdR`uhD-J#952!4QMjz!E;_iaa%$2>Z?-G~#E60b<#TU~uVsvn+Z}8o@ zCpZTR;#~c^=-Q1-!VDf(LA_48CJpLnK9|93>x}T#Z#yvV>bX`r9gp|tUpO#4i>!3g zzjajfB=IhFVlnGs`k5?0vd!llc5jDMxj|3x%MU97#0z;zBX4PdA`gxt{C3dVo;d6} z1rF;MubprLZtC!Q_^^JV@(@_GA_~7k{xsSH74p<7;-qpDY5d_ffl)r2<81BE(bgM> zRv5i|{g`qWrhzX*K|;hp=agy%*s;bov6pAMbjx}_aw{lYmDF^bu+aXX}R zCobmq$pO`5++-l(TIzz!vAGe-&n2ru?qOp9EF**G%O{PXqLl{2)H4FV@ zypcMfnu$uL&D)*%au7aaby-7RY_X@pKbtehrvvA1Ffi zL(rkpJF6NCRj2kQcl%c8A>9oX2YkCDBEKr{VoUO>(r2*m%lP}hJ^6CVCAc-n$Pa0z zHaT$b1eQ=nkJ4t-h8$qh#T^}NB>)R2njLxAhbx5CBw*U0G{_L`;nU#fJbFk0{b5LI zyMRkUfG{L2K#cbkhiL;Wz0ROlYr~9fGTrtrKCn);LcWkN|L~2`x_pb7`ZMbRfZB1W z_FC*dpTjMdL&vXMk@I?RsY$p^zHp*HU$I+~>u6Q9C=(rYjO|@PQwliyu^_+u&9=Th z#Va0dt*dMbcU<%6$J&)8X$1XysTl$-oPFSGusnP*S@FpbUJR}jaEi@KE#kC929S~=;={dd~)t_dw6>!NVP?T)@Hxmp{%8(DjPeRhG z8oS43Sn{r%x5N{w$qBz}6eBL-GMG|VP1Irh7;6qAIfMqCe`MQfAQ)-K?vdJ|9kxLD)Uop9@1QBj7tDEkWvb&1+Z7#f_I~M zqwOPS^xRK@?M?biE-pV7((^t)>b$^EtUv?#I_)JL4fVn$cKGAXFr$4g19F&7VEFX5 zuU_E(Q^1~vhI&9c9U|af;6^d(3Aj1fpUZ1#j3m_wFJ(YF@RKycwOolpn3O2+$P|zJHys2MI`W$FzOqX?(r*=J2cE85t?L(ix z$tSBFm9`X?mwu_`8?APRdm5NGEG>qWUj;HXRaDjdI)jXM~&w6G$#UZd&MirHgqONm>63fYdX&Q{`~8JcJr)L zM@AneI3{g6tNbiNJ(5>UWF3b9eA>k4B2T!x5ouZ}h!jgiSGP-pNT;LDl_^@z+{Y!@69A3WO#yAXbRn~ox8+0~cpyu0a$TlU)*FJm=3Bj4J-t0t}} zr7{46mwuzL_P`FTh!cSwc!uCWWj3LyD_EK#oa%uJ!=jrar7)^ur&rq(e*S7?(iyFJ zi0&myp$`ZAyq0_#Iu_2*J=gf|wQ9y`sc)*U9HypCiI;$;=Vk{y#R2q#=}&=V9hfi% zOg&daBcWJ&q|I`23woIAgV&{M@UG*ltS=BEjCajHbD)^AFq|>#fun=VvlqlwbHCfl zVN42Y04HcC(yP~V68W-&fLo*yx?kgxkAS?#jOFq#@}8%^8IbE+hMd7@)gq-P$YD~e zbvcr+S$fB^a+hCR*{HSNJ$D7R^(m))DNA7mW!AhLUPt|r@^1x$^PY;&&u2AY^Pf*f zvgt#KTnTIk?|$)uYR}~~22aLwbEW>g8o}$UR`xD_cKzV_qYl&XmK8^7tUY9oh@U2G zD4dH?X!G@>u?_%ajR&tSj<>C4W;kzQl93!Q1+RymNiy>dTzd-b0KS5yeav+(UA1HZ z>`;*{!NdDR^KZ`N2U(2kYsQQlwMp+*Y*RV!FaiK1bPa4#(h?W0a}`803_>AV2Z6Ty z6-|eq@IYO6OHxq{%Ztuc^xr@GB}!ny$L0S?`lvI~w`)fFeu)~OPQLK{?KenXsQMRm ze0CB`BF+}wVHn5Lf{==*6TUVOAaiLU{i!YF;20x7T$YABXZ{qpGHmR{^0uW^rl>e& zkMH`raWW;`Z*R9o$Qr*|t@gXY>TeJdk*x@f$qsn18Xx`~g*0{&v;ptjYL{rbaGW|8 z%nPS61{=_24opSv;efTMY|k5mGU*?kx-CQFwao>edaW)Qyp?;m^2H%#@z7B6502QU zU{j(v+{=#kU4^Uy9-L(}KP&~0N&q8v+KX?N(xZAPFrk=^$Jdg(#!Lk=@=m_-=%;*n z-bDz*Nq7rUvUrx?*WNfww(I3_ruIbkjU{DsG8;8l=8A~5R*QtzA@`4xSjVR@na5lA z)Mxj2-2fR`WVDeq*a|r4=(79#$ z<(C+FZ8jv{Lf?w_)|yl_pkkLaK*medeS-y9n7w~UN zmcEx=_`3!s&72exPqYXy7+)dD2a)R5PCAJ|v|mUQ5B)vZUm?D7*(A6*@wWUhd8O?0hZwpeabgQ@`jPYCy<`D zpk;djk}K^Xv3AiSU-9C~3DW&0k~SMElBL~zZKUq3f4i{g42)rKzq&x@PKLO6F13F) z5j9B0B^)1ZDI6Q$S_bf8l3_t{xppGi;Oca z0#wzaAC!qMuuJ!@(@NDXKA$q&=+xqvCs60QYR9dY2R=SM9N!;L{(Ky?nmEj0+jZ7c zyTvlE(Rlumyeo@rT2Fps#%zuTBk2QBZDJ9dfCcH(27ir^GQv)a;|jLXv-)t&^AY&9 z8gi3H2BxL~8^5DDC;j7a9z}Fk&X>*K&wdSV?Q^Tg{D7Zo0d#V(O7*;*nd&7x}I$sVRYJt)tA@Hipq>$OBgXp!iS2=4vyU z8h2KEDIb%*;kqmI5qKOFz#1z+DJe-NhXpQ%(#22K$`{w#y~@A*vUYy>ar2tt(-0d= z`48G0M06HOtVjcr6)=v<*ax3;ZY)2)Y=8As`l0fTtE4zZ3-fr7jsU2FW4NcaE80JrbN*sPY{13^HjRvB8=A^VH!z*MOoN`hyOlax8EOO zi~`AS`U~-`EL&r~s6hL`82Mw%C4})wA6@EOO`YvVQ7LkL>{q!uPO;T2hptzgK6~?W zpZN*~vzpp+cM}%1N*Do+8L4(a&V=8HTckf!Z;2Ts2Z{5nQ^r^7+;{4Z+KblYPD>=5|cBPk6)vZ{u9iJ7A22E8r z^3g03*2f$KqLy5?D_+$BVbN{hs8eQCRuS(FjGZ#k1*fwO#H@e+HNnw+_UG-HtCuWP z6iU4mOzwB%LMoh}Y~Ah^MSorxT&(SvzMGvea8Uh2)@q%k2)i&BnGWasInp+{i_BJ* zEMd1p9rc6FgcQDq;>{I0!<7xtCQj(UFxwqT*d!HGdKbwK&?n~hAtkfi*{oZrYwn`$)@AdANpQ(G>?7|kwcD!wTtG?@0 z#qwSwk=s(R_BGu9HC^y|_GufW&GjG?hy zVWZR?#yeQDjKWJaSKCmv1*ihUGSvo($0YX|r>n!(G;dfqoIA)!;!SDC{-Nz;`LbXl8*y633Q2xMclo zpq~Xar0K<+fMY2^++F`1z>3nYS_ypfTnG(+ZuN-lcWPN$Q~LU| z$A|To^<6(v;i%K_5mDX(l$7dOtCs7PyR$@)Vx1|xrr|;3;E92_myM!b`a_<%c!9?K; zzda^(8xsC?we$ZYR|mlZYS?0`>fCdI?L^j2VmRd)B%|h_+iEwlWA9D+CVp{le^9o# zbV*RvLErtuc1>zJ+>wJJGfnb@cQEqG>)*!fKH))o$4mO?6=ED|Qs)4_0LnFu(CmdS z^2HvMGxT{eYv`Og4qPh-v;bCe4(FCnJ(}ob;u( z+m|_q;iot36DNj3s&?=5$RYXQ-u$rpn-2nIDw zTaim3KB;2Zz*EuTy?))o4g;TAGMV3!bc-(O)6n*bQFz68&m;7Cm&Wa4%|_4s4N-2o z-(No^uUYb9EAwOPs`B31@6L46q>*j|U}Qbm4(C*v;f3)DqrW`{OtA<*pc)d3Nf9SDoS<Hq>>`-M}MV*bEWg(H+u``cR%iZ~(vlqX7m4D)(hn4(pN1?W5+S{TTp^$5_MRs8&B3V8ERYZiLHFMinO_8d`Hq*w1r)Ozc+1;M!fptEFw`)Kc5A5Fm zFji&+=tzUFU3e#scM%FxP5$0KEkWW;LD>SdSmDnDE&!g5fCFh!ya?)-SW9!eh@$3h zSzqgp8B}_lQ?);U6K|z*p`>cv{*bi-)rzVfJl$RRTsYAoZmVDeE>%~nfW4{i$N;njpeE!axqljt zCmFuokoM#BAIFo(76lhr)*6o>K&JY@?^i#|@c~xYyeGY3b6Nc9Fe5q8aQ;J5d=P!7bCZ3Y>eB_4Kklz4 zQo^sltG{qba?09Vd4+g}qS`%Loqi?4@+ZS&Y%{jSgL|#$hEg$|4Vh`1n+yHhl%K&$ zJ&-yLg;v`5BIIpo4g%3UXyICogs!b1O%o>?xU=?yP&ZQtpkNgCzZPNKPiao{N$>F6 zP%5PJ;-JFf=8;85?K6n|mbaB~E1x&^7rh-`HRpt66((A32>YRAp=&x>(+yoSIT2+k zp2hjy2i97=T4L$zGgr#tk)!JF9y-w20UbR=?VxGT)rr-7Cu=`iUe#^=df)NWh3A4! z#S5z-_fl6~gxE{H@dkiqqH(+cOGps`6!9AYgfMCz0CxG14X7-ubn5qW4fx!5uty&I zIw~&^qK!`-RB#aJXm!#4%>a+o;e29IBC?G&x?(nzy}GcWt^fMd)-4S&m5a>H^`d_> z?F}-_CryjBBY$*oXjY9(>*8RppMC}u176k0CU2qkEv({muab2X z14N+!jV^t$h6jGyy$5E$vySq3c#}Eax8|c^*r`3%*P`VY3f=~I&N=mw+$#z0eHyg2 z!d~03NVrf3KdMG#D~V(CsrV&2#9SD9bchkORK+XK3nTtldxj5PJ_&AQIW*x)OG!)YPZN;{@n90FKj5l z5z-)qaIzX-Lfgo#yVUQ*ZaSs}LMXySYuJ`hZDefK-Jn?h$S=|EWmAs*oY!_rwchI6 zx`drq&Yq)kG?kxAN`ur2Wdx4HS^Y0&_;zX+i5Ceii3yPRfR94K7iFa6UajpRsGNb+ zqIDi<@}PX){s0Zs>mEi#4|k1(nW!c>o`sIK{^nNC=kGVg-^-72+5Aml&b)LZVc;KI zuf5JndZlVGbu5sVF_Zq%iYSe1Rj;B;6s_&j7p==cy`73aF*`jxo~xUC=`2oO{C(3A zaU-ZOxSHoM9YR5kx%lk8{-B>WlmNV#;|p)U2ahJ^##NDI0zc4!x)X`*ToJB((V^)weO=@0IarN$v;DjAo!u^kzWvyCC?%TjUhLpC^SUZuggOktDc$Psa zF3a+)O6np?53IMjpS-5U#-QBoG~1$4?#0gikE0oasgR_JzUD9;id38+vg~ObS$@57 zP$7aPSz;C-M4~z%QEUHvw%vj_#qfXUa#ud&c>Ysh)eLr@!TL{l1lpm7SX`I)h~(lG zPR}F8rWNKl8@qIIRrikSnAQr}c+Y;@y`u7lmR^-r4`d&6e%y-`_#-2%^VfFtQ_Tuj zXy3?S7IS2P0OZ{>?v-)k+r8LE(L?|0Qx94Wjp=E_`5OiPXmH~Hjr{zd9CASDRmPdSGcfJa{w!?JQd)g2>v1d_6Zdxev{J()lCB{5$VqTFYKRr$duo zZxjes5`*M=GPcQ?JC`t>X|n960PQpqBY@oeE{ESBvqgP}R)Qv0uOv+k6W9;H8_+=x z$8nbZUtE}uFU;YE6R@VZ`9B5dFGy8Eq#IP!kdE&%W5jWV25Fov6L`K4*2d>1WwesM zLa;xZyW5SF?^QOU79Q|oEZ~nht;2I2&{ zyDlhY5E`1mAKlQ3)9=16^jFMrF4x zggAG5TnP;TeKj4`_o`IY$0e+|W`5+Ytc*)0x*Htbl^({9Ze+f4RzHw=*=khAX`a*t zS@J^?Thwc41=4#T*o1hcxS=5KTI$_+HQHfn7f6~5d>L5Pqx3JzlcMY?<^k4Nwc5hZ zK5?@*dFj2$?%urUYV8@7B`bHO#-(nE#O3M)*39&$LrSuhhOTvuLv`sTp+5!e_-v-` zd^S_)W3~MfI{AzpF=W8q1}aP2F2v} zm^WZDmOafI{Pqq`XCqOY78tGO?FnolvCfGzFL23_W?IR+S>(zfO7*vrqsRLV2XEbS zTxwvca-(+P;b?XzEooN`_&;QHkr(y=rU@7$mAe39@n;ENKG}lW1imHbQ6ot%4XqBP z^t_gkth6Q{vY7kIfi=x7TP`0>SaCYX;kI`x4Jc~iO}*dr=mM9Xh~q5`!GRRctj|yrS@Y#HH09HV0cJ5(HHeTDLuW{8FVW zlF?1e-ZrCcrxpIe%ZT|C$HN%1zjHVNG^Rt%3-ovXmyO(A2p43(W(b2QO>Q9wa^nkN zZdnAgIcb})N5bE2fjxWsYd$elN+F(5L+(PrIugOHEvDDgIx~92%P;EpD3np9F|Fp_ zgwBSF6wmrwhPl7>t2TKmuj+dt8BczDpTar@hhH~l8JgIAJ6DspUlrzI2tcz+iVklD zt$X&!7Z*IzG7=eSC-zta;M$9q3*+v#zkUGToile;CMnOie5?(Zu>o7St(!U(ayeLr zCyFSnAHkI5xQciKN29MgPBLHBbvmc5K2sxXg73&)-z$6MkA zTi(lg*Wg&{g!S-_ReM26B0${5$wt&M@VPHHA*T+~;X~j@t|}vx(&VYX<`n=f>A%k_z{`q2&xzl_+fwyd1rNHc8zR&cVpVrEdswM1q5PKh zc4z#sU^gwc?Pf}EuXXr>RV$B(zpN8DZ(AxG7o#?hG@%3vZu}wWW&b;R5tDzis4An@ z6>CkT+0fKjYv{PNak%UC7UWf(a8{7YIgQgET$u=*u$}jEVTh3esEDoRa}xEc565kT zMczug@}AV?j9x;TRzb9GOFj~sQGrZrIZg2{ouZ99tD`QjeV8M^u{m8m*yX2_-BdP znmR)jSDjL_B6Yl!ZxU&Ilc@H)?bQFfF@@r&5~R!I10*LGftgOb$-rmP{s~pKQ^X?g+|Oi{cn1qczXl;#O{&)UfavRhz;Iz*LFJU054> zGEl|u;6B3$53P=G6mN}}qfM0G6qdpUf4M#PgA`;I&(hG1>XSXiP1!>aQO~YJiQ6u3 zy&lkYSuF{_N|Guv^X?uII1#A(#yr+@mu*2&yzKCk2AQ`d7lhh(io2wqu`u6Tbgiw; z`t&-j=#qe1LnEl{33lKYHd@hw2E^SS?VzP`uoOTsQTc%3`4zlhf2L zTI;%L;DVJ7kP4UjqkPLZeFm4qBPq|lDzbO=Xb$lh)F`L+Wu6zh56D#8r)pi%O4wiW zY*mhbiC(3^1-nC1%e=Q=4YThIU5)!tfLBOOmU|*V32JXWSjlFnf7g>+BuNk7)}u1D zOS(EYICg3rFR9wPKLv!teBGao@9D$>dgUpk_gClpASZyxI*%~xmW|#5fqL3b>};U3 zsLhBtYXBXI0D{GbQ#` z2$rtK8PF&g8i#5UN{Zs9XEctBWTF$NktW;tcJapGN4*M5J48N z1DEy?42LZiXW_YEnL=BBL(O&+zwy7`+uOF)#bf?+QDw7NRdWQZYy^<|{)coh6B^@D z33%((f$NBzI}^Fn2d&AC;9rG;PTT1X6<;l7w5iD+OlMk~cQTHnWA!P(QjuugEqGQ4$b&O&xD>90W@CB8{wT! zoQ4GIzpBvE=05qe+H9|Aa=UFBzPYgLhn7&EPv(Hs;I5e#Q+}K2Osnb7|H)woA_7vP zdi)y)g4rttxX4H&&VvcM5YZ^0a(4$9ejJ*pbJ)>z@?>K(_To}@)9VEZmsElqX;Dzs zdDM~7>Lsp5$n-V`b zU`hb#mqE1`P{rlj@)ljEwLGJ}wZ$dRfd}&u(*r-kDPyMn{HrOkfr6>d6UWwhbyIj} z!BUUEQi39rz07gylnOjl7+%5e#0|NOzcX3POAK7>o_u~yV)vQ?x16EPHxq|G>DJ~M z>C|jjnWXlb;R_$~A}=dKPZyOts|bC4Du9lzQ@9snCVB+9+xF;~|H&ZZ`Cjc%Fq938A_8ghxEQ-WQ|Ip!FfjlvzpVS&L$>m)OaNug9 z+>(RDMnkH+`CNmJuQlsZSI*B|p{*J@x>Gm>KM(_M22T;*a|d3nCds3byvTxh4Nhvx zIUA!{>D_L=vE#-r<0H!cOGSl`{#IjMeRDr$k(1CwL=B#e7<>El-XhhtOYp;b5`rHeb8l z!g2Wh18|pr|0MZu4~p$ZN(rRzx@2Kw=x78a9KZ)~ z%pb;(MrhPW)myL!`V0cp_FCfC^pJvlVNSOnYa-7f8JA*B-pz6@T@&!~==0$3y92!Q`MAF!Zbp<}V?a~R z!(Ks6+aqC~j@Ex$ZbxeW!C`>3L@y@=Z#C8c`+P9)Gp&FcsY^PFCr!meHs};SW27 z*{OLBQGp+6GCrhGieSJowu?o`hFdhrABC^3t+c)qJNY5+adzzKZTuAy^FlY0CqKrz zMhC256~3!LvR(}#oq#Gb{+pWc!`Nhs01mq{&j2#uf3{rL&9q$ScMxzXjF$r>xvIw- z7^KT4Dt_*b#D+E6g;%Z$WWgFa)HQHk zpMr=R-_tSV=38T9{zgIR4#uzDl$=ad#9-GN0?@0|1Capo0f19>V$NEM}6_Ka}>@x4_-G8{!|U= z18s;1IGSI*WLf^|eWzMWmJip5pb=ByJ3ITsjoU`zZHsBSJOWOz;5|R9C*QCoqAM<} zLzfXN>Y`DgV`>NZNn5n3z}tYGO4dTv}) zkwYf>Y^2*YQayS1>s01Pmx9)8^6 z6tsdM|Je$X;POkF>_TeUq{{u?cuw0Ri1@aAk9`Sp85r1tJRi>YXDO6JL5J~b|HF)% z-#&QeEKOio>8|%s-W^|7uy$3IM71L5>4bzJmqYwwss`q&l@PBmB~GkzhxDf&;v56_ z%EDW>KR~)1P>1+VYbw5+77PBtiq~jp&l`{GluI6SIUT$@EdJrSC+%+=pYaiOu&K&L zIb9cTZ8ZoxMbl;>jJmilZQYt>G)Z*I4o&Lzb8SA`=J8cMcEiQTpDJ=TfA}BT%Mp;55V}ObT2`V3>II{nZ6V3PIcs38QQ|Mt zA*8cnZ`D>62tR#lL!V$I8Y5E}U$cq*38={dPg`qH`^mVvBQLK(y$cD0Z0f^g5_TZ) zj-N8H`}2|!#=TTYKrCLw%c;g}u_$W5;rysdPFPr^sb_YMkB@@AxrC3dozmv{x%C3z z|JQSWRXOgM~Zlv z@)cvf!$`nbcHMg=|Iq};kQFykhi*rApG-MyXXiz#wZ|K&Xk!bmM6T)(sKT_dE%A@k;(IpW^BgA1~(Bn9$ZE<(x zol1Zm44mjt2DG4qrM$bHf{&TVBc+bU{_wf6-z%6>re8&!^(A2Whaz+3L3@u@-BJ31 zD$dTa#?|3n_N$($NoYoLt|6uIq~}(OKz85LO~&M%e$5RxYM&{6wj`k;nv-r|9{&80 z)IE#xH3N@|17;OR_@o-rbv@@&Hp6JzukiZ@xa4I%Wv9qHUf2ZGBD4X6%xbDE{l`@6 z#e4lz1#4JkgBRA9y+62CcXnueC88A593%rUhwuY_^Y2KW{7n;d-fQF?&`Uvgu^xCI zTaQyGhSJ1_=fA8AJPD`L2|{kNNa;G#VX{JCiyGlWHjKCT^JL0zG#PdXS2X!_`s$sA z1-dJU>O!Be{Vhpd@f>7+8@87vb(1w^ioUQ(>ZIN8sPpwbUhvpU*nI9l?#tsr%i1kPQaEk+ z!ppoAFA7^G!sX=@X#fegZFlkO zZdJ1~`#N2xiydJ_JKrA<>soZ*Qg?+)s<=CzZ4L$Yz0tkJpbtWB1h#-LG^D|m#LReE zn0|UGm5%DzI)6FK%~aBIbum1{^fh&~AR0@3#!TS+5%V4qNB_E9%_K}^t0~-RRe-N} zaRd=hG9&jK4yO+HK)?Z0kp!aAIkaxBKDq=-tRh}H1f3$pOCAohqB?shXyv#pV63_45Zb}3@z`L z+&fmPRGE`2d7I<&*nC6l+a=PkpCml4I22F*vZCK~_KtOi!NEo&%~M0CB;?^#mA7<{w0 z9`AJv{_i>jw!^68;k8WR)j-qQOkuf33{7#@ z;qMQcpi}ACQMT9v9|lv5yDz-ibNy%ao6JRSTTa$4TDWq*jNoEguz!=K`W8aR5t6n5 zr)?7MmDs^gY`N^X35zH}9`@q1&4LJv5rXOSsl~Gb4zU`lBv*aZ&oMU0bRMvCep_kp zPSS`ZD(fO}`alFMszPuVF_CIv+M=)%^E0NWI zDv?fcCb%|B53u(kO_@)Se%&NoVJcMNx46l+>6S@Xb~;Ybn5DmZETZLQ-n&@&bodB?~4FD<_HY&X_JZY}LPMZ#I3d+nK>s^-tqg z`>T?ZSDNmdnG#3cJkWGm$4ni;(jM2RCiOD8;URW;QlQfHFQqJECE^GQsAc`hzeT;Y^vtXO_>>hF(Rxz zEF#;agzE+_MoN=`ajb%LS!pD07+;OXwjS`_`LaoE{wGgI7mu4x_CYsSZN6T@kI8ts zrA0e%jI2N)%>(PUg+){oWmu#iGn!Bb8^d+|Oz2%Bl>}*2sQ&niD6`MY$H$XO_QrcY z^b1{~BDDEicaYIm@sX>bt4Mc{*Yu&+)k7T&eXangQfOts=bcf_QdX5j+~`sy#yGik zZC7C5Hw)JvAky$$fx{v){$cH}>$80u!mxZA+m zNU~aR$>pHAIv{tqKV#)HE||_N_X; zIT^uD(W2vGD6-T*LyXL}tnAqKSvSYn_VEGt1m~*C1kG1T5036NzWU&)1iv>sQcKYC z57Ju_8&rQVXx$$aj-?zD!BAhYnimsT2hjk(2&OpKC;*zMOH`E^^oWTKwqhrm&l@{< zyRWrc((fu$@oat8QJoe0uYvyD?1V`{J%zNm5+?RVRNZ$R7{VpEct2HZ_(?PN&v z$;FMs*x|?R1J9i{SN4Pr?|-@F`1g>$5J+9k5?&guG@sE+dt>`01D6!#QE2X7Up2d0 zn94Z;huLCD{ym;Twfj=)B~4nlrd@b+Dctbf>GzL!2&zMHNd8ceH<<}OipY5q`w2am zIH}Pu47F-5)ZWXm;_<}iR>aZ$McYWf?5Pe{fOn1N+MzkQhj-gpj=S#1`RLYsO2~NB zfu+i|mv$$d&A)O)W_`})=Yk1>l>!&}(Z)X_^Z#=^xF2x*kNPHU;ijeu!GRM(S$8Cl z>~C)8|M@|r7~HL^Yk38?V9{5fop|qOhMUqSTfKM-K7aabdm0F!?$4wg_-MEnMv4!> zT$>ImbUAqvK;ugy*TkIoDUaFDVO{zGG%+%X&-A8-ZgrDn+x;wBx%SVRcBFfrD%|n# zo$U(E>qDWEpVsx{gWts#9x0Qe6Mf=fx+=f6QXmj+So**i{%{FcN#Xpf69jjJ+rklH zopdN4L6?d%&pgDN)l|{kN9eP@9l5WcDfv|0lFU1`yywflw~6bN&j%N#;13mydVdwl z_cq{CU@xyNN9lOElLCJI0g`Zckj#XiiEanP6LzQY;8`ISw?fAl75`Y>6@7#!s@@QQTta*bDV_x~U&_~^@a__>`* zIy)G%T-onCcRNAi{S^c0JQKfzpLvI+0AO;E(vJd2`E`65=yV&fz}9!Eqw+^AR`!#b zJb1h1yzsLKwDb@8c6)tQ-p>}n74q5YPS4%fu=Fb8FQ%NmA*lRJtbLbwd^7DD2@U=! z&?QY8CHusZ2IhD{7;e5G28&dh54G*(efY1OLcA?`$dX*6iOIs$%VyE{uzcVP-;N97 zm$wKvU#CS;;5cBaE^KIGmgYh?^W@fMoX}sal+QjnR>6r@y5AyW_N`=0Nd3glHRoIs z$*o#|4x&h`GX(5zVowV%C=m`ZlwjqOd{^V&e+m@#4MjZ+1jzJ^5KB7x#?>jgyt#9A zuJf|h3n1+*Aanoq^)<)l25vtDD;BSh26MY42H8_!QcLw{U`uz3kAhwl8w!AV6%D~~ zG(McN4PJ`5mWFl)XRSdH#Mk6u<fJnMuGWvrz)#4>0dRU9hg$;ly37b%j(q&Jt|TaU z{vgg-_>o*rA0mLAKNDcXdybs6ji5j4eZnGJg9q#R#Qb3mj2Abgo_Z$ao8>7KuCBUD zhZ{0qV1Umwvt24LZ>L}+{FAg19w3W68e~g$< zXFF5Darp{2PjCumEPv_4XOmb4q;caD6pkabJZK;hK(O$)(aULsP`LXe+Gs2(mNdC! zkj$}!D|szzWsqXz4)Q`AlT*O$s-XbC;tp7ut>37CP+tPQ(-jh(^!|g0Kzw6#{)M$e zOjZHCC!(4>3sZevaHk2jkoj*XW4a^!FmN+R{rYU=FQ3g^ z9AU6wPT&F{40E91wA5q6e_%4~4W580>i&b4goOcLiJ~?1=C1XEbS1-Nfo|%Z} zLi;V>qI(o85rI`fSq>k1M$-uHeCGadCPe7EsfJDN-@gZE<^B|SLQbQyHldz9iYBVq z<_|^4T%leY!p?#wd3n!!W}e#b=G|5i(~0hRgLswWgdp^VfG|To4s?PYtR0`*;-ggb z5nBB?QpzGLkbVT803fm+Q24j=WE3QH#4Ct@Li_%5LZ^%P=ZG5M$c~6Pi%8V6h+8kY zCpS-qZ_28#6dBDr7Q{8GH@NMS{;CsvlaM5US+2Hp(@iPaG4=i&fWx1l^R&4wQOmkk2@au)Ta*$FcT|U15gs0BAEKl))R?vQZW<(xZ*rx1gGS$R50`MK`~4le%NOrP1ayuWL_#X(-m=Uj9I1 zOwyyA)ULSRtA@vqKgoy{sCK%2O8U|Mo*~S2J}5agI22bUVI*D+lr5&f`x0Z9#Ki=z z5XM7)wO+Tag2$)G=Qk^Yu z+xul&Je}-MFBzOeCgd$BS1dgg#*z@YeLRTNV76V8YeC0fWsFg$ZlW5X&s@q#lcG-{ zOL$@sy-8vnbtv0tto$9F&XN@$5m>KREY%6>m?$^)rW#Z8XFGR^jIge^Fb*bOPzbxx zFL9dl{>LZ1E2AP|<}YXOTe?-G2V(pYc@EQ^aePhH$D79==pgPu#NuEtrAx2_Ox7Gr z(2}rMSR6PI!NJz#K7aVAG~fVnY(~+wBcmHP;OfCB9Lg;-7dQQW@kM=30S5rFIHlDC zJkM_NKSt!W0xKPhsGzuw669nuOC2}!g)bymh*wY+)Usu}rC#?;Gvpg`+}ApVZq9yT zaq5`cHsR7mzK7J%G$}0^t(wfkf;TiX<-nj~+Mo+`i)ArO^cW~+Vtc^3&)G+t$GIN8 zVTOK+E(TT40VR7B+H1Cr+t?wMW;shErCrhof&a4I|O$Ui_I@hdaSR@;~N4Chw!HqBg53(;>? z@A51xCBzmlM>o3qwO%@?_&F!<^UJf+d1alOgWrXmON{PW8f1Jtd1*o<6fG#aZ_^U| z?L^(i9pax-a3*rp7B#5t@?x&$u0UlcLz_(247}Qnul$(pQI+ZT4Tdn*Q%ln*L8tnGg|CTjnu3hR+@Yp=Vah~&HHeI?IYKGKEO?KQkF(;yc2q-j z9_))W&7C;DSsi2xEJ4q1K|C0ONL+}cwnp&gf%n2XiJ+f}|G&E1ydBFt&~89e;Bl9J zo}#mMlG}EblAmp-%t|bO(%s*CX+Yg>EZStZj15J%;`Q_Asux-&jn?lJ2ii@QM+R^T z56l#@?EO{9BGFGCR)m$;;6>kqn}*<32Mu@$n#~OnvXvjoC7Sl?8<&6Ik?306{_9q3 z4vFzrPX~`G0@5rZe~9pH;Eqp+{)fMk7q#LfXg?*f+>;_O*`CRImMhbXP+4Pa(J7PS zZ*lu zZ#br(pxL2qUiiM;lRI5c*oGS*-D^9x_@D%EE*fTLCif$_6liZ~pa6&BsS3i`BZ zAubfwPO#Y!GO$Xq2)+PFM*GJ(o^$qPr^s(H}>;k^X$CcAz!XD?Crc=ydEHQyz$ zrxtbS2Jh4t)S+?MpOKWy>i|Cy7W@==0h*g{2>yE~P-cgin_z<+W`4hU?QG<4!Jl6vy(JWzYzt`nsFy{CSMtJt4# z^`an6m`2vXY&Jk6LKDG!A2#3DGoT}$s@y)f$e}^%?yg?*iXfA&dE?%f{}xJ4EB~k< zHV9Vkd+1{T_!a`L@?fcHu?8hs+*86Rb0d?k7Qsp%T1)SyaGNZZq&kmL6O4#e@$hy>2^V-k!ZS(>PmbvyOKf_PTb zVh%!R&{1AOi_7VpiCCUOG97Sv^!5zZ`<}YpYacrChfKRaih%j^_slnEcXecl4^>D9 zk)p23Ibd}>mI8SwY6&fi47bs`0S_Ey)kl&xDa9C{0=W`5JBcaUkiPairH9sLF%j5f zM;K%~*b)F-Q+zDRf{nCc`^jk5=WC*r3v~F?NPB^uIDBaOb$~;vA~ zi`r>m<*lG>jVm`U#oG#9^2jmoOt(z!JAZk%*Rj?%HR4RY&9m;C8E;UDbQK{n*? zuDQiiBlQ4;=zMyr+C!{9$F7eXLfwS#h-0uK(O|Ql(LIDshTdie{1&_tdxutMvAQlC zIeBL=)4R<({RpgaTk8rm){|Hw8IM*TjRavQ4ptFphY3sEPUb8Ij2UnWLZ|n{2?A`7 zaWJ4}Y7PV)!Tis&fJ^%pEu7%!AXGYraP1I$01EG8)Kz$rpgt1dPyZBpj~Yn#V1@gJuiW;_HCU{5lUM=TsB>BVOZo$Qhs60v(gf z{2unuiL9oZnYyV-+T(W31QR%;dNhW$c-Kc_M>$bmM=#g=peb|VX}vr+(TSH>W_7ai53#HR%X>G z>MG?62Dkd#fc+(?!$>f)`D8`av@s&v2RqF`eg?wBjsp{zaEWwF3=>boZd$b(!Le zkM1;U7q7Vs)idm6&|zKR2ROBx*oX`mL60%J0kX{yP4v9Q;uZ@rU&;am#jh3s0`F~$lTC6j;ZEx z^my_6;H%RnoJ?E>SnQ43(>9e{QOe6)@VYVKJC4yr-eGAnE*tQc^-k6&2sY+j7`Zso z!hG0n4OX`Y9}zNJgqM*$v{7W)Ja?qNl#{JmZKV`%dIzc7v5ENOsc6>UWt2!xQ8xZu zmHP4Ll#=ygmv;}!w<8yl@Z)20|14LfY=QdJB_G|Pq3BPwuKy?*|A=3>6IArJh z!e7((D48Mi4Cn5Xi7~oAgqp9}O7?&IdxYVxn~N-vg=6C zi*3hJ;uiGHJ<{TLT;novTofRFLLG{210#Sfyz7r(I}BITi$x8K$?=bDaUG=l4yU|M z(^tz}9je8jx;2xPWMv=0oFRSPoPOn5+3UlsE+Zffe!;*OK)HvOb}@6G4AP%C=68yk z{Gf+065}+}hL&j;vjk$=)PU1#7wqj zAV7w+1#LjNL2hfm(6^JXd$XXUD{TfOqjr}aZ^pt5Eo38fX4^Urm+~hGuVPIphzCQ= zuN14t-ZQz;U!J&O`+Ett)@zBLx{I00Kz&yaf#6YXtm1FSheC?!u(}uXK|}9ms>q6>c9GH(!rJ&JA+RP4k;K^pL;{D#Fj2PtqTjG)s>Mw$W z8b6cXovI*yk=_h04GDih;dh8pkkHlwXRzuLi{B7SeFG8-`o!ICXz3B2RmI3iWAv;U znrYG<#FYS9libF@T*U5ynnd&W-D-3NLe9F+$7t^7ia+mt`7zzh@C_=rj2wGvuQ#s+ zzM3X{?vMEIzZ4hw4oml}FCfPbASv2iH7$&Kd7$Y7O0_BVW@QifNEfDGE?A^ghrL;e~a&c29NPxoU}dBWGM_^bCAiU32<7<&G9*`E5xCkg0Wa&7cA*>`G z;M+unHBv4}o@cn90-j-RB1|VI-rQKdPOzZr=@7R2GU%^23qGcUSM-p*c9dMPutK`k zLPJULZ(-`qh@{g8kjBSFo`Rkqrs`_UH~xzfbTb|kxy!#Q1<}` zc!?q@O-l3R@MT4gmSL*MV!<4KV^3+B$&lRa>)vIC=_;(ArEJLP2(52i?1|e;47opG zcKc<3(2Ow>6W02|I1&0_%?d`j(}r!Pqk-m$)^GP4LU$_qQ0-u33uK2M)K~T~bl^r2 zIt5gI{N)$SYr>>V7p9tJriN9SiHjnk?EHp*{p_51;a}e{e!Ed2y_=iPJiAqcHXsj| zv9grjqXdoitjQp_W&*lUA&r#PP<$5>VhT=}5SOca=iJU{xUs+YhPk20!76yk9V9fW zCj|m|0+m#7xW-@X6Z=t^6yE44S!OE2O#KsXWBw=}KCC~iDUvdf>TwsMxj+9Cq)w}J`?V|~ZW6DBw7n1Bu9;(4{no1y*GLr}sc?+epo>pYx}oJ?+9$ZuQFr!e2J zJDTEo1~?c1g0hYw{ql598?P{um+%7g`ZtDq5iiRG^re)5%x(n~Zi5Df6zdP^`OO78 zpz#)5Mgu$Pn2ueL^ezJG>%Zk+*XAKZTIO(q9_vBFgJL@8RXhYwe)v73anK+?`90gk ztJ@Q6BTvRuIRzND-rsw?TZP487h&AvAGT6?=aZTc+&%_MxVlHSBb;0wuF-YO)MY<( z?rGc`yLV@o2SG#J%7+%}pF#NFlbOAWNviKzIN@Ov%culAj=%<$M&yaz5Y1O=b~xjw zIoiBbn0ep%i^iIx$56Lx@pUnVA^@*$3M17Ai-?N$)LZ8CbRv_Q3 zI`LFUESp$)il(t>f=?A(X7ms$M#3r2zk=&FA*bo?f}@0%>WUbV5HMvY=Rfgmh!kN} zo`uE(li|A}2kuzUhC$NS6Y8|XA?Sa}41M;R{pEVQ2Qd%wkJ&?JtWY-J2x3ZhZ^e3b zOefO6N`s_Lr|}+9f67v5FVYp97G4oqz&hw2U~4GI#VA z!A5;UmrF;!Hr#L#MRG0S>%Y_7AN`Z&{yj>p6v%XPK~HknknmB zEmfr)SHGyL@-NMqB!4O){NG7r|ARwPr6~})Y3c0*z=%5U z_3BZRUjBCStL|`#Gz2A?^k3384gCmF(5cY5A7s2w! z@c9_I4uP-VJk2RCEyBm5B==w(YES~^(78174kq2B8fK8z*wuEk)MLWBf*E*YV< zH={;cE)^ahLlM@&iFLzatt)+%k^Z3*Add`d1>?s{=@Ls&&Q;`T@O8KuxG>#xqF{^r z@YtLg>D)_wCBu6SWsi#dqm)%8O$X@6p?+3ZC#iKB2SK@9{KrB=6=p4=4tt@q3x}uN z2D-cIo-VQ1_9LsC4jwm{)GoLbT;bUqO}|%*T=J5Z6{Ks-f9y2v?Vg(3H}zg#S!kmC zPC{>RUU_b( zAs|T_G02OT832012c$NB&5bv{fYU_QL=`-Je&~uJT_&W?H1*lB_)RYjzrI>lJ-M?M z;{E!;7A~ZM)ctayB_l^_a>0CA;pB8sDVh>I<(!5-i*<^m%tn$xkP3$uatc8 z4C^;y@#wMCN;W=Cnuw=yl!OdbguE1wKK+QE2EUREk6+7o5I)G!Agh$>Jvt}gTF%Mu zw!WA)(Kx?;)wFpbQv$Att7tZ3&tbu(^9Tc4pXYD_Nfj-ZBZcf~XSLAypsND&%r(dw zG?je{+0#i2SahCY5{J|+!JlsuROv24uT_a zu{NQtsu(PGm+f1)CDtsS9CApNvRm;h^v%gz%&fQM2Xk$6br<(n#q(m)cHf!)>CCQ8 zs=d3k*Q^?*0mI|Y5U0q9rbZr$5nbB z4o+@ZaLlqM1Ax(LNo6EtLgN;MZ`8vn@tXa5xT;kRx zuG6%pjP0F$;z^NL<^tAN#|E&e@JKJu$+;k@?o#J1+bI~ozXj<7a{GKmMPZXDXp72W z?Hr?WVj|dZub>?LUtSO+H03}#~_Wpr*jm9;EgZfIz8vRFe3U_^qjaeD# zX6mB$B&J{Nm8g&v+;U>GX`R)=CZbcwMs}B>d)Q*46Ko6LaEQeX3=u0}Hfh!lc5@$) z)HV1%F1sn_6nEIuzy4-*d-Tb)v)2yaTW_^3eCKSjr;rbGu0ncHumY84^VGEq^oLD^ z>p+lQ0AGCbg|YHgSQ@P^iX7XD2MH|6at#?==*BBpQ`O9iIlKE|{Pp&} zRTX+&x^vHEWZ1n}whicPqfzOHHL{jhPL_M-qJs*<#|Py%$%OEDWas0*t^4Bd`Wwo`e0+$-u!* z*s+oU;Nz7A?M~_s$?*NB;!llD(>;Es0;KHzlkxBI>_0l5S|~w_Q#&94{4=XSl==4i z|NnT@XRv=kuTCNNp`0!YC35t$mUC!@%ThQ7pIE(XSp$#Yia)_xWn;j;cw80)W^`rk zkWR&q++^u~6)^VyUi`-D1&tf{wjqwb$r{h=?Y5oH<@2koNn4P%#hNiN4O2 zzDss%S(wja*n|l`b^dl=LM4E6_f#{E%ZHDEEEGhRrOMr zaJOFY0?jx|C`9q0OVNJj`>0}eG9%>h86#P?a-F4zqOj1$$nH|i!Z3p=W0%OBryeJ# z*b*u-f~9=?$FqH*kOwhNFs=7Wk3YsYc7#GAXdB9ELiuo?0xh9SP!aPfy#T8Zq~kAc zcOKa~rl$Gr-6!eQ`*&P9d~SlhbNsBhHb|VR7(=nS!#qCfRMnMG3L%1{O=yXB&O-Ek z8bStNCUqg0L`yI4s(t`RFlmIHs|A zSa0@Fx2Hm~)ywlu2UJdpSLLu^0gyr3WS-LGyskyTrpd{9YoA?ujO@(A6I*BZu26UXdT-6D`x3RUFrI{V zw@fvulpd1tJLTTU`HH2sLT7FO<})>LW#o%bCDNa~2)ZJ{QIG83_10@T{1)$v`lh6+J%cQ2#@@t=@1#LsM!y?`@?qP>K2Cnlp5*Jngb3*TI2IHNf>z=9P?B~+9IOSh1O?_ut6$dEr- z^4#ngct!L5kzjwT4r_Wntr%MMIJMN~(E?+%aqOP2M};ScTXxO+zV$whsDq4I4Mn?j z9`4qZKR4>Q0}q|reKG2hi~^-Z_pP~%@4PI$h8L0#!KFijxF^iQ;rP2zlM6$gr+VDq zsM!@8u=L8SQCRA3IL}isS4%s~Voz#cIT0{&PtkEk4*W zZ*j|_4&Tpy?sinCH!0O-=88R+f@UjGqRBaTZRc!cJQ9KLiERpTk=8%ZC!H(EU*{tz zj@Cc5@2-=Z@%7FVikDCAKKH@uNwDU&3$iYX!pCsG@-CvOlWXD}ba}M}xmJld`RaKK zblZU>I3D!QVd7hICej9eWFUO*|2YEzjbTB*))G2{oWRQoZ+@1upk01O6zX}T^3ml( zvJd4R1}*9H(cW)JZ{O}Flb#y-@%*Z(_suhuV`WG1u}BMU+?bs>Bji@2$J!~7j2dw; zmUB_{O3_Ikz3@>s9ElXN{P_6v6Su9ohJ^;kH+n5Lxj&7a7arPnHQ7*<@R-Uzvke&U zl9!pgs{XTFT)0SX8ospzD|7m0peGuZYrf5qg;mkUUdD2Mf>u!$jyid3#<5l=(FYhoLWcA363e4nl{j-nn&7V9WVaW8BjRO~TaN9G1t% zY`idY>J~YZ9sWol$Lo#6XY83L9w5&L1q<2)@;XH7wCBlYqa97B*kB5}n4pj^2E zJ(xg_4Hn+A105?{TTm#*9Fh?II;riag)eGpVi)S2$W`*zd;MJF*5QP#m-8Qn@6%nr zNxWj-7U00x4aiJ3@|=By8*qn(9uzh~JzAbVZn(OoyoeQ;GYQX5fMdOP8_zt-Bhk`x zM%2MKavzr`pWY+qzbZIS6J{KCTvu?#6E{%e2WyzVkA1Jzn4o z+UUcwaXHKa)%eD!>4on0yFE5LPW=)WA+GpR0vxx}Pk@v+TD&wSr!u)4N1;uv zbakZbfXK4mk(0VZ26UnjYUj6gOk3!1qS|0&Fpla{5s&3z% zNzPxMY%XcGp5k;(_Do%^SZJGyax69aulcfCfV&}ImYjB(gz%;nb!bI-E+UYFR(vxY zDZH#8CZj!~Z`#wg1}f=ii}JjML>gw6yI!+GThP z?KpIpTxCqAvAIV~h+E#p2w$ZF>wgXkZ@TIq2@Yf+!m0HfvZ3X(hkvx-yi6`dxgP1n zT}G3q@L&znAM6h3vxGvPL^$n?p{48>$Z$tI_~;H9lS{)JI_RAM-hPc*DZ(mH7ohE0 z4FNY|on#N>-0jyDp4Amq{QnzgDn&#}8+aLQm1V0Uon@tv0l1CDYc`WaANTwcyXuG! zx{%`%+DO@D+-+!eYtIOq+oFY^u|hCtECsuKR`W**v$A9aoA8$3p1m_FA7kq^``wkG zmA*u!Xu0v>0V zt^`hb$a6CCK3>Lq*!}fujQH3!TE`5Uw%pi0?*+QSBGN*XzVyQi5L@9}DcAv2WE#Tr zFowRWUv_L92H7Rj>N~2|^ao{iAg|`DcY!a;6Z{6+ki;S|TVi*LZ7m;@Ibi55?+H7fe@m zhm_UmMTY)PF(0M~phy7fkra+{4uB!wumX@j5FY31WjsBHCL@>8)3seeyzAD&o2WZI z1k*6?+^qjFi^h8I-q(nV|&lG;jdOAx@}$-Bu+qvtUgIkNb(2ZRl`^$jaH?ZMv1rQBIzB`!BK7bjF#7Mp|3owcbV#Tvev9>_L-w78FcbS!D(Zss2QLE3 z8_*YY6YZn2MV<`B_k5qPF&kKU0aVpp%X*r;`FC7wozJbJ)v-&+gZ(U&+|I5WU5e(El(A=?SnMXDxk~!{tB>VdW!g4 z>IW2w9X_i`s;Mx-)VcYMe*O_mmRPOuJpRL^Gu_jl*0_DzR6t|K>P6D);!~SHOWZgx zd70RNN~}!#L;A!{d!M5gIkjzTdG8xa50qqEon1Y_@Ix~eKJElbwGv)Dq<3&|+4Uy$ ztaDdegw%GJVR&eSxgVqobh$JunTyp*3zvOG``0<(hQu-`PK4A6@Gu0Gli?tXS1yL$ z6}9|TyzJB%*%W+dvsL^%g*vU&D}oJ=q^~G9_4zx^z>L0PoY&0y(qg5xK2_z-A;Gu9 zHtqC1Q~BY_iVX7|_wx^nJ$4nxU*X5f8L9K2a!r;l1h~+Lz+L{dtP<(W2f?#K553XK zB2*#;jy|ei^i@9+cHFm>u3h>!CoR74`f1ziSq4{M$)1WC7e(ROT&R-~erytVnF!1i%(eo-XW~Z_Q0F%)5Ko!nbt)LoeYl z`P!7$yufsOlbQaDD^F7G-LG75zHQRDA;$QE_<7F!ag;-c)+-r?`Dh$b86D5y7}AUA zz{1pr)oJIO&#p1mKFY-klUK)oEO-5y?6f^vM%BQwyy7yf=+{tE2W>WH*GZFSJE{fDkL7Ns@H;QP`Vx1<2r+Huwc-2JL@Xk!;5vdlJ`>d{2QwDnmH! zJa!6ZFJ(v8x`XbO{o}V;{RYAtq^dhj%1;b8Y|=Yy@}2*VYYk}FA}{YAP-gX232$SZvhsRXi+(YtLO53W&G0$$uC}ng=4*n&+TsIS zjh^3Z-LQ9tfj9+-d6B@-|KO1TU2vG()##9=bZ!a|pqPVZi1Tvb!rv#rQ{6blvn>F5 z_eiuTeqp#FIi4YeNF3*;xm-264Ws#pJHH#i0-sjmO9gf*K9d?hw^-SGOskYWkT>~G z$yW*ou&X4!36z^8L|`s4d=G*aV>^KmdZjD4|I_IJrd_xT=N*QI{A$@MhZyzR{JVG% zugxOuN=a#vx#nBrGkTvjUnv}oOietu{$hqy-V2Ivs|Wy$u!xRiwr%khhtahxVf0=*uN$l+(aW? z2-S7qi3x|KX#c}Zp*CwEVzML!o&PJi3LPG)I5fScwm>s%&n~5A{kp|XWlvmU-Jic8 znMWtL4reZX;Bp{k?sg5Q*-}!vcp6*_ZFKLG>gM7gq~`_ucZ$3P?S+k^@BvyVk3HSo z$=jLF9Bc+OD&oMsDNM%NANs5#31_&AKwq)Gl3tGyOUwk|?5NxXJgj&o(C-0>wG6vK zmkj-B%q!S0kTR)-axQ>}$AjIi4;ZOSm~BcEZ1%Ol1Dk~2tGLZuXTLjQzh!o>+V!JB zN20`h^dgz7$PW(j6x1B!9{ztcfe#pB!Wy!U-oMdw zCXheQ0oWQ1X~!q{rC`860it&v2ryt_ZVn;Tvdf7{9-IY8zd9lR1HWd~DA?y6|j2Niv2?E@~~FPH4Z@BPTy zW<_IrY4z(0m`~ceDqE0x?-zU})w|a?+4F5yHCMg87rmaLvR<$EQ(HEw1C@MAU9b$x z@4P?<$?6eGZ8i;`6Ee~-5(|2S_D^T=_P#2v!YUJhU{BpmV20-H@RdwN^XBDD@KM{s z#m?p!=K3y7`11V51+$xRgS5FZ1S5 zUi<~@OA@t^RKpj~t?lX0HduT*^LePIU9tSCQBu>3 zOw=9Pp@{K?K$(x*HkS(^#|iL^oizXc3RDsO@odZ=&jvB)9l~>1dnC>xf!q&>!o`I5 zFHmP(!6NkQPLShUg_jU_v|cxlo)Q&As`^B&d271wL4zxQ&_Uiw`Bp?nq5D_Gb!!*d%qGi&*J`=e3j(-P6N}XjhHJ3WxxnMimEF9mn zI-r>&udsH@O8d?EpDxG%(abdKjMZ7r2ecNr z1oc4bsyUYD%uVr-X3t>OL*L1F8a?D2^2Sv}Z0c%u+dSbh#ZWRAir<1GBcmz5HQ(6G zaNMULHqg^0QRvrzj)B&lkV6?NUtkZ%6IF#q)(Zt+;>mh z?%RxAE`3q2b@Q0}rn^p^iUw{sBbdQ^bf>XJLX%}^uQYfRcsNBoa-(PB0WJ6I-}|)dgg6k-$Q| zE*60cJF^`5G6!ax>nc8^BD`izOGOKS(PjAaxW7J+qv%iFNE!)n!Lis_GA2zI-qPARxUxQ(0H_e%Cw;4HJQ=ciGsX7M4J1 zKwmgL0?QW%*W-nxTHz6x!ct>$fUZDjA8ug>Z=J$5JNIC0ecAQHyVV&HiM10_w_g{T z3op={)7uD@)q>qyKCV7!KHJCcScKuumziI^g%Cy3?h7^W5SZ{>2o+SY!^blJI5YHX z&Me$iIE49fa}OW-4-qNN{}!f{N&&D$eJ{jJd^WvxcXH=|Mx zNiClxqY-FYJ5-88;5k|NWB{A^1*nk~vu2+?>TUGpu58vRtt%8F`v8~+3EYmq(4d{e z)ga~9A+N1=IY`R>XmBPihMr}mv<9_hmk;01(e>F-_4;hxM%k>3wAU6T1I?K2AvSJ$ zl)nUDlCXq(eFN^?9CjU(Es9=VZE1YZdfF3v@#mxJ#=Ji5f2`IPKh20mSx?K?lu62K<==+|8k7nt}W#6gwN`Y6R(7h4yg_S9 zfCF${2}`QBd*BZNaJJLvNbpfdJnWCl&-xbRr!WY~)>nMlfa#_vL(08Ec{i_f9 z#4LtlU-7WOmYjtp(?VhllA)1;nOgDlL$>2`Qg%yrdA$mFnG$;->AZm$z{(?wH~P<8 z4P>BpYivduNEj?jlRfTDFK#OWlhBgC9M|S~7BWW~Q?V$QU@9BCb&?HQRH%)DhjlXlxN4J2# zXvK>+8QT?-9SfhN#O^;50SPWAZU5qw1AM+|D|i|&`VEp9*2IG&KX(eTliKh{M-bgn zrC+rx&?!#beg!e_=Whq358rR6{N!#H|GoB~rVazR_kZUiR<^~b40aE36G)l+g5$2g zQ-JAkpNW0VAv44VOO6<)xAR26p#9#Z5|6_Cj}tY*3}`|Lkaf+(VnVZV0jP!(`l2r2 zT?V0~Db{QRJ#3VOqjSE$VGS_C3Je&pqg?&-2v-AOU4*N+VBBqtp^t}akuonDa{u^c)(~esQ-yGD z(EF$2EzqjI3mcl@k62*8c2j?}Bqjmvj;AmT;oO35kSrCl1eNif*_>p!l_y_kq@;Rj z{;R#1%d_))K2*N$F1ar*`wj~V>FRg#Q_D^DzrEaf^ZnQ<>N7uMcVlgjQyjln?I1eP5dClIu^*0-W6T65criam z7(z}bU$d!?lJ=&tR9sWs-g!4(aXNHm>0AgY=lOJvLzxUh#H=C-I^tGOF5J(FCq~k~ zMUF)YxUZKod?mpOzS{srK<)rk0cs#rT$@U$UKSiCi=iA*slc}0r5PFoYn`}_8*UD> z&k88>cp78XA~8|fNAall2N2vAHT&L*4XuOkDQF>O%tX1c_rK-BvIeKZp$sF-w$2qk z!ghF`VW-gj=OjH$V=8sXZF02r*<=y1c&60SQ&;sTL)+LcSV^c(zJvCg;h^eA>7zrF z{1L_&-QnB9IE+(;HB|F=;F2xWdbOe)L+;7ONY0yCZ4Q~i?@pwcUUa;5@!i~aU*gZG z?>a><6`!GC0wuBxz;OTCYVZM}k8JBXpQ9waitNRL>8TFkNnhvmGbHiQVu!4N3USj5 z^(Bu=m3Jt4iJ!YjZ$ALdzEE6yJo;%Rub33C8_AH~e#oJhC0Go@H=vJ^?&-Ypd?tmM z%wa3nf61BiHnysJ&$EDvr4b+Iu5cUkVbU0xHi}Sp`0iS3BQY-eD=+$>6fD&pv}8dJ z_)y+6V23)?(@2^T#eQohSDM2<1G#MBjqaR3 z7}>EL4c%cDgB%OUDInZh^<~atuDP=K?-ZI6z@YATu#}5*1=L?GG6N52Ajg;hv}Oa) z;!Yg|Aq-%HdmDQUhvpaW{bG!78}(Xb=}Vz#o<+`hLZ3%{fll;k$cudM9u2SmZ-f00>)Otv-tsSl#k&?-qX8`=}#FhW?lC+WRsV`6CAX7C0lIfF{H^PsetkV z4mOle&&rYH746)!EbN4Nq8HpH03*)69xa^OFotjmVAcMZRj@i{4nS&(4Aj8CwdfsD zfK>9DXk1%H*diNteztU;xlyQ~up%dN-?Yb@&s2HT%+*?`G%aW>=c(9&_Hlk{Als$3 zqj@WhRn+XlinorIU1&h<4{Hp6`8y?wew;$u=1B8+3DBbB&T*8yR7~$NzUzIrfjPW~ z`O;zNI`BohV1$BN)0`)6JO^i9dcfx=+}D*~YUjN$Yr5E!yxla^zk#wTal$iAm-~u> za3w+mWQIiBvWSf+f*Qb*(ZoBfd_#@P?P^F_cL-8kY`x!s&&Bqz(K6(-U3i!vvG9Pr8{sH*~7w7+t(9(cBiR4Thbu=Op*X z9yzn#4rJPG)Gd9MVDWM3%T|AL+(_&ew~w@hmMQs@*u%yi!q0v*^kvX3xG3fzyTB_R zkPviu!6CYQNBx_ZdXsOyHO$x=zw@ zC0$RKfXdi)jQ*Hw3Sz=b_Vi-tns5=GwyB5X2@UVfz`mMR3C3fbs1b1cPq%+-%#;Nc z`V8UXS74yP%z@j?lyK+4<+=B*cF$qh`UC9iXNA7pG+JG?qH{Yt2#g_GD^du&)k6#P4-gb`sDwBjH|VUR7Hm#uJK_Hj;x@Yn(KRIeZ% zN;iYiHei9YRrERDnZFf!vg`Sl^mRn7bF_MSFuv*FIfn%cru)Q>{lPmmJy3WY%|d`I zxUK71c~=7m`I9X80%y^)t`%rpTOJEHbs}ZjxP46~!CBG|e6))qoL%d8CtWI6pR`+r zbJjjI^1yf*6_ru`NSeDY4oBi|U?J)|jyb|L`3c92{S(Lh3`5+|5gbGnj{X5aL;z{3 z=YUPWv8Zk@y@!aYeX(6sK#%}#H7dP*3y5+$uB{$esv*fMep5`Gr~_G-9!=Qs*o5jW z!6Cmo(x?XUKti_8<%XXRU2DrNuhH>3PVQ-1+acfM5bI{Er3X ze{uwj$bo(&Y*Dpe9u@S95?od2g^M zM+y0(P4B`4lD^xy389C<~$A$znaIoa+OOq9(OnOFYm8na-lRN*wN6F1q zYH@N?Un(RAQ~Cm2#T)+c_kH*K{c~N(|8Xi{kWK%J=b;=allfSw2*qbQ5b~gTgi)k~ zv)emA&8&Au4gHZDE;pt3c28$(&OSTk<9VhYO2v3mA%r5045mt;ocy*CtvNJB;{2su zwF8;pas>dNw^rJAf{zy5K_waywp#Qenzxopyw;G0H-pL{-yNFYEm?hq)p?ZEtN$rU1KJo2RpXG|xcU#EPV(h#_L{+-$KT=d$ zb{r66@n67H|sybWnt8zbDuS2{_BHZOEKk(f!nJbrFM zG%3j-RABCs)~2)xK7d4}l%Ocmg7#{>RR^%9Cb4lyARcpJkxyVA4WF9<_LC#5l9Xs; z!T*xKgoA$HJ&NCM$d-O+OHA_QECh#oB)Az}Sp47Y0W~8iK3$1W9fGfs6hbaSEE^(? z?)0!l#izgensJ0H9`)>aoE5V=cxA9vZn)}4g&P@P3Pfl-C|3FhtIBEdcjv}9>WF1J zC?`M?IhFbJM<@R+75IKTTuPX))+{$(*SO+&f`_kW+`HkX&vP4IPCo6v@9m^qj-0MgEviSAg2Q1ChYaOr94_7t^K{Uu2WxTP zTEQc-9+4*O1xx&1A##3F_$;gKHH33hwxxA)L~Ce%%EyK*t6ke<#;62aNyCCJpyGjW z+9HmTY%ZA789~J(cM>QQ) zR;UZRp52sQtgq^kGd{|S%Q(~D?a0$(eO28PT-7w)OZED0$unEUv;Zhro;c*%ys074 zpURP>*!nM8yKQZyg`~1{l7gKyV2Sht;(SQ^U<@l7dkUw4qn7L;&0Bw`oWwx&0*0T* zZQFul&~BC8L=WDoGeDc@dbIv<%9>S0)1RGcS^89?req*S!po zKVu6)8Z2!C%YX%5y_Fr-+DSh)Z`$va*WUyV+N%dWt^id@T7tnia=76GqSC`%=$>WZ zaH4Su^z9$KycfUm^7iTDi(w)BV25pd#d+TOjSJr$?vc!n34c&WUo#cUn6oqe=ClK` z4HWA!2V~T1hh1Z5opQQ2(oDm(ylbchzVOdx8XjZOwEb9TDsy-uYw$MytZCn}V%nIj z)tu~Uw?DcMJGWk%vFx;*t!z>J8ENs^hQ$D=4OjQzhCAUf96e^7j|b*athFAO-SWO6 z4-=e|Dn=yx%4?HKCa2LZW55FO6Ke%;-2Bl!4L|tL&}&Ema0nEWV~#aJ`vvORv~g&j z&Az)|=BKYc{dSJ5Lhyzox$_IOWs9@Xy91Z>JzW&!PJUXkj$sGRb#GoeX;mDCb;+V| z0HF=Jq5J8ZE1~r68@%uF;L$9yc`gNSPUP3SZFu&!zw?1rScHOUrNPV{1{SYAW3PjI z$g%I>X59YZyxPEn94+?os*ByTGMREh0F>N@5i@m%+27>oJwVuVzv&W?N7g?S4EkL@ ze^U;c2fSCdRE-EJxFC$bXoGM1>W|wxVqQn61B)t|jx~>?e%Az4!I>Yr2P{?4ZZz>9 z3IoCui~hv}9gb^=;d&ebd`9vBHGyrHs00F4L+C^swllF|>l+rXwQKF2r^d49nyj=* zXAhyVxth=M!F)RP5KY#7g-y}51jqVS>n`h$)m7VPX7zGG?D$&Q)litmOcWDd1C_Cb z3GNAFax7a7`IPLFioX3`|2D|X33@d9Zn|yuR0GC+NPd`#gQX$mgXY*1Lr0#o}tWN7&g8R9|TjsS*# z(YHT@>G&8;?h=t~`hgd~o%`u0j6tP;#~5%feqam)js@~fZ3AZ5WZmZgeF4kRZMH)O zoCDO$3W@EeN!!&NS8^ z-~ROY$?Dt}x81g#wJ(`Rx%_eg=cJB!qlljCH@!KO4yY+SeNdNJ4hQC;G&z3*aHSWI zgJS`9VEX2=o_I(_>@I*!x(U+45_2uDhgF5T1#yoUYS>ENe(c!&y7-#@4ub-(DTn2B z(zhv#zmz!j1-8rQ6^ma7w*>J=I8ihT)@4-*x8a&DF7X3nFmWr>@!vs48hMWP*fO_V z$?r8FVgRspeZ^G`Z<2gW3XZ7Sbk@)FQ9ZFKU}5;xnX{}~H0`8ojsCx?sQV{bc7OD; z_+S0=XQ%%EP5bYvf6&AKzj{GMJ=H~j{;7qn{_nzmThM=cVgK{uTF^CxSJ7HGtbeD2 z?HVV4!2SL|Ga>$O|NGzk180Ew=Uepep7^Xkf_#dnc9Th?`)MyX5@wSBh7HK=_9jnqAbR`; zstE7(rIim2211gK$?p`^B*&Rs#QIX2(Yo--ZgP1ahT5J4f;32jZ`x?Hc^>>W#8C*8Oi7S;} zu-*OLz5DiImttP&rng^w6Ix_{%jO-+SUIV|ZTgqSkUN$1$q;a~qrZPpxF~@?|1wW^ z`HAp`1ARjtk>rO{4!k!fcgK4;nrCjG^x(Ghieny&dO}^dSgYOt`tX^=iNLKf-UY|b zJ5N_Hpf0lME?0r709?-UTBO%}r>hCGv58~ud%n^s?cG~_JMw^0T5}A?-fGkMae5Sm zwYyXuzgJXVcs$uv%=ON~4yX}dyURr(m11vwab;^K6a-4NL|f$c-_G-Ljhij)Xw$jo zwZpP|anr;itHKN7r-ZFAIX+yWV!OO zec`cqG{te|txNr8ZmV^^z&GCgIx6Zw2fV0L?*7~POO#Z5tId`Y6{9MbEckq3!g0G% zl`z*-DJh6D!z5zim0ZV}2GT~m#-~zJf13YK-}fV#`rrKg8<+z6QSjkEl0#=vO8(^3 z-u5RUjQ<>9fQbc)?Qb+w-%VijA5EV6pO4yaeWCH_@#vH12K$6BC&nQwB$%yhW_^9R z_J~aiIIqXJfG={YZ5`d(HV7HrsK!wLGSAa(nsY}&Zr!Mr^ygJi1ZR9&rfC=wul*_! zD(bE_c_q8qrCV|5gEgdP_#94+zIBXBmHo5%zs29N56mBx(XQa2D6sIgeV1cEc<@XL zu?eeOm>b_}Uz)VAPNP@LzFN$VGLK#ZY$hQsj8OAA^b65oOg_e@Nn?TEaj$i<30TI*0DA|csg}$#5rMm-}E)Mt}pIB z+uM`Px=&iwl)1D>%kAj>aAeM*nRzBl=Y?G|x_te{#-y}8ANzCoj*zLLyf2fIXZD;& zck(M`K4lc6by|g$9thRgdU{p8g{M(iu)fOyX5kq1u7isfZDXum!0Iq*u_feW`3T;v zP(N6Zd4eVJ!oa`n#B}4^z{im&^%K=UB4G=H-BlI@pkq=w)Fa& zN!^}F2g5(>%$D0k|GZMSyyP|3?a=8vx&JH&nK<)dpo-f1#H20{%gt`+;Ri3$9@rSq zc}}~+3-RCLl`Btv$S>*d=Qzf z6?=baFKvs%z*dVEvv4#Lu|j0G=Mn49ylS_8etgxqp3>=un9()9i~)K(IFWoaGJ|#Y z)NDxfP>D}BI&`G>IC720>Rv5y!7cfYID;DkNsdMyZ`Pt=K3#Cn_Laq<11JAn+mr3J3xs9Sn$oAclmFbg2qRuZj?*N-?1q5$PQ%0hJa> zKtmwqo$YttGuJognwc}_nsc2WGw+|YXXiW^pJZcYW=O(tK;1x1@!J1>yb4NmHaM1b>Td zR*AR_qIO=W+0QT}F=OJF3jx{VS%mM@jjRBH7KZMzd3#fqkqqdD8w2tI&;1h+1r5|X zssr;5(#7h(vJ?ArXP(Ujzu&tgxWRMzI|!Mz0w7y_ywi~+)UZDsru}$%DPDwST=yJYd!lGc zSww&_a2jQ6ViQ;G)uNjp&>IwidAil28N zYz3z(Rg|Nnx>Ay*!LgvId6DH!Dfb!n2$>1!^0Ph&-ef1N6}fIShWwf5tznrWq50#^ z`Ge-$hdTNyb`L&FJq(2of432|VQe3|8(eTFwkux+j+7d;ycC>!z+GbW(j(&c5OGUOo{gcJDAF+cr zYmVut7e;GLm11k@=S|B30wHx36s{}FW_^$uJ3T6f;;2SBBf>ZZqhCW+4+#Jj-sO48 zoNEJmEq@$|oQ!_xh$s1y#D^V84*47)rv2+rI;P*EK9NuUq;pM)%a;KNV+cvG$5Dua z$k!p6Vv%eEzMlA1>(TpeRM`1V|{Qe}OyP(djM{6k($z$28nlqGDLQN}Q>a?94YMA`?rBg=hMYVuT zmT2bg?Y+oEAQ_P|8vEh58ueoRot#q(a=niLB| zYQ(mQ!m+f6k5Q?Z z*?F(Hz4IJrR`zottNOQZ{l7l3{dP-c!19~jV&9HD$k_t}C`rvndZD64prk4?)Rq(p zyq{>_@A_4qY67NH*{?%i3qES*d1t%H==058FWPEO?KC4lN}GV9a2F7L@E8hb>mT6C zQ`Scow?z1+N@`7n&o3CXygZEl+Buh}E(*=MQ2ackKN`;h`+UNi0Fq2z0X;e<6N3Dz zFYfUQ>J;1o-xnDer|r$*l|#!cZY=Jzy;kI}Qq}<2wzDEL%C7eNv0AYq2W~*DGo9?j zHopZB>Ihdm+Q_2^o&s{uDe~TS>uuuq8@sv{H`8s?&leCo(+~uf62*f{FBuHQ<~l9E z`T2f|eVUhU{GCWG@*3aLq|d4ga#t--`O;>oQ`Wnrc_Q@+U+K)J){(sGh8HeE92(l0 z!#${It00csi@~`@F*YwsN*3GSys4nSCjU%tlL6a|oL%Zb@*qUuA|n)D!g#M}A5FS8 zE(?f;+421x4gNZtNgLqoPlZ|V9!d7g|AmJn0$4p7f;ZZE>Tq$gEp$~rvL%#=s&Ozw zhCZBbNvoe$Q{;Dj7S+PUti^FT>_jkM4c|k5T{qm?qk#cA_CqZ|J55+Q4FVWdAi&qY z1q#bF20)+c9wn%(j8jyqJr9#QU-@%af3nma?SNH&dZ0ATeegS+KmlmI2*?A7^Td!K zuaUdG4b6e zc*|OStf}P-m0cmvWD~=QOA#UscVXDHd8X{_ z3x=dE;UCE3jEaI!E?Y+dy0G^LW3qv7a>(lqf!;D_Xa7*~f|x?rQ2XNKS>d&OkB@eJ z91)+ze8M=O8ZLQ2!933PmO(Z!HHyfBHbOIvSJXYn*(bwHS8XltIsJfy}H@N zu2(wq>QoxNqk?&Xr+kxFu*P-9vQ~wv_^;`hE5rmVFui);3MY+T;0TJ-`dQ41W0vT;5*vGCnr38tR2}#D1rkXF)yiIz7ZBfA2E><;JsZt&&rl zN?v;5+EdFMf^xRlw{N5D%-ruu-V;BcZdbBymIRcPf&VSOzkNWkyhD;AL}D{*Qm{zY zg@*hR7h+JYGUplkjQR(PFI^roav#V1qqT$($c;m;JRw>3sx6A+KW-hSfKe3aq zwR)tP0pE3hO#0x8*2nE({$Xx>1%(?)Or~aP1B8uUmYPjtrj0m=j=~jIb{Oq9A?$5> z{*_L*%Im^$0l_Y3ZL-of=5C&>jILUOE*FEljDjPcU~ekmqK$!U$^yQbJ6^Vp@5+OH z^``1Qic>`=exPtGFQiTQ4N}i|s=Me^sl_IhcziG>?$}#JQw?dSM1L;{9xVZfjn!|0 zYo?}VS<;5oq8hrB*bPjSR9-L@J_KqI+ZA0%2+1GY6sGJLhdz~N6p(WIwqE6J?RWJT zM(&(RQyhKI5><8dR+#7qgeEYXAh>BLexJrj4?=ycUTR0}-FtXi45OoFj#Ly_erRlb zFY0kjp8dr#TRAvA|Eu~E@)vKgz=aE%N!^AY;DO~B!a$p`8pS%!&lf3HlHG+lz zk9(e?Df^x+27_#w*N>>5tR=AmJu+y$L5U@lqxc6t0oajS3~Uqf5b zEZ2Mb#>Sg0ZWTCSC&%YVp5U>8`mCIk`Wi7&AXJY$Lekm}2`x`4^s=I##sx|FvhZZ# zU;;0dTN`8eZgdCqJOaF}=edps4`0`PI0*O!eVx}XB!Gv+L+Z`CAhBqn=?3OdccY1h zYM!ZYw?=y_JbnGO@uC+SsXxOEMc>Y-ElX)ne*#3U{cSVAX3i&@6Bg?TC0O2W65Oa_ zUe?M%^+Uvl?1MkhE>c|VT$z#g)5T*>$l%pz%=BbwFwm-KDca>E@i$cwq7~Yz4d)&#e2KJ4-<^+9Xx8>Zm=R7_0LIs&ASU@WmZK)&WDUt zkkGPR0JRaVo^Vb zu7U8O`4_fqr=yka9|DKO4o{5=$s^|a`UyS1F@HS~Z%jDWY;iZ~>pMZalPBY8-fjHk zv&ar+T{+t9q1v`HSryKN|K6Q@Yve*8ij$OUI9c92{!=|QO4ZLyEa=p8&M*D^j||Va z9P>NQnNmSJwjN>@3tg^-Hkif!l(bs>+_mB!frG8y!KxR>9Iyj+<1IRt{JgE(BayJ) zL+@7#)iN>#Aa~&{1{bu#HeW|{#4q8DmP60)T^*xbn;vA%BqY^{l3BNjW^LV{3Y6ME zy@v)Q?&Z8%h0e=#EHVRWb#5}-;s$TJH$oNvqXMf1(-(l&cAd%=^BdrbxBImC?b6L? zQM$G1Z_wXd!fT{l6u3HV;SxCSWOV==ul;S(YhghPbQAejL)qSz?X!r-V8zUO*LRj> zfO6>`MgbuX=LE*z38L%tktEmUYRA>sNO^>W9)8~eJs#LkQTjSFyJ7HSQReCS`UUua z+p{7grCmD15Yd=~mnBbZvaP)oLP1*Wr-7pxZ^<(%6fd~v?@i0s44k$@5qaO2krGOZ z021sZz10uQomskP@ujSV{pVl~omw$*)!D9&3icV4**W8v5&A)ncyo~0>}x@drX7Ln zP>Xa=Qy!3-pS*D(t#%%|5D%LYf0?vu$nAU=38_DS!dNk08_p(t#{>xPt8iqrHUfSAJvgFSPUz;g#)!v zrzxC$#GJ`eY6Qj`5|gi4U_yv}DvB@V^rG>jYsX6`7X7(6Y`rds-nzSmmMRXNM>zHB z2V*Ea5-S)A2Wt6xLa`lOp9Cu;kyi`i0_GiANirr=fih1^bF}oyIv;vV0_7$=Su-oO z%HQX8z`0pIU01WYIhUCAO{wP1+&ZF=_6do%Kt(x1XcsBABnZ~E9*=0nSJ)<)K6CJT zG}iU%w!Qv$TQ8p43q1;dyxd+ZzvP^Ma*>G8#k*A(*^^?s4wKMT-uIICbfOXhlj3h@ zFqm-62uZFHxhdZ3$ami514x zSoK#lTcxgfOoYFeT02FrZOUXGlwwginQmL~TXOs);9uxe|Ju-&^5adhycsjIq?3v_ ze5&6$7g_(=N!70VQUpsB?06s+ok_|BGxi!amj*%fittl0vdfddFfk6|ZEvHzc(Po@ zY8x=ks>O#tm?ODT=X=g{$I{0baDBt%^}Yp0Tp0`!x%3GM<`7NsTy}8Jkv8UKn73hD zJ#8qq%Y5qu*`VcKf-ihpsiW5AneqIJ#M!>H&sD}P8Lvy3j1x4R;$EWmC~WH{tM0fS zU{}k-I5s?zct6q@O*sF7ko;t{0_nz=ExY~Zj9nb+2fWYhr>hmeRC5ZxN`!4*FERjy z1;juqnTxixRL_P}BAmqPMw5-Ntl*Huz{wgK+lIB-iizDsy8d|C<;ANO_OIvp+sasw zPtKoUNmMGpDfXO?pHr`K9Li+BwZym3{%{7D{#ho4)A*v7)WSxQg@x&WP4hpThJVc+ z{XcpB9{}tBn~Ht^nd#gAJ4Eq#-%w6FK_?@pAU2Ry7lP5Le}|fU7EakJEUtOE;8NPc zZoy>k7izW_H~q6UkIex+>ieve7{qaDv+EnHV5Y`?KdKG74D!tyA#oTtOdPw1W>bPZ zMQ3@);m)R`b7RboOPG`R?svoX?XM@cL#9Uy@YZZI&+SR#gnRp|!%?z{`5Xk(jvTi{ z4%r9sqz9WY#fDnQ+0ir?NXuM7u`-?+pZdXg#;#d! zUgaa>(H#qeNvGbEAK)G2@$q>?t5P&HqrZ0nN{y!&=tyEC;0QBjE6mRx%ou#f6bA&C`i zTUH@}>&D~;sL-dTw`tM@uRgJ7O-l`Rbjs#xC22wG!l#Z9sUhh8j(AE#n<4>u@zE@3#TuewZPRlgpZTZ%azP+99UEW9d z?H^piOHjDBR!`z?k<~q*^_aLe75r9L$Z(HQ0EV$Cyt{7e5i8OASx(|KZzVp?VdAu? z{l}VvGq9r=?>2AX)%pU7JaLK&-o4BB>Eao}kJtH3$p?2QkSo#?xSg7>6XkXZuTbR? zHbdP>=f0}>38YnW2cxnu-Z**UYwBCfaxwBKO@!pXJp2@7i4RAhq0hT>o+mYEyopH< z^<+(aXK+7FBPU3)KKGJGhvkH`Sg1YO3N8#5#*W~=Ref43;AK1E%_1qRVyau7Wm_T4 zON)kzz4Tpet1)cnZyP%GAhjkc4@9|e7%WoXAjcxziwu;hW@6)e+$i3cX&ZMt+V6D1cYq)pjq$fZB$w)_G+96E;FpK`t*Y&0GQr=y9U7@Al3>5M&&K6~W9H(jp5ukoblvD2a+#H@A4#fP68( zIRR)S6cvyg8W{?8veSC8b8cE!j3;_>T32O?;o~y{!-D=_!dvS*B*~jlym{3Q>pXy? z&-kA3v}HLoC|o=Xh3g zar`|aG#f<{Fe9rldmz!;JtbXG4UdDJ(?G{!mw&H3%J zW(yjWls_8HeN;z*>%u}(W@Pv^pLd&481yk>?oT`IFKO0f%(Fga_}A%Q$R8{vR9G$ zXql0Ldo^qv#bx2wyoaK2auQ*rK(Yj~1DJ{@6nuxtb@KdZ{SJNbJr76gi3=)PwbRT( zDD=0DMJvGuQmm#4!~>PsKA`o_cCUIO-ClM@hA300(z z=acuWTAw5&yeK)}`6Z3#TJ{YEw~mDdawzDS8EP8XvI~(6;IX^_REwZ3@CqP7m#s+Q z(Q!_m*gTOrEn&?=6Z^sr?wUPmW%i_+&xJ}y^=h1njnp`V7C?AG$GAWF4UnL^a7=UZ zr<#&{e=#;JH2?%kFR+F|JjZ(jH?xpk0mF)1JGlHHo^C4d7EWce8k3r_p=*{01hKj? z?$Bfi-c&vk6tF{UAwELtA?@=^v%Z}?&pbJU(bMLW;;m}1H(zB5DpAv%qMCgj210=y zsNQB-FlpVuQ0JC1VhNznlxW;z>7mrJ8f5<BMBUAP_fLz{5nC7Q$tTm=RTDiqsW=Y=^7*3A2dwzy@9_vOB)gJ=cUj3H-} ze`A4TDFK{$ORziR5M~QcA=fXWdj9oL~kS6g427N z8f!`EH>Tua_P)N3X5nSt1|K*~#cKAn(!K@{a330yy@N|=Opaxwgf27>HLOiy8D~6h z5%IMlDX3?eb=(Y8}jxk z=)ZHUx{kBR?jcY-=(M3Zfe|j-2f(50U#^g?5!yu^imS(jL9Fr#jp6xgMBjAIg>)ER zo)|h!<0I8V={E}IXc7|??K%7M8r4hUryY3Ctq$exuhRKA=3_JflN`htB339M{vUDK z0{{sss7^=vbu{19?n)SN>~igW78&*Ezc*+kYfRU4wPP{BAnvvnkYpUgceJ_j zmc)G0<7HC_h`1@+1iWgnhgiRFoiKXWU;~@EOuFpiB1l#Av$uI2*J%{=!|C$NjPcM` z$P25bcl3Zwi&WTcW z_v&-4T)$z`*rG{T>sp+;#iL!i%FMw2+>lws{mK!aM<%9b1CKx%;^i!8qdQ>lg#}@n z)KU;<7B@rk+FBeEpmEO4ic6g)mqrbUFZOP%z1sHvWkzFw5RPF}x(jA5z=E*f<9?a( zd0C+q395M~D&E{tnOw+f)Qrj+iA`<6*-&E(QSzi|LOEe-B)|C6nB+i_}OSTvzK9cdJY6WhG)I8Y}+ufWK;1$9mvF9W;32X zYFX75`f7sM14pmvx~0z%H;gYv4b?pmM_k9&R%>-Et=Ti!S{rUeb5jm~!$LZQ1fywP z9Y-$=cLOSMiTMY`{0KfaUL{qX*Po#jcH=g4oU2=7gnKfZqR!iXLfuum49FG-3Rj?Fs-nUUzE9i9Fe2Xob|&zEoc(3B(dZ1tlJ zebws>skEMOjh;W8gYK2dcp-ujpR5}Hq^q5#@b0Wuw6{7D?FCmWqUwjA3cxQp37Wr6 zee>(Y$=24o{;nFqu~IOqJ$gZ8bJ{edb_Y;U?b23BPxA8$V=I~YI9HbM|;2VcvgL4-2p_$%(82cIehqvZ7LcgXc+)uFk`B9B4 zM>6!6Um&1>3)k`O6JQ@2D^~9C@g)CSCTa8j{tg^;uE92Yx`lOaU~ohz~U2 zjd8YJA#2bE&?h37%_8f`-^X`o;-su6PX@^^ZibC}=7UeoB#tmG+~3W=j%G(XCmVQZMHeHn~S+ zzG?5u6irw>iUD3+*xe%69S=VU-e(461?$13_Y1nAhsX(;bdsJQq)(ot&WXh&*AHk9 zvPC+wDyJ~i$;%L_4>ua5#vjnc;?={2pZ48Z1=BXd8?Heuh290EE@-T_5*Dbm*n2}c z+N4UhG+&dhkJ;pGdbbXeTi0bHcXszPv$^pUq44ycV7Be*mEk#1?Bxkrku46POwzzQ zCSmw>)we=o6STeHl2cd6Q1kbp$Bu` zbzG(RLbqi=SBp>j?G6$K&7MLWCAtN?fMBxYI}F}o=inkL21&Hg262RvA-F#3x2uEn z5w@l=R1x;{3C|-FDBh&6tMA*qXaeuD_)F_=H+;|x3||U<+IF!+Yz*u)K`CttR!|@0 zOcRYconW~PHUPK^!LA3*ManJoL@VHb6!{f_0K2L{l)~I~z6Is+T8ZaA&3kkj_wlTO z8a*E`Y#APk<{8|nwhKJ0JDhO4`=Eic2fstedrsj7#9_~sl-Y`vh8vg0@IzhrhthW#Fiamwn_k&D> z9&0QGRW1%j>U>@J;F4ptmfo2ty>gRjZ$sty*Xg1!%HIw`0-+LURqwYTz>8U~97T-V zzD|&jD5xK|^~jfQfyjXhEo6O{x0u{o{ZMuV;|01+`xX%AyUfmh?uX_3(+oO#C)zBD z3Di6^2VxS8RKrdfMEGa$FAPg?^31&b+(3u$*tLa+X}!zZK5bQW-h0o1(h?q^b`oUi z;3W%cGSZV#4t)+Th5wO%9`D;N`rzFr(%q1~r_9SL@sG42F88l5Tws6j+z)$g_0nhm zN`zl#i%1aoA<&&Ej z?jFj*=eROt<0`$LYo#gHUVPMe?1|K7^H9i13ePd3G_?{6b{ZzQIG$~LxY{Pz3du@Z zobR@ceR|>ZMtyy4(~B#9FyDv^VjMo3(0sY= zW3D3aW7PUdq7F5u9yRlX$ZHm}R4<*1>$Uf+&-7Td0-oAtnYc}1%PhY zIX#%voS!H_8SfxFj?y?bp{%}bPFN7+Ot+yH*Hk1}#bUn$+F^ zVyHNxTq(kcE~O6Y&c*-_1jxiaQ}<_W4< zI)q0AJ6*5cH4ZDKvwmIS4c!{-zgdV~j3rc%k5H7T>1b8TWl}401v-;pN5WK?*66?T zP@NK}-N1V+3brL-YMpPjw8dMinT!5mpWxFkN*7P9DIjKmP77c7 z%R@4^res0f*gNma4UYMtl20%nKuuL1+y00;%a=MV*P*gyE8h<8)})#`%t%sdeJD;u zr!E8DAa^;`*$khQ`I4qA5Z#dR&NIh$zSP#8_~?ku#Q{mb-DibCs4C=g0<;dIW6}q4|5|sZ)3GK?(e|T^l!DHJ+Efbe2Aw9_tPcBADF3(K7Ga0Ox1k()1w}@ zH3}JAqHwNSga1tz48>0Ovhg8C9h6KjEE$lJf`XR$LFm-ZTs8W5zH3JJCAOERMvPwa z3zk?Po*-Fwx5y{9={RGTSQe@m^IqHi&Y6wiv2nXWc+Ya=NJfOBl4D!}-q?3G))El? zk#pr_>~Of{^}1>Fg)Q5dsKcZvrwakI!fmgr6JDJ-Ayl&{volPR^o^wHlIn=(NU_UH zvzLuvaR0!PHcjK%kBv>NF^;N&o^iKgJkM|#GOrzfWSJWOz4vC4QI*<0$RW|ZIPsNX z8-vOwwrJ`;I_=ldF|2$@^T90VBN??JMYK5wA%f$!4!*UwfRQGAJJ^agjg>`6H=*A( zO{xTWK+(6?A7-DR&t>Hr{jUn# z|K9U|l0x@?C2{kVV>gWrzn>JqjA0_7KeC#bViJ4&U=PAqQw3kw<|_aZVhCnA3c~_- zd_9B$BzrM5j}k|p*@r7Z9;L*jec4KMH(HB{x>t<3iYNCZpIz}9t4_UFW|AA&=DNij zK!qWd_Rvs#Ni3;2z9es@#PYd&eFMoP;Pc zcSQC5~NoF0>ar>0C`T`HDW@^xN*yrJv z*4og#nilRU4#?x4>xN}=+!PXm_N@I4o2`8_-}yZKW1K>uu95##R=lj%?`aUF(Z?0V zp}uv!I81al-QaWZE6K9E0`0Zf7wqqnrER5>sSJ&rhM9Zr+95aVE&tHd=d$>%4teD0 zYEEL2&f(4a8(OJR>VrZ#!9sUIG5MIx{yr~0w>*6w;r`CM-{S5i2j?(mXsSzp>^c)U z!M-4n7B9%2SGHk`NOC@vUD_dBUERWfjp1VNeX1#49L9C3-OpBMy*TWev|?16m$4)> z%go6uMlru$Nfx(XynxtOj^=CBdnH+T{O+~(a(dAs@RH%9f;uidBd8pOGdy6E%mzoKjJk>g_ckB|fZ^K!@k7qndG>xzDMiheo$ zKxcR5BxE+_2zhi^Qgric>^Fv**KMCnf=V5QFZaOPml^N%3x=)ht@9yD;=h;)c+C5& z4bSJ6Qjd zZXe){ohc;Ke|nd2QFBfPfPGQZ|N5<^f1Ko>-~N4)zx(9xdGfdI04w-=DfoLS_%B}y z!0z^Y^V$HL*R$aHOWKeB`{q_tgY4lrK60UhCYgOA)GiEaTMG9q_Ip?AJi|~>@ac$+ zn%Z+)+?e*dQ)66{n9J9^SOerGEqU!-&m#>k3Ig9p*!n2(i?M}oO#H$NLgr$dRxa^A z`JWa7|7Q`16Dvt(H!zEZOk>Fn*@|mZt>F<~QK@N?$8z!&{lSiYe&9QW7C33jF2S8G zc2dmpk5e$L#8mNa|29%{(c5yvoSKap07X(Pzu&!g09!{wX$>NFA&}ue>8kCZZeF^d zCj8$jv^D;Og1igc6uA#<-~XhWsxGKgfw*~T%QFWF6711RT^lCiqv|%H8=98j|WY zOI9)m$MV9H{%Sh$_+S3weQI_kIQF@{_pg@PfBnNo#HWeS?8@(Vo&Re2_r?4@GJj9A z-)8-{&HTHo8CK~B6hdpzId|~>TSIi6<`ntV6u?ONCTh_Ap(ixiB$EaHv<0^*EXHj()C(E>@}(I$Ukc=B z(1j2#@u%tMH{8PQ<;*I)+PG0~^DRul^exVa=(k~~3z(;1?X^D}$4ciMnyn*ARt&M~ zOUrmHb%_r!Z_JE!Kjmk>TPqVWzHtINn?9COCsB6)v6Nw2#*q}F%g>VSfekwmWAn8# zI%n$N-~PMT{@<7_&)OKx1^AM}lo9@9_!R$Xd*-7hCN(dAp1XcDW< z4X(R4PK<_|Tsmc*#jT=XEtC`bZ*23Qq=|lGGM+a=ZaxtxFAh5^ZKyo=PM^-Y_>K7` z?)56J8}uR4JzPf|;{SjDu;RXo3fGZSy%opLtW9M6Q{Co&so{L`&wmob_>aI`|F5+D HbN0Uh?g< Date: Mon, 23 Mar 2020 11:16:34 +0800 Subject: [PATCH 170/523] =?UTF-8?q?Delete=20=E9=92=89=E9=92=89=E7=BE=A4.jp?= =?UTF-8?q?g?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../\351\222\211\351\222\211\347\276\244.jpg" | Bin 91707 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 "images/\351\222\211\351\222\211\347\276\244.jpg" diff --git "a/images/\351\222\211\351\222\211\347\276\244.jpg" "b/images/\351\222\211\351\222\211\347\276\244.jpg" deleted file mode 100644 index a005c552544342162169e85ddc053d9f80915240..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 91707 zcmeFYbyQnX);JnUTco&aDIo-h;tmCZBxrCe6fat&&_a>6Sa1@81b26bQfRT_4#isB zr4%d9W9FNgZ|1%Et@Zof`u+3Xnwzz9_ucp0efHgVpL4dI+v(dy00~S9sszBo0sye? zet_FKfCAv*{Ra;p+<*A+!9yIJhq(A8`1p8u_!LCM1SHfHG&Ix{R8(|~tW0$DEDTgs zPq?43fY~@WIcS-9p7O9iWo74J{|gZ;92^{cTzqnTd~$X=DmwQ6@p1bFK#F@m6WbXZ z3kbMJiiJ&zb=wJGykqhnHrCw%`1^r_``{rS_WgT*xot%P0NlTS4;vr%(S4kU*blMs z09g01?_P%#hYXjTnSzF2R8HR$T{T8Y1r&t4xsM76Y3o^hK*eQMcTj_n<19*+uY+nj zMdYnLK7M5tR)FZ}8hEvf#pf&?fOA)VD_UjWbwdA_?*DrHorS-2awjmHyPJZfcMM|P z$HIDe|KS~@e`$erj}-ep88g4!gNHx?xP{w?s;}fMS))q_6iAS~w&m?CfZ(p-T_aL} zETHbmKZyPZ_6(608Bl5#)v0M>B|s#H$cS%PmF|$Z5pgKuPqv_dGS~)qiQpmZ z#t+BU@8^?pFK3+Km62@$>WH~(e#@p*&>U&}kj>Ojd!p)P274O}m)tzs48?zahjQi> zs1=(H-_j;pwK=Bw$g!on-ZpxnfTuo8$sJ}NuOJo3MIrZOkgkrqC6^RQy1T?u6&qaS zGgbL@<_Oi8E1|W=`%BeQ&nrRs=8&Mm@hW%s_~#UEKD3oO@?nmm<^Ax|z{FxA3EH9{ z6Nz6qWv3IDHje4mxdZ*<4RNDiM3z!5DxTnbn%H9*i;82ueCd*VCQMAW!mdNos~w5T z0X9{L%#lo3t~Z96lCru?c<86iAcMr58E*mAA3@hvZUr)>0c6yl3rxJllii3wk$qc| z@%=%enIB$T=a(3(CvDPwGQZ*Z+>hiV@pZuA{S??R%Qe>N`;B3#H`I* zN;>#Bzu>IDc)RbU;FX_Q;vB@jcjLd5td%V_%{zr`ZBI&`mL4oD#EGw zAH+-3Hy|Dr{Ib?`*%Xo=_RmM{4LuC|9R=#GQDr;KJVr5GRAC3l04%yY1$`*tDG}dG zqi}>trD%#T0h&hm@vy?`c9n8~f>n>d9`+U&-QNZ+=h0Blsr)FDZVNkfGGZXntA5UDL=>x*l;*Y@nM+Am`tY^dtfoRH52s???-KB6pQ5eXAXchWd1#k#@|Qlt?`oDa!H!6c!tk34 zl~B$rvv7oV*4ESzWjZ#(6Xa-6ZHjksdJ$wtOHImfksj&y{igx`JiKuDET%!#%JGeN z$I<2R+uGWK=pgruih^^^8Y#p4%zZUUQo8J=2_b9qE+ZwZE9YQXWD-iI_Kam|L`yGDPF(ow`fcpDI?4Y~2D>6p#5r&la|sMDp9MnpLM}-}Rad zGj_TCiD5d~I5DiGq0=r(&^-t_uG=Y-af{zI@vpuGG&#{O{Wh*K>TH7+6g@ZY+7b@~ z0F=J`lko3AtjNNG>RB)S<8Dv70SbD>HL{qebWZ?=s`lCGkh zOlpaI6EPCv3$kffdWI@|I~N7_%xdC9?Yt;jmSm5yLYl+&zGTSz*Q_$P^sM))7`rGppqn5ku+0VDICU_5e6w?45hKh%ARHwg{+(5NOJK3jgy{RCRQ98< z8iHGzzxGa}fAfcRM0TzV3=Z*sA`vxJAlBuySLabL;_eVav9MI_ffiyN>L7i2MNmPG z!*%{13tD{U(+PWLurGBTB67gYC)raSPIiji|G4qGHmZgI=! z@w?M`KM8d{e5LDLmV`!89KpEg5mgyj*EW^imN!*|>V9C|Avt2v)Xpk4G=r`}1!jB6 z#I+~Et7*Di@g2#0W+nJ@xImqhGw2rZ(}y*8ZhO!zqLWkNRfwmhk@1gLy6deE#Xlh> zAa7tO-*BEpUWE&SNVV54aU+`vS;_Tqid`#v&_047MB^G9Oy{0axOYqqm>4)O;U`@- z=f$>QRMwoT?)$Kb(OzWMJ<=p6mFS{f)h3k0|IYe&OmLMdQacqZU1O!nmayoxudWbx zH&0JtJv}_d5lWWe9T6DEbLO>6ZRjy^G?ynOKBFg9=rHDbhzHKk=PeC0eQ3RuKFa

o*CzL5g5v@2^EcMEV9s<;KPZ~eLjD4%2Jew*Wk z%f3!|PxDo}6P6(JTs$~Ipr6E!Bwk*mws1mK-pM;5yACbn1nE8>lr>`&%3q>Dg)&ZFD5A$@p0s+ zMrXH$ZZw#6jc(p^O(?H)IA~xn1#{8;_Rmu5b8p9ZUeTd~LzNj*wd8agMEzi%I5B>1 zBT7aWQ@U%lH!2&Y60jfRMRKJ_Q>LN5N_u$4Er?PDTGEM(QVmr|vK@vGf_`laFnuOJ z8xOtN{9{2Fup=$F^QiDCTL;{^oK59t-N;`Kz0=|%Vw@%wI`|77yr`nk(261@VP&wKrq08hyd>HtSkp3u7 z?$7R>Bz+|}w|*trF4(udeU($tYLViXf4vewFN?kZvqR}>OUv(4wvTXE=+#g>&O*&q z*1@m8H(F|qPCpNEK27-(+U-l+@J}Q)qMoEUN zz7}EY9Q5Y$dD;2+(@UFuT6_=69Wl&qRbsLI$5}7dS{`t9HDUr{^M&{OhqX3Qx$8c^ zjtdN=F)~WvOh8~(Nv*(ckxlGExZS@{QZ3_@Om$50M!WM>)*o7aKbl|XQDH3q4jW;P z()PE`%NyIk&3XbeeuZv=gW=tSH$>)75?}p!6_9;;ao;oRaO1J)Hz37=IPW^Vi*!L( zwaNNI&|jo61w##d5y2Um5VxalU#@Ty!$fi~)d)6flDv&-EmYlb&y|Z8P>^;-#(3>9`7`{%r(;=Y zMfOY%1BMWm>4Fu!M+OsU_K342^wS*b)whTOW(yOqdk$kB#iU0p!ff#>a3?tBP2h#N zy!}(fY;%3--->ur`ElQmGjU#yhZKAUh_E*(i zH!sacXqpq=VX5~5*K6Jeb$nG|k(lyj+XV9J2u>?*k;uQ!i7#tc&>i?{n^Uclp*}oX z-I1zq*WvLphC`5Uk!@60S51#iAmU^4m!HfWp?VGSxGlM~?rHUjWBO~9`f1(G?aw7X zL`0=4DYsW|>+VVxv#1u{0=%UjN1JPmdTwoQ7NOIrcxB9*oD?=$Q@J8oi#{EYMI!3w zf|?$PjcL5+R4UhfjcsIoq{p#dozR_n3-D{pv0vxQ+%hD1F?NL$*$ss;V+z`%5#&p( z=&M;pEy(s#-X5?u=OXSYT3Ydu7y__%f26fG}a2uCM<`sL+Q|Y zj0i-NnlcUEv;3fJV;9zHjuyXc7UFoY&1%Cf0Lu9J&2dy5**HmybkX>Im{`23C0?~A z;fcfIg_5&)dUC0yq~5Onx8C`6dvZKgB}uM0p&WTeJF8)&(~!dwl!=+L6`9-d-E#nW z3LFZMv7SFn5i3a5X5iD%`@#9xP^QV9pDsnb&qVj(jvRL?ANUrqzkVTT{D&$4W{=LM zCa}z02luU$swyqNyrJbrD>?FN1J(Bf)SHx{UA8PlegkT2CyPX!kAW~C^SXNd#6BRd zd@>$&NNgd7TAkGwIPq~}{Q$2OO@uTVsjuTg(exzXj__6CGJB!K1E&EKtO6JX)%+j@5wyBa5#+R1XnFXv5iIH=U82S|?{;SkP5B6omuVU8uYCqaGdQ;7_Pl%_DZ&-q2q7o&ax^S|1ILsM= zzr*4^nwmamEJf$!hYu?LKpNHQEDn|ga1gyE{|I^>=SEOTm(Y;jJw%o6avsU_xO+ZQ$P8IWfF#^#L05SQRmQ>}7*m@z7dLlw@-!G*z0BZ7^(p0ltJ zNdTWKY%eX*!!y^3o+LDAj?FyoEZCC79PE<0W?|vo zFoC)ljTP;HnLYo!s$aXWVxlE?FnmNs>}@vzk`g!XxjOq9K}jamZv_7=8jS5iEeye%cw4K1Ro2c8wSkYaO2 z1-;WMIEmo^r-1rxhYb^galR%mg+DA2Z_%Z%ib|E3zFRq=Ka`Z;WmNOZP#JJr5(*V{ z3XxZ;$8B5bFZU|HMl$Z3jhHna)YkhpkN{|${_cuk9jRFvCO&JUrdbjtYA1ArAClsr zF^I^1Mm^o9x)ViiquYn7y$bec8eFT1V_Z=dddd^O*RLEy60`$GpT-^1^Ic}--U60X5x}`#h#dZ1Vg8D)>W`1v;Az^a=uY!+(#?Wr3s$*S zog}4NRAcupBIulykoq#>4-;O%b$7Qycb#obSirNf$?`Jbacbv#088fIW%HLacDDdC zNo+-@q@M}9PI#4b73FZ{JK|(3!cR|t7E#{9dmN=xttQC$y_5H$K%vO2^ia=|L7g;k z%D(oD`)Lr1vSjGQ4^^&|25n6s<-H4Ya6C5*>Lv4Nhp1gqKY|1bEz+}=*H-ip{HPvr z3%Dd5^pw*&E^*2E){yVEHLX?o)ZS=eBoB31up@bmUH0}Yly+@9-m&FnM{{dp4C-6H zeF;K2c72Vjw)%{Vk1HtVN1x><8BG?) zbAKY@vOvKDm}mstQCtm~zAfjY`YJQB)KaXQS73Vx6>yZ465l;p7g2{)eXbv*mxRsLb(P^WLGlYaRLx-HFV}>2>M>%Y7yM&-kp34OVZk^!1gjKXd^>v0kBxrNeIg4}ZC0yE@2Et* zRxSYjrr6a}aX%Inw-lZhY6&&15-oAsLr>V>0^Xj^E`z$HxSwOgbVlJwb_C{|%%Qmm zB_=mF4Ezzg+uEOYlj3fwp3G1fLj*2bATjhA82Olx9P z=G%nmJ6&6%Z173;UcK{zxo-g%*+YO0s-%N0;jqKvqbS-e8?wTtNKYqcdHLFcRHJ?| zGE~zoU_1OUo4Z>@P-wvdV>s~HL@|1-pOmZ={kmu;51-AJOKw+?r_gbGI_DOU+e8qO zmjh-@)6@og3Rn-Ra~iSNP<>lig2|xxf&$>~(UTa&K$9`hj%0W|`DvFL$d$i_4zV`s zMo3;Rzt3nCmvlr&67-CS1XlK*)bA)nF(nX2g!_~7U?-L24;WW{y=uOTuB4GqPlrF{Bckt=2m#Uw@VHi=iS&LS zEODgvsA3guqAvZlP1c$maPG-=3y@%|stOtDxCO}Exlx~l&j)~RYPsD`!_bsGGXOx)7;ZE#hx!;6lThSblVCt!O z+DD(meU0@#lT+Sk2a{&E&CU@2S(hD9sZH%%ZL#!u9{FxLyx?ZrJPnw>xT>Jlh*Xy) z+orQ9?BwYx-9_itV=#x5!R_B#*^2}A^UlZE-zU2z#ZPQouvrapd@@u21G+agDgCf! z;>juf_Tp6GC)~3~QCkBD(&i7cs!q+O;zz^1-7UrE}B#~1s+}TBVfKd z80QYHE@Kh8rW}?${)a(Q_EQ~F71wOqX0Xqup~tjQE%H8Emc-!`7%1y6n-ul2mx9ne z`{)N;o5H?iw}8TP9Er|><1PE0IvLLz`>rgdIBm5tZ?(wQ9G=heGoN5lY78Ws7LNZh#MG*ORsOB>mf#|AjnRAs=jYEP30RP+eJ=mYa=GWzv^Q?KH*O_aRg|$fnoDB*x*foM0Dc zF9LUFltL%!2b^?>ZW1ZGpBcHI16py)4z5_XMGA(dI~d(Zy+rjQIlk#D zJAd*5f4@-c-(oH{5X#*kc^S$tCwE8~k z$wB8L7c!CkVkym*2rG_NmYQN&+l71AXey`5bg0uDPSW33EY3aqWcn_{RyCXlu5qDb zC1Q_r=lWYph4KxFTmIw9ueE*4q$P!B2{mAS|SfIVFPSnz_$@%4Ur3C-a}Jl8r4UTduk+$|WjLr^STtUFh_4;Yx%(51py& zXu`L5yDVh+$*um#Q_D}rbY@Y5Jh+rK!mJ~ zmuwE4<0?}0r+7CkVXxfzj20$X{e*4-hufMruQjc#2(v{!6K&TXpF>nZ4%^OGelu0mP|kbf=fV%{|<9@0{vLh5YPkcpfy`zRr(b zI}D$%j8lO69mQE~i4U0TDi5DUF`%xR>DNF&64G8gtoLo_@h6%;U6h*F6KxWT(qDT* zvfO~l{;U&19dLw6gwYHK)LBq)mFrP(j4xdG7Vty$C>M?;w8GC-G|{f{Emg;>C1qdb zL0Pd6p`2&EJ`#nxQEKFLSFQAv2~@=q`}?zNnM>J1!7Gn5D9Uf z0vQUqrc~#qILEG!Fj~WTuXo9;*xU{dNQ#X_G+IlGglTD!=Hr#!yijxPn%+PyGR$$S zfbf`9Wc3yYtco@*ViPDxQwjvym(|UlQ<*Z{_YzpD+GTEB@$%;^KW0!RjU9x@ zOU)hZ=O)ZW6{6#F&j=zLRrEfyx(=o0aTwYUBru+?WUoW*C&dmElwb-G(xG4LKy**0 zNNq$T^Rl1o#PSIg7FrknDs}@KAwJ*Fmb~cS;%~B#>S`&;OSy~9j3$MZRo3u3tl_*o zSLxf~{GmOwq9OTED_T0ffeSt(SV(W*p1XIfv}vDWrqs z$aO&Cs1H;-tL|{;mbxYto!syB24HN+{NX@9d-;f*Ztn9N0;y03gxsjF+o~M-F}aZMA1QeLbdzUO>m=>E!%#R%kH9* z&-)~QA%8Hb)evyZyl$uBFA6eF3-#p9OxPxf9DBocF8gdrvd^mYGyXdqw?1(-tKW(z zdvqtDD;tByYLDuy$|jwWkSmmwBe{Nhd*h>PZcTy z1wcuuUxsdI_k=Fwv77m>Gs4@uP_97_@6A>%QGqlahlr4tvVMX!fLCA>_p zy;v^3rL>KA^AS(IYxsnTaNgUR_rC@FTAl>}ynjvp?F0YU>FJ3n_n3$r-N45enAf8! z(!(VAW3_pvLDty_AOG6HM`Tp zOPmf<3-m=eT6JY(Pocx*J7Mk9M32Te;O9xl+(vJLt}7sL5y-Vi$));wIGYPt*6j{^EK4^bV$s`ezrqure7*Ai!mi=78 zh2wruy0+k~jfY^Bd*nLrGB3mA@n!8%99s@B%}p|GQ^k`@>(4!Xn7+2aoK@8UEdrMM z<>IH8iZ&?XgN>q}wNGz}&zQgYBB2yzzvltIswr!0brH7aoZfy2M2gYIM z-Q4R|;u5>M9$sN@h{{fw*>Z+l-m?0MCR?v(+;Ns^dO7wQ3ni*)v^jvwBxd}TsCXbx zk-d_7E9ye!D&9}oB;##&Iz2r0P|Ifsk+nkgWOlXYPkyvsOr_RcVqbiP zi!2?58=f9a%3@*1Z5=p4x zy5y9FAAscfV+F%XZ=@S6ELUu|{kkB`^@EP0f{#4}pswqUb2f(JX2bKUeMrTI(%Ay5 zDRo+(UR&4|&MTU|ZpmTI2r6qnol*1^2(r5INH?#*v-fqX^L)_LN z=D5_T(grB7(*W3%h(ma7^($_P@QswJaj}m$A*l^4M-2li*kmJ!{+W4v=&!j7SIenP z%YzJOEib*WLgm{u^}4jzXKSl>NdCYRHVk9inFf{7JF37m2}{_g>!&VQtJEN3V+3I^ z)qM4H(V2GzfswO(IOSDV=O0KnK)5SjL$;FyE;jMzG0X6b+t?Un4 zKc=U@g7kK!l* zBRRw6b@iq7b@SLx)&%)7dmNj!Ejq^vl2z8xt5utuU94J2yqAq+X4(`v(m^J=2C|v% zh-kytm{c3Xv0p)8>)mc1s90i+EJ?L6##XY()|68^-*QCfY$vofSQ>{dPENxtBt%JG zWNAdHW=XhkqI%Y@_S%v!VROq=F*j<{kcR#cq6ivj?RHE1FzS_U{>@H1M0U-QVGpLP zs%|&XG{y6n9<70PzmMFvzdt#A9^uRwVkzAf`RaXz{dZR0vHU(%wI?D^i7cV79~~F> zzHimW-y9y5O{{2R8qHe@O5`;D(6;P&3#c5DfB8xCNrdNc{ty#y5l>Dn8l=u#6ai}( zrId1JhJP7iEfMgieIrCWou+)*V=!V<^Lv#!fJ5RVeqq5P8`(F(_O5yjwz0T1=r!mK z%Y}*_x9`^^7#$o-q~K>HA~E@d)TGE>JGN#v{y7L$wa54TGmGcQ)i#)QTm<-nNnPZ3 z+`(_&bC9MJ3y5@2Tdf|G&@ObL)_zuNi*L(D*8LCb`S(k!n^9f!AM{ttrrj-IZgqF9 zHo~`jUstzG+;{3QhXKwa^_m4iZx_3X{RhE1G;Kr-!(IXj)h_B9s*JyUw;NM)qNyVf z`-}isgQI|d?>}2qB@zYL3$W9^36`R5%6_9%3lp_<6lzh1S)87aoPI7=PU*;TA7SOF8@E!DG$2RA&;S=UBv+ ze9`5b{>VMZSZpzQn7fYfUrghGf0)KeRV9y9$(h%9&aA$se9+*$OnF{91ihJcc52ha zJv%VRxdqt9;h_yeiR?bnsT#cj7qCYEY4tSKfPcfGT}xQ_w;uT>p&rM7x^{XlwSthYDTB z2^)jmWoVQPSeQ+>38^!4Hb|=k`jFaZqN$>E!SfP>P^Qb=GD{c5-kQMib{=P%&VhcOm0TY0=f=a2$lE>2o{XZ$nmUz3@jh>OWVW(Bi6A)AkJ6J%%S z@emJoLj&lfd@50Cn!qf?VT#LU_s%U@{zdO0w*VZl{SN((&Ir)qcBO>9etN>`~&P^a33LHPe5>9AWU#s|{&O93)gium8c9VhiPjJryXMzxza#r

qVsND4ALb<)Gug%Ade`XySmg7 zIj0br1Pyu}YQFEO2q=v7(^HBVA>~)~N=^TKzc2bSouka2ZIcy=oSRY#Q0eM_ZTQYC z4x_u5hmz%ZdXVoRa7>yUzVM~H#wfKW0%Gy}!M9O80bPq~9d^XjH~K!)E-o$}qV#qy z-_%@vnddDy?o6OxN*BhQsSf6x{@{l%=7QRAo2~`sXX?1*=wLr1jK7`4tkW&(c7G_u zB&%`>8vEaeNaRgH6nu+hx4istdIG-iysYJM6i&#>ugJ*tODdlV?aGR|RD=h-esvV) zL3fZ;CEP22CA`>%6PM@QUp{5mdtEy_Ba=?L5=bL`DTGn8yb|+}p6)a5Z4wsee)IXL zc-cK#%uKMK!y14i3c>pKndOjfQ}OHEo=lsc6bH8JVL#{whsy_7i$vStO=kYvMCKV6)ADsF7pH(nHyHbeRYA4@i4e<2&Wra)b`#2EI^+@=4;$a7ffz( z)T7b2fTZcl>HbjJS_}ojr}lJ$`BBLnqo$LYmU!^gK#DIfeUu|k<+IB5e6Lq4SOayj zB_We8(t72&>5e;;hi`fMNMEjebd;)|5E{P)^cTsVk4i^HiUi*R&QFxDXDWs)ExSv3 zxYa_@4NEZ(T^@*)>^pnah_hYV*HvFjv8;UcWx7sQ1H;W2#M5`}d2HsBQxhbAo=>`o zD>E?IC7Aeg{I=IWRx>zwr(%}6lLE(SV`HEspAlw@-Rk?_>Ru>T z@YY6ZOyvrmiclPwogfv$UGTpGgBB zXTNsiG%G?6`Pz$>eI-deksg^;d7KkEd6&uxa-}DwmYY=3)t+!STmf5=nBvW?HY;wa-;H>T&x1{4?b;$PCwilB07GMvyRz8l+s|~ski6~yQt=vtvvKfAC`Fu10JSPGqk-@zTl^67(@uGX#91poopT{f;FSyPOl8R1DKMa>>W>vv zq?+zg02TSqszt`{NVloP*D7lp*GSjRh@dD;t%1}}If7b&KXp{C3=J+QQs&#lVKd=P^lvSe?We2O*1C=x`dQHY)L=f`Rj5;N;wUM5mUkpEVb!6O z6j6eD`iQ%={lX=nV@zIS?3A)6@@bG~?T@G9;54HPvX^euH(boTI_`Kj!ko4ypg;#A zyo!%=HtoLow(r!2$KEOLXQRGMXdCwG&V2O8=b>vLv2#uhQSPy}R#M)gNwJ-A;6k*= zE(+UVK@14h@_XMd>bEaV<;P)E2CZ04(B*e|5q~WALWwC&Q&k|C4ClEQWoR^`gIkR< z8VYM9M)fh@8$34B&Y^M-k7vcTd!5bscn9i3Fy>`Y-U%k4?ilX~w*~93cIFd76gfFG z$c?L^$=a#829j4qOt}t6M*Cz){xMLRy+YH*uqbz|C@F`1hx~K5Bh~2Km?g^a-Zo~& z&PmCNHB^h_?=K@emG~hL1@(lk((jfqnZSW;mSbm9iXFu>+}_J7vh&MW_Xbl@QT`6H zsTtb&v3Ofx1G8zp`@C9QGM$c4<|LGdNR*4+T0z8KvW%AdC7kvEL#J|j5GLi*{*CuA zl${WyCtaLr6Nll_ime)E4f$y?4zI6cj7y5Fj~@tMb99Z`U~sc<5w^ridV4Phql7{S z@px;^l3NOM&S|WB0r?7EPHghfFK*3d6FCZ`^I8s&=Ux(k>Gd%7lDSj1c}^^ImP=O; zx1wk?al%|HRmIl&*PnWa74&|#)QvYBL14v z2$GYGuS<^IvGr;)x2I)9+=Zlh5T71YhLaQO(32FNsM*5p!BeLd`sgRo=h`V5c+&#| z=a1gQSVX#g9Gbqp0N{?pyHr-*XRZ%{b{puysyB6Hng@cExj}zoG*VtWva{4z^We!A zSjOD6-)G+5WjXhdeX}gR+A`-sd7=F0FJ%N3Sn)YM@w!lze%n2}qgdo|sHgR6e9O0z zn|PyFmBT%Avn%*uPTxFxW&D(f=_pt9M@EY;*_Qv4eGwRSJ2VtTB`>O-LJjT^5MWXc{LRqz z6XSOw*^xSXe!euD{LY0x?ei_5gx+we#Z#2S^sAc_L?94otgRGJIn)`TG0vPv4fwe} z@#BrKJ@)`jbeIiWQtZm6(89RWg36b{!t>pHLO1x>H{%fHb|VhX{^PXBQ=$uFBx!^( z*v$Wrv!IKygnpU-x$V>EUBLQa5c`;_eP>|=y87orC*oVV7%Rt}4pyZ~ItU&gBwmMx z5ZnS9NbYQ^i}|a3>EHd=wFZCgs;2h&V>WIy^CeHMH*Nv>p*>hV*gJ1Ogrz7ajq4ID z-ygPeVO~wFRZ>33RZs$zG5rvlKn|K)E_s+tWdDgtc|$vvShA}+D2l#Zm(BJ8V6Dhz zzW)sH{^0>Gx^FUTEd zuT@lICEIMwzrh~v$a2&yQc|)O$izN_Kp={h5hJph|HYCFG`E?^@%}19A7xUb0k@;g zDp_pF-P|8zu?OTot`DjY%Bsy{hZ>?#Rir++{_p?J-oL)k|BI{pjxp-=Xh$w!YdH_T zY)&scVE$PycRlB`R6iUM=Lc)`Gg9Wq?C5|GG-V*^=Rq49R{K}VE$HNZ%LT)mUyA_Z zxn6B$%&rXh>!xm^m8+Lbe44jMVQxu&A$!QP9X1ca^xS2mg z9)+NQ!W?za&7sphZSSxBXV}`VxZCz8f07XtM`YWUa%}n1lk%_{5jwZl0A=WX`Q&24 znjSOqvJyO5)V-KeXT=LM!W0g5Dz5n`KlTw6JWw5ts=ugTO^|QiltI9So%TTvLBBVQ}pGBubs;EW6s=rgdT{zHZsSD&rvHsrhwd;4L6wV~zh; zwBjw`cX8V%(OZDq6M3~ly5}0os;R;3k@={JG7avVk6dXLuil1~;H|_HTyVaqO(t1F zY27{S3DO737W|a7SKD?tLUdXj>@TN0&$x1Nb5z^NXNkp*#hE@ZA1Bj~*`q5~*>J8R zX?nE-e;4%9(sp$F{wTIsG~CIpQy}hCW#ASdPMq>a>@ULvrv3+gX&aE6cK~=Sc!B%B z&?Di#=1ec8tGs$cg#t!0wZ~Qheag;xS4!P9&;H$xqyGh!{}~!D|XeqsvV=SntQYvdF1)T2#v!$3n$FVR4I>(W!9x+iT$&Q+GmLT zqxT3?3R5Z7hmf5|s)d5Q+L8&wMuZ<*O3h1hXxzH-=@^4B%qX*WcME4)#E62CK;!wb zP=0*6nO**l1)QJhj!7vNd3l{iAb&3Ao2^J>dQ6T5UP@`qig*P(=Om&(F&HH}3&Q z3q}8|5%)Sak|cIW_<1`7EBxvw@@GHS6o#*6l%5i20u(88>fEh0l?SZo5*3hntdH<3 zxc9wgv)f*xKwt-Cf8@Zx5N(a1taV6TMBA>e=2iEdt3ZglGsAlF+J2;5<#M0dO4ZJO z*qlwM)cr`uv&rGhQd&?uVRa8XVE1A;1d4{Zeas0*aT&xxh7f6*agA3?+VRTc4p3w zEtA#K(&-wo9LGk{eN?p9o#4l@k?ZJSxG!-on0A#do4FrgL>NkF8~U`sBm~iHWwX&_ zCf}$sUfFi<24p!L6fzCCC-lz}{XYqxuPjc-1o}|%RC_V5(ox+_M{gtj0C@c5H1Ve8 zTQ9?nT^sH2RD+F@vwp}tC>Y)0#lDfs72b(3&>cU}<1Xr~eaMhFLJSYW_mS$6a-1C_aQfpp`-M!p`4``dOT=i*r?JylHGHjYV^!+` zFE6WWr3HyqRb~Sn`u)sI_U{r$)P4Thi+q_*&#=4e^2afEWtrpU@j!GW1n_?=%e*Vh zRP9I3A8B@%NR;V23 zF5ptnjs5s}7oz{1cW(d6nwB$$!ADd4}Z=Sy{PXD0o z%Zz{Jv5^jB*2tY$Zi{)ia2ZqXrmV95yCfp%*-o}T1ZGxe|2e#?60P_}Mt0^0|B>A- zp#0Zcv&yJ+#?x#PM~kT+a*Q?1knFYzq1oje583zDY|Y@`=A$M0?`WV0cblo_&Rj1Z zdWL%u;j5DK#M4(2N_!0ThYK?+Db_{~49A8pC+013OzCQc=^%-`r$a@`{Fg6q?<)FL zW(smkpH*AuvGuum)q7^(-2&z&Aj0AKkHLxQXT3c38ee6t)o7G+))H?4ZN#PBH?rYS z9&kJQ?yc{S!yQi3HT6(gsUYn@Y^+onZ-;(^(#FE4(6Ww&8J5p0Mx4Qg~ZutfOj{oK+_PvtQ63w*os#HTj!cvok?~2#7 zQ)16))hHq0gmL&cZAwgvPJrMf3$cm){tphR2CCs%sepxy(ZDf)z-O!O%OY`TEOSyxjKI=+Kd|2|x_-6Bz( zxDax;4dI`_`{no87w|$%S7oxuuPXl=B5TCW@deX*#9jnx9(Mc#`T|=7Mja&C3@L1r z%Jvif1H$6^pLam;fL6b^L|krQtMSL&J&hEJLG8K2zHS?vJ7dr^PzG>MB*V%R3z*Uk zTaUYj__BnI54wm2Tf<%B#{yX{B7G&7p7y7PP5=WE?ipZFyb9{zb!XBFyD|K42+#CB zyp^1wsmD*A?cCxY&qQl&FBfeq#p+>Eu5e?af!zbMlGgXn)l|4JUYD)^S;IGf0x zGvM_z#d?kEeML42&G)<6EPSQ!tpNcP)+j-ld!*X6A3KB!$2`Gfl1%$|vclp5MCBST zgdX+QGL#$1e*f0htX4g>jL`7B%RUy}S3F5VJ-buAhXv1<35Dg+q?3ZuUq1_i2kpmG_W0_|)G~y_X@!!d z3qJ(C{OLVktPe@Vn*rGdzyX*4JCc}vsiu(cn5UGVfZ0C^1f_HpdX{1;5|{EsTXhz( z8UTc3HC^E%Q7LbEb5i%&kL>vz8n}kM+1@dZ1XDA=|M|~dfV>HQ;`^}&Cc`p*kdQjZ zm3~4(hvF6`*%AFaPiX&ZNYc>tmBzuVWAJs|K852LCe$_OHLWr3>dWi;hV}?K|8MCc zHkOiJwZo8>hlGo$*O$2bp&H%#tLp!YySEODs%!g(XXuuOp_DG^1_6~EQW^wW@XwCX|`C>OP*>Gmc6xl%la-gDZl~7vd*kF}&Dh zG%a(*dD`~#BWC4vvqGEirpCM#0Eve4I&2$Csajs|7p=U+=mbB_k|1ccSb0Ttvx22o zVf{w*i7XQQ|HIh*Z>9#=1%J38(NXc;P(A#zzc`9joGwiXMvpg!=+-c1j~B)$R8Npv$rt(P$OrbtM5Hl{dQ+7@GDkg8li8+7iUxlsPU@; z&4&5dlSI$wjW=^wQJ!tSo`D)%*ZsUf!X*q?qnOUF+1)Dic+32Q^WeG;KB?HlCyM=; zk`r$Alp}0+?ct|Ajqgd?P-4Rl75Q3;72^4lZgdbgPczs$>cqx!d`Svhej4F-7Z>+1 z*DWXGxPU10@xIVv{A`TpZl;Q#$Ux+@-Du;L`SL(%<&yT9;oYeD!`R zXT>D?8FWIBVtwtodNl0tE%O9$pPB4WFHHptPT$7avf!hQ&tc%w{K{XUi1o?(4b73^ z`=!M0ShdMjn;K})g~ny8wNqiHEi7|8)dh^*BkpL#u^eYysefTSDQ)0KyM5N#(7|xb z#m6yg-#SIwb6Xo3a`@W`7-D3)%%UWrIDw=~va?z+$N2tv4&zyi0h7+Fi>LetHD!bL zB)QQAde4xV0lKUy*xDjZfx^)VIrps(lMD66920+XWrB0Z0$ZHL4eK0H`^4}4WI zg`-E2rva;ZB$p2_^34$0_49gPRKW1Nqd45t=^W}(KA8^n<@F<1$C32?)l zSlN3~-rQ>rt#)k6a#rb)--v24gp}^Ggyq0CZ)IogScJ6`IQxr-5<3IEdokK_4@n|p zti`gl7?Kt82vn&wqAA`v$?CJjXiu8H>LMilfn}At7ghDa9W@gAB)bUBGSo~@?1^cI zZq)r}QnvcM^55~M&>ozH2imhxt!WzSVMfuU7HJut7g7{894Q-(Q5Mnq_i>xGB1Eo@ zUj7t%LYq8isO;q9@Ga#F4z+s4kdsqZ0vB?+WYX+*;%QmaSWm1FTFw_=IHV|R`+;3M zdn3ca^7Sw`#3amWo>xYN1!<~(9A>F~Q1C=XgrFp-FQJ3$*Tt`~@kLTzWI? zcd=`-7tgy1^F2hXS87PbLrHtMkbxoc_8vC@yCI%%^@-Z+ppkIFKCL5=1x6jUi zyWPuzy4S2fI>qNX$+0||TC!F)^S@9X1pdGZORad;;53W0;P=9vI&#lTsx(xNC$9M> zUtOB7ZVRwz+EiMc(6#(+mg?s-r(~Zv+{h>N>Slp&7#m>5+8Vu(Cnsw^rmH0e+ z&b^L|Z=;h!26#)Z(d$gFWuI>Jb&^)HQ-t@L7JT0uX;)=Wm*SloxGv2^i(6|RFB{t! zId$E6^G3bVj`W=?14aRNz=nkfJZ=adqL6G0eM}IdM5+-vgU@8PV)l*D`gMGiFjfQ7 zqMd4fUAy3qnv=JU5*e$+Q(IF9ko^0cO#4TkX&QE~?WWmJMQ`0oa=A^X$Q2eGW_YW# z&FV58;~m+on!beH>vH#jqUrwQ-m1i$O^zS8Su{RbU}?z+(DTxZ3C`7`x;JkLS!Mva?{6mGPvmLn{Vv+pqRM`z*R~!yNu6 zqv&2!Zv5~q^UGu0xhZ9-)|0;SFa3T65ZlT>^y=w_F^wRjs?9F07nyx~=he8T!A#uA zQpV1{V)>$7h{L9u2CFm*74dgdu7`Bgib52Mskm@P>N1^WsGgsJ^7{UPaNeBjT&+<4 z@bqG;_ms~(@#bxBwBxUan=PjW{a+CM*CP8Cka!tBUMWsgGFno(uN?Dr9d6o{GFI+u z{TIX!lLewP?bNB)L_Jm*qnOK0V4cS9K+!o;p*3R3ZY{a#_wqXH3V6#LXw;gQw1|}w zT!?$DB`@P7j!n6B$4cC6dQXY`w~c0Gl++~op+ZX?mkgGNm3o6wDoOvT%^Q2H_4B=@ zWcVt2oM8^~*OD5$%P>mX8@;fgUV)|Z=2MhMkdcU{#@jI-NzE;kns0;`0|^fucp=` zi+#$y(V$A~qf8E9y~cA|dPVuwPF`4)8%#2AVJ}3Z9t5e+pR#qn%y`=)+pr;?MXl(6 zSWMI63Gsw{aKQ5U%p;?+daJi)ZyUcjSjrZ*td>)YHzjj-&QQ(CRpTJiv)O=~cn|DD z+;1dgLZ*`Ls|~2FP*ovBz-?drbAnma59RJ*GG$KO0YL%YJStKf<9z%R46#kajmOXF zAmEBFMY-~OZI^(cAuw3*SXNn*Ss4KDR&gUz{S7cV!lqb z+G#7aG5gg1fw33j3V?sCTI!^!4hSr|Oa@DxaSW`i8)l7FOp-sd$A>p%Qe>u8A-<8I zaTT}8@ZKHYCcY|d!od-f`MkK}(p%+|)dD5LK~%dcAJ|!C4FZt8fmBUyzjMi^DZ+g3 z>Rsq_?OJ8Iu~^3xP2Or5IU=~Zt{R@u$M{Wc1;DlmT@-S!^oy{~9@f#PlkM5Pz&Y+0 zWpC!7H!052CKt|2izS|zn*@3f_5ween9|-IIP84AsCyHRp`h_HM*9kdgouNjj+0__+Vlf{VEtmNTA&luq) z3M;*0h2vO8gOV)Xk5D$h%-0JJ%&Q+=1U(;n3~JzS++OdFCb`y@?FN%dm3;raq3&zq zA%o5#V+&GqsAQfbrOm#TWU|pV7I=OCO8I%7+$qb=9EzowJq2xoKybm8w_k0r#BnjQ zY;DyOUF5pTZR(u{Z<+&3nMGnZsV1SQ?Ag!q4c8%8IM{a3a-ORxAC_~o(1#00xb3qz z3EsKWL_pm{u#JKJ|EiC&1y~JnmWQ^-q$&w`a8ET2G`%nXvcg9?#+H`gW#H!Rf1dp zxQ#8+UwAl=zWb7^33FLDRbRq)pPy}otaD>wH<1zg_{eFP*K*KAXjv~-wpRCUurz+4 zO8THTW;qna5W*y55>3YH7vCweGZk_4Du;@i9Gr4!;UCiFs-z0-eMyYRRqron-*urG z#LNAa`}^@XfTJNNTmC!er&|8Gwm{wa!SmQ-?Dun_IuGGrx*v)D|9V;KvvDBXe|o8T z_N7?Fq+CwQXR=Ash`dNxtf-^1*Aw6A)O}C-jeuN>L;1kDCr`VRysOQmB5n%CUpZkg z(V>T?K=7LqVlldGNr-mWYJ=qtJXy!M7ix-OY|DQI<)q!Yus4@Aht)!qq7|5m^0)1wgUS!1*gW9>2;Mw`6V7FT+@ z+M@=?!n#h8Pi1a?<_^EYlY;zPXvjlO+IPYE>D~vIc&A-tb6HN9%&U6cFD_qE4|Ic` z(^2}6tdjri=7OP|XF1_D(S95D;H<*(Sc+Hf7a6$Y)F7c3FC~)Py1~R35^hY!Xt_wI zy=ZHIZp_E-*N&~4IhRo)o^)@NcjA^oNQy(Je$c^*e zL*}lC=JZn9*AHSXQ)`u6j|=Sw-yMY#(%4X`^}nj8pH$p3WytF4eVrbI?t(VAX zwp88J5vNpxH{U*pwdTGLG{@;mp6){(M^-=_&7I<$tq|F3|1F|~ATPVia*dPG?m-;6 zl^9(nvTKR5(r@1wt5i_8+GtA<*1@O)HNZ$ijU0~cQ#-AFF+?JNI|5&|Bj56IAwXiw z3S{wC4e)gzH5ABpb-q?t8)h;A%K!{-m-n52MEak&`XPx;C8_Jpl+`1l1?@~d+ z+twT9Nu$L<)~U(^0RXn_@YU~9>(6X-^VcZ{Il;k~!gEUJ!HoUyDMX?9>o6nvvi#Cb zX~ARme(oQZy+M?KYk(viaP5PRETDl2=N@?1UX^TtPJBTlO6El59h?fIZPKM})3taV zW;)PNV52`@oz!;DuTj>(q;9Plkw5wIn_W!k&^i9L#Y1y1%ngA^1GUi^sxS9ffAJ9h zbfiN^EVUHmb3BNs{=adqbvmH5DEz3xKB|U4q?g(&LbLdE$-Y=!^KJ|;M|S3Q)GajsT&4RFQ8lv5OtMiBY;m=3g(}KH-l0~jkc|rtBOT0r z1=qaYHhV*`x*M>quG^v(yd2apoYCdi+%j6VVK~XCy3CMugCM#?XzgoM9z}k%jKmcF zaGmi$t;ek?2YK~4pT~4!$&n^9#};2~?qs_lPLb%c0UWtiCf7B@M(iOu5U0!S6j7;rK6$IPdJ))+d@$o`i{9KF`;@j;LOj}dp_Iv~$&WeP!6c(p2Yf<+&2f#U+a+A8)S`~nlT*ejEbU?+a%tlZg?0Axdd)6-}oj5v);R=dFW}X;<^c@>cX@rrra}DE&N7GV ze2j2SS!vTNsW4wBr|I)x%DeR|qxc5x^(D5Huop((Ikw zE_#=(-xr(8PQmWBSV5ruUam_%rR(TufkM5AE94@-pK0$u{9r2stcpjT?aZyW7lDWD zdA5S;+kA`Zm*o~FmtCk9xYhD0KDAA}Ts)3duy#{Ws>9K`1@%2>Hhd)Ew?GQ(B zmGEKq$=mA?-~=%T4V+$7)%2~UoadfOgKtC~ph6&Ie?h#^D4jyY)lTQ_{90`tn2RK* z0XxfkC)?yU{S05_emrVI#;t4101n~yw;)hM{<1bK%UT)!l)Gk0pF14TzNXyfH5;8; zV0ht`-^@M^xiGQ*R+vO_kmxAFlMxJ0P+HCDbv{FKx6HNTWm}S3DS$B) zF%mdLGySj{<&Uk8k7pC04tpBhduLoSw0x8KrG@+NV>6@$Y$dE-jHFp0zhGG(@mnPN97M~BUOg#t(LCOyY6;*g1zf@KZvNsorVS?eO=CUhF4rTU^q?J)< zRf^bD2ZsEBv}M?`mzK|w@|`Dd$cY*gO4gxvWLHyot>tbC+guwgh(8-;dS1x_FPpP= z2&Mg=2S1OC*AdOSl;R=iAKh)tLQ7;imHIz(c&BV+YZ|NANMqU z%h>n*v}n&$y1%D)^da8_ZaK16nN*mIGZ{ap63no35-lm>B&=h2clw+@5>eelTHw08 zJwIfU5+V{_a<5)|n#6D#xy_CG_Ef&+1GVIw1C48!tz~|;9nn}wjwTcF7{8Vwb77H( zXr@U_L7Ys7bn539nnNlZ+L{F;uADdNd5m8nrZUbfSOUrv6_7Kkrq)#s)l=>3<`mP; zcex!z@G-v3O|Dw2yKO0T1#@F^2gzT7v`?3GFE8uXE zHmhB!V`WM|Kl*Hk=n5oX48-3-WIrq%!rBj_`$8b7xCFs+0`XYfbEvqs87b`x0^OGa zk$q0?|K+D+d_sD_(kq&k$doM8DkIIt`Yysm!n#I{p|mRa%f}F62}F3G2W3l^dPhvc~4xY;S+5pHt-oV38)EOSSBIKyD#(t-|N z5VhcL&WLv1_!is#uLI~cKa`+HF4HtL+NTsrEp0MVX?)?93%e`z;U%6*VNd1!ke2LT z=GDCs{-Nv^Aeke`p~xdrM`-GJI4+5v+R-?i*gX(4)@{Cv!V5t<#~%NVLu!>s57u~V zjP2D;Q*CJX5bUsfse6;x^Unu?=V`*H}M+~Qn16DjT4lzO`YwC5X7;D+dYW(VLmVfx;px%hlg6N&pWw*XCbt>P8p{Cf>O|5 zUWhhAG_-l8)k{{68A^E-Yk5LvW6|YNR%|mVvqnmO;Vep;24d(}NaW9N?~u+wr`npeK4O9X(;*5VV&c9MJg3B z;|3l4P+T7I6Pr=1DaI;gEo5bp5Py`dxGeaSSRn1TD;)|!Dm7w?sNBeSmEJ$Us$QcT z6{5dpGvvf8Kj62Sit8=PqM01}6(wV~r*Y8XI(jOG;DimD8-Gp=f7eSqx{lUr&^Jb(ggZN?ITL(wB8uLc`>GzA-@Yl`hQ^o`QLu}Kcw%!{t4)p z_K-+&cjUJ*Yr=CFTj4)4NAv_9zSd_=#Cr)~v`Ot+!_SUlPsL7zaRfmumJ(zSlr|&1 z`+{H)EP!Zr=rrVNWeRn{NGCV#YsUqX}d|K`Rej;s_K>dblNRc4pzNcoEqihRzE&% z5RE%;=5j7ULk{uvTAJQq-jYB(BF$QQGNwHitN@Yi2k;;-}pr) z8b`I8eWvOZYI9)o!~dbTct7i_nO9q+*CE;`{|cfnm_RQ4c|hY6O=2c$4YrtOS!tPy z{0{^rUcvD$1SB;8bna|>9r8~|KWR9arxO-AzbvrT$u|`QEK*h*4A4kzc_({IEa5DU z?H6I*W{SID{`Mybk7Wz2wE96HsF#N?1JYxGXT9a9A;U^Of>Uc_zPxRu5Xr)8Mf99f zwWknln%VeKEWkCik@Z@xUEpei&pFpX-w!n!gnwH>q?n`@qDKv)#d74WVH#Ab(uI65 zX9fbrzE;^)^(aBFSB(QdP>-c8|Ge=a+W&U&5#o3Yc@XmJzRLU7@7#KhsTwvE+pT?Q z>)CD(A?mxAle3$b#}p$ULRFq%yfBLQit=_JoROqVN6g@QGen4ry=-xND}Q?$g`U99 zlC-0Ak5@U~=C$B1rvJrzzk5PcZfP9SuS&p|c;l`6wPb! zsaahI=AVy&wkFF7s>9LmQeXk+`n_AZqdA4^@Mu`{olfzk$vaU*UW!php~(tYuFU86 zuzC*SZ|Kzy8H@u1_}TFdOX>eCFGG!9Ay^p;uNKTu6zq2kSf_DjB{)uQj*m9P5n319 zjO%8*pdO`$EkSXxSq1$)r(edjIoe zo7H(*9sIy)zV2+eJ#|fJBRHGGh|a6d*!1Ar=Y%A7dq{;CxB`LvrNvheyx{wHJjsHq zZm;rT$h*r86MvK__2*NLfs1_>o9iaLa* zuJPotkmk3NY4zjma@dTl`8%omz~ewww!;jaAG-BY0IKJ3P^y^sSmE=zI)ewv+>zZlv~@&3!ee*ZR;4D%QY}I>J2) z`GJGZTAR-iRwUdK{?QA|G6M1NgkEa~qA1?Rv5g8q*X=3*yFAAHFNK`nsL32liCzgy z-Hq<`^(@2t%A?}N!miW1ksrhgnY;q{xi8!^Hs;L*fy;M>cvJEU|i+UfqG1ZdN$oDec=m_eDPSS z%CN)X*~!<}t;cm({4*9gM6RX?lmSmGvq?_gK@q!6* zIk#Y}npMt;uP$NT!j)P(TVlVlO-a)H=orNjU(H1rJE#Y1->2M z#bEjsqZfiwoNdF*T(U#?HlCQay*ZAu42W*}ZhGa&bzZ=-5A61KH$;Jt3m;1_SDQvd zlZ~}{Xm{`NOc1`RuhJ_%Q=qH~n_9RGdC@YqeojL|_R1J~HD%1PeE9{}eFMjXV1dC$FTf05UM-|f!Nxta|BIp68%jsgL1XRWgi`Nx;>N`-~V zoYsP#HL+{aQR7gHL6_&@}p09e`iRpN~v9<>_A_x9W2RE_bL<2D?5 z-xHn62}r(S53xoj^0HR75>EvK)QxtY`EME}M$}fyU5XQq>;|YF&@>P~ArDS+dw$O$ zIKja5+W^~?15-T137TA$p9jFLL2!Hq2cP%jkOfgV;Z}UwW?Au8{ju8ld&=UCPAh6(PFhj`16MOBd z!}o9#Y5iqx7tzjXjnEoR>$>Fhnvjy4DfHK6dR~@x7%$M#-WBmLdiTTZi9Va)c=tCV zT`XusywWKdNl}5j-sWfmWuya(MeQrXxWvrh$TlhXPIula)oNMYEDTZA3PhF!2?0Xa z$coH&bPE+)+f~h8aDRN8Q<_gVt*$BSg(mCDOYi{PoSs)BfGVL4v%o3}nDyB_EAIhm~A=>w-ifkSF zAAU-o2XE!GGUqbwkg+p+dFMN1^^z5)BS5)OTpJnAgXQ*{Lf0vMICPn0f>;CW{Iocyh1oYHnj z{-cDxOu8kbD^xZ~?(PA?OQF2uaXEkqz#6SIi2L~6Z#G#Wgf-Mfc(rI<#%SODbUElY zk*`c|a3D7XedcOhuUX)NqUla_{cH2-&BESS^Nqoun~k@?CP4k6UzM)VGY(RO1TaX` zk1<-#0l>>cYZE!cUB63fJvWE>#`7qBC=~fTg0Ok1Y)LE?!iHpJjA@!!HScjNpJ9lX zD@rVl=;#Xa8346A#j3l9jFuK)-XAUl4C2d0$I@3uz!={3rx%0a@I*(CegWa>Zo3(#A@HDZe*%*ei8*ZqQ+pzlG#3!nz_ zJO3p~-i(%ZD(H!69=VOkttmJ|F5?;@bs=xe7dd;1siI?Iiw=%HGcz7d`9J|bM4z-bFTG?xe$@JvMIgEhj3(_LMe-DH}OqeQ@)24J5;BnTj zW#5GqH=H;QZIb$@QFxZN5){n%r0jDm4&Tk(soZkaT+!dY+vYmYa{%IGkQz}2ja`cb z(mDd3IK{=eZU4ftH#k8gMhb!k@A_LFfE_}B7dp7@yalU6mJBu0p>Wu$t6NHAqUyGg ziH~;`qmHtB&vcoxu-{%mbjfF%>a^pXnIX^JA|sN7al_gJ!oVpwBP-Vp0&6-7eSHI& z69ypz>tHi>l8yX@bemtCfoej4f)t(5N}=(yptNqyL8(l|$K?Dm1re9(cvAmGH??Ja`!_Fq=xNmDW=uVdFCAbdsSI4=fEHt+|0k1>%kt$A>))YN>di$A=s_u^ke@bVk53wP?%R#CFcguXxz z!7+gSDJY3R)R=d>9H*UoYI{I%50BJDcrxS+An!5s`vn<0_H!+%dRl&_R`TrQn&9a@ zEKTnmp^edoehz}%n{9c?)DX-dDDlg+&+OlswmuRqRk0ZYb0jj@6qn(Tm)AVT(KMPdZ3%f$ne*M+5|FT3 z`o6@0uPoM2QEwCIf?F5t#W)(np2HoR$ivB(`KVSCoOHvn@RBD=H*U(-?DbPA%0)A{ z@s+dmo@SEiLhct+l&Q)oTw}h~K}>~L&S8a%buZ4XT#v#0-F?1Hx>5vGaRFXNDMDfc zTO;&Puq1YjH#T{={0R(2g;KP|Mt(OGPzDoRW`ztuAitSl2m##42lRb3)ooqGvfCAxNrYfxjdmJt%O@^W)~fcY-@=L zv%H%bAFi)3Z1Z4HsGEV)Et4pr=D)Wa(5C6-+6cSNdN)+ytB}-d^}|747UK3eiM-Do z9(|%c$3JmbK!|$|{h&HuDD{}bL(5ck7ulFJH!WnZ=V9i#1UvzfSG8n4X}Rv3OrSil zWA|rPoc%kQ7E1g{cUn&SjXngGAYcdxnrmdPIZ43W+qivLhn(feR!Ksf2cuQ`h>1ZC zGRnDlLFO4^(0ADG-&92*6VYZ)^b56Bilgye|5{%%aXbbbK{6=qH!XWn}ds^@t#Jn5?yU8M7w_A!H>Andfg~sv;zY91mVRWHCHZlonAh%THiKcZn0>bGy~> z-6ipmO-!H1J`4jXYCWoB%9~WqxlgsgrOF9ZRX*bUyWuxB(Lcj zyEDVV%nK;N&!&<0S~HG(f7-}msrQ@4$~madmEQ#KYYQGTlgzAn}s-$z%K4U24P@z@|Y-{Qmvhk%+Sv z>)a+Xp+{|5O{bUcBs$51BbvzAVG^b3OmoKq=avpBN<1#jMR$#4O#bvn1~VilC`p3k zLR~U}CVc8I2*!cEg1j0BtGdd%nn5`F8UzBe%b!q_9l$NM9dr%fDq4)whext>q>sj& z@x*n+3f|SpXrK5QsS1&djeqfrLnvdi#zGIW0xyqLNyW2mco=26=}E;3xnK_@>iIs- zuC64*cSAPG^nu|o+ ze$6FzVMa+E9#CXR`u$~agfUupjast4Mf6-3lncacuWEx@OiMl`tTZF%7P^{vWz%hY zzR@%%?j43kV~e{(3Kh}ufNHzauY3vt#=HmNPrtG&%nx77cAfAk^{%S%)#>+M=Pr0s zGa-^J9n%PS+y-Cy15x8|S{JU!r{e(j`te-@@PtJ)#}ilY;nbgypxiEs2#BIWSwZz? z`hUiPFiZGjo@$U>>xUCDrd&Hl-Y~Qkv{_b5WHtC=7z7GNA?3~N0J`v7k!re5&B#E) zp?o!_70wj?OWPbctC<26Sg1FV+%8I@_NZCpd!@4!6gfkKo&>Wb%#BH#^_awz>0=DU zp^N!llz>K-`(m288-=LnIbjI5BvA%L^egNF^{hPtuL>yT1qxAA80I70b*9#OMs;{x zksHyw3=1^n-tcW$)WQ!ybW+eXD8@JB7i~ggSHNcgJ zP;j1~Q}T5Gh#qPmAn@u;_FznZVtXd1pt!gFD@1x;8Wj6H#rG3`%54c8F}%(K@1F(l zQ|BLcRr^<^*4V?dPh>NxRJ{a<$MAZZkd*E`g6JUzV(dZujtKow*;aKY7LP9RQB(N3DQxJF3GuV>=Cu$?$ZM z8y#z3d3GkV!-7(-e-x%3fTrpt_vO5}8eF0$euX0mBr!?xHMOLy94~zqn8iPRI$nLX zZRI08zGvzFF^wI?fyzC02}OO@XYZgm=d4Mj|2#?{Vqo-YT*F{4%{%XPt6Iu^%_X8d z0+OmCD=z;>86``}K_QHvC>6fNXvKxU*x)8f`?oq^6rFRaVX?4xw!vviX-;fG-E`^7 z6}o8rnn`9pmKeA4JA$N1Ki7soK`F=xnRvvu7LM4Ef|QaPA)LV=;Q=7KduM3N|7&V& z3DONUrSQ@1{8Upz$}=T7uBO>w)0MM?fMr=aDveG(3F?t{h9p4WZI_A>JjVT?#}pP{ zlYAzZ-bRqaAGd>;|C2jyw;!cs0l4gfo$E_ICoTEpLEG%1(ZhJ5^M zYn3hTC_m3()I6T=jgfZM@SZ2VjzS>6&U&TL(!wNx0?u!Gcrc6?M0G|*aIG8vVkqVZ z8QxKN?cZ8&lCME7ADkz!W~|*_c}J)GGcUU}0UkKX0ogg_{nIlFf|F(slRNr2wSx#hZA()Mswg5vM zvi$kU_z${>C)DSK#DBznXg`C{+n^ z+frd{KMAYkg0#Ek%~CEZW{_4EL&afja19C8v~2k<-B021gdGElD_Hnl6-LC*8G`j+ zLRfxr=S4*(VCH!=(GIOLL;>1)zz^Z5L$gwA*gI8aI*SH~!E=rl%G#C2bN>%;!~exE zrR@KWKug1y)nJa)%`Sq!G?_Fl=|Z_KHvgjXyLH{Ed{ux2BvjWGTwU0g7tJy9_$kFn`&jO3n~Q!ifL z?|{%kHrx3D>i9cttUs#SNyL7N_M2~({SxiR%!}EOc!H=Rpl#M^z&q=^v3B~T?)$W` z>3+!)L*PaQC#p`P4IvPz`kRXloj4lw+a~pR{3{fjXXFJ(6JSN~D!&A_cAon-VxC?p zuQMtI0K#fsa}5k(ce$7r+OO8I6zf>NrkPi}fhiElpA<+AG@1t!425gSSGU+acum(1 zO`2+^Zt~^~JS~&DFHX$5$Q~~vSZA|m^TSWK_E(j~6U+Ufwk8LxCgq`~g2=Jwi^e18*HBcT zr~i&Qwq>sl0LfLE3r>%nI21oAKyV>Li$dj_m<>?}NY+478zu{-TSxOGW?8G^#oFbQ zfAANCkPSnYky}qi%=vn9YI5p0ykblH+LAFC34Zsheyhr}#)3bfzl^yPh@Db)hbJsT zS@kj5a0nYDjc^)tTwOZed>F=bPXw6h}2sq zL*K(`xo=zPg;&S|v@iOcMQip70c~}Bl~Lz?$f@F3?@GLIKLP;fgoxj;Y|6T+0Z0k1 zAD9(f21ASN7}U~BAR>7 zbVSaSp}t1~(J5ib~e*2f2nQ}PK!p6*iLH53+lni1M(q!HbXmf5@pz{jP$)-E$%fSAD%p+|y zYHXlYk#aXbC65;_6f8|;A|h2YQ9IP%tzwcBmzu~5BgbGtJvs-iZDVwm<=9hTefx*si8Rot^?9(AH`P3mKs=1vjwqi$rmjRFa}t#3!C+yawda`pbvXu7`5X z$T=mB^R)cVgb4N3rn@54D< zii!&L0b}g~jQXed(Z~>tU6eI}Bev++K}V`#o9XU|)CwC?6JL?C2>L=DH?9E;fBMc7PAYc9)Ox%3So+cWTL1IH)TS3$0 zUYqsdKOn=%nip=rO+P1rNNvQH`tWvoJR6~qEB~1H`^DpU!ZL03@H&W=!+$U1K2M#v z&xQ6DZzok$!>eM7!lj>xRDI^b_MbsnOgargk&Ron9!gIhuqi{z0Kr*qv03}66_T$X zek?YA5H8of74JNmkee6sIJ(=21QQ+4{&QWijTlBH2UGUf zB1$)(Us>p0W@GU4R!vRO!6N63z({3I^G4*Erl(!Lm@3A=b3uPXGw- zY9E0o^K{X1YC!!UWRh6iH~ljfM#ThDfVp68Ku6=@SvazywmTjteeV|5eWYV9xiq17 zB^4yIQ<7euO)LgC;g{;;;$z}rWINCm2n1w}((T}xssSbu;c4#b8KEq?pKqs(QCi%K zajnynHhm}M3G#BFK7P0EY}dx_?TV&~H=nqMx9eCs2(|-cH zuUxH!#5U~2t4RUudUEwx2{jGc`vSy-9IND$I+V7{J6Cw33GCwDj(oc8pu#qxRn9xr5q+J?Kvusq;3U)dTK08 z#DQS#eYnHJ^qB6K{drOGUCZy)-fwdX_bT!8Qaa=kSgUKs(8CjRo{)oJG1Pwlw~GXp zDDNL-(d3!uMr|AqmZyuHc;iwtu#-BPKXE*Bx|X*$_4vEeix%TA=by`VUP)~q8#KFz zs=kmN{gs&E+N5xHx}CFJ6~G06%fu$TiIEJL4>A4)0dXtxtH1qwe$~7OHVq5RdDB!1 zb9W0HTCmV)1Z*mCaS3atEEt>Pa+#!Kjxa1{g-1H{Em_aB`pDS71d967iIV z)>yQdnzS$nIPfqn7;OBVGsamAYTNLe6R~PIKLaQdPj9z|4AjkOc$5}Z?91N8Wf{7|Yx(`yJK3^)z)LJS zV)b_q10FEvCnJ$dF{YgU$*o%+;Z*!T%@Qb?e`23719T2Q=MP}7X-Dvz40UNZCwDKu zw+Gt9m?nm4Cs3{3q|Uy5dW9Y1Tr|gW*xh3Z>#ew~#*x~4f4tXnw6(3YY8v3cndm@= zcMy;R8a9Yz-!-5Hp0i}llV?VxC*33{N{5LjPDF$gMCBsalFKS>mVKmzF464VH7S@p zS~l`A)$cFJH>MxiDQeO(ggXSfA?B0F@6bN^<8QVvma?^Ja5ViKmg%#H?`@y?S-c0C zIPa;tp~LsU*abmVsDU$>#D~7f-63I{;dFlMPQ0i>e`o&7%-1>>*S?sE(xlJ6@X%!t z?L^MY6p06aqX9>iul9GH;Fs!Pi;Z!uHi(Wc!{uQ5T93mhMO^lNa>g*v;ZH&}!d(h2 z5wRc7{6-=`I|Agi%(Jk9Ag!!Lt9Iq--u@`vp~e;F+k%&W`q^YT|;qM!y|wg=bGh|83hHSP0f(vBUS~a5XOCcJiniNDgqaVb$p^o z6qVm#?E`ovU;DS$yacF~*{HL%#FzvqFnaqj8()h*r5gD6{n(pHEdciF-@4trYA{xj zQF+Lm9bgAr;V~rGZ!$KFjo$t)|DyFUwB>@1CLM+B+&=IBv_!XVl3vx6@c61DjiG?w z6QB!=wU2M&7gMO!bhy3DyK+H=y~OqJh1bhqS?wpWH`iE6tXx-1s}g58>e|P~k>@o- zmghJ1lkQ4O}V(WjTLqYI-mTUyBg|t5`qWejqp`TU z!tkTRpCxTPon#nwlp(kX4Cek%EjZw&0IsbnH*pasVWdbu2;}1AKEDRku_qmqmL5c* zUp1wFV()UbIl{le_pk?R_>ez{vhh6Uj|1(lwrv=RnzBU&JPwMo>CB>|9RmzeqAcUG z>FDV8lNvJ+D)}X3#Kc%-!OQH)-wg>n3h|n}M|u)*yM_+GT7TGxk0glDIR%XjCi=3mAUWahE*nBPm!I}6C#5UN zDdnndKfSf{xprS#T6D^$D*Cf}oQbsb-B@Z25byBgn+4mso~XJ^MD~^NdsP$14~lAQ zXBKmDYXJLDG|AK{IVT^RK@CarBP$dJWzJO}NOmy_gL z6rX?H!>Dza<%Ew(EYJ!`C?-ea*hP%23NunWym7z4G(;M+cx-@=*PO6Ue1eewLXWtv4b6^Nxy?aiS$IlCXPvE%8?vQjS zAFLC3f1^SI;M#OY)bKV2j6p7(z3l*tfirViT%7-cE2}ee^#MgfEt~GThf($c&Blj` z5qc(X?Rx=Pjl1mr@2}!RR{w&4IJkvC1gmg%Ui;O8%zixnDP|Y5sux{i(bPj`27#j1 z=cgmT$L%96)|Qzc-5pfAS1N~$1PO6*uT8X`3BeU(l%h6>Oa5q<@b9e{&}9oYwX=&c z%N1F_*U5AOjEv%2i`QbR`#0cZthj4xGrxs(bUTny_M+3Y=b<~#SJGjH@otl(G>cl3#+5; zlVP}IzfZtTkW2@14wY#j(`8ek5o9e$WEH+bk;cSlZGiP)l}en-$+_zlBQD)mq-rxajL zer{>;SSDI|ezhVPo>sc(gT_(elE|Cl#Bhk$A#M6K@(#$FKn9R3iVXH{V`g_JwH{OJ z)SW?l75*NREEN}+HG6V2*%<3RFrJ(t5RGv1TxQ)Hx`zY7_U25{TP`hW3TcXSL!1*D z@s;W|%Myy1J)m;??YUbtx&n*AjlAO6a5f_owN4DUP|46>tibLuL;RPxL$qX^lYu9WXSgbD}UUdSnn=M>i4fK@m_+YgEY5#mW+J-&=sp7)WI zszPD-fZQ z-xkAivOyvU%Pzp?hS-8bp`RT`nAJuM#UZ+#9N<;mR{I#uw(0^SV=0;gqv1USdUyX; z>rq0mW9}W*tD1P9SRBo^o_p+m;@Z|cGvgq|hoEW=8Dq0kv=XTC=QC_XJZ51^7Q*~C zxJ*#AY|0W97oY*sW`NTj)n5>hq4H=Rk0BbCE?i7@tRcSh9{M}0(dA%zLZPXxMjamX z@!>B+pX_(QAfWmU$~NV#xmn($V3I*fst6jH$$cS`50F@5&qypU<}V8KDH|!Z;h$je z-u3Yoz?KL%-B2*cLk4(0KyXxC5U*YXmi{Op^4$3;;D5L+Xa)VNy#mSKyr7UCpEmO_r-pvAc5D3x4Y@Y0qi@_K3V{HY z*I=id)1-iEN0B>z>|4}+t&rt;y3|-k^k+xjw#9}44{tWi}F-Y*Il2-ei3f=f&alomhI)6P=6_ufSI!qj*8?dnN6e-`uq>Gi^W z{gG3ebZM+uLZ9h&NgTWVQSIZqRyiYX<8qbVKwM~~pF0jb{`3dGaJ+O77(|vj`${P7 z{2e#+p!hd|dEDg$w^nhMC~Cdx%CgyMhUq?%UINAJtnVPzS_(bzzWz|%20Z6eW6zd* zNA*ASIhCdp~2*HlHi|w4@!3b6Fa2z8}SFB)yo!T+hvH8AP#B=$b`9+ zViCD(t(e%Rre3Gif+yp;RfWC3$_>E+9ke)vLB`1S|A)7?4vS)a+_-lM1*E%?jwP24 zQQD=uLs~jT5mZECNu?WUKyMK7C2(mNt z%ri63eczv3;9ADkEu1o$8139QHj){gqI$J#re6df2PUz6qDZ@YgB)Y%fp9#<9EL{o z0KI!!W0VL%tC4#YP~vpj=Vk}oc9Lrc#~a&d7Kh2_@f>6Iv$cwX<>!v$z(q^(Ms-Fq z1_ts!4+FvFKG42G{Q_d`1@0$|^FkKFBKJGUzP;vPiuemy0{TjLuojFTWScfQOIw_G zTxL3GN9S|&fJ9%g6!*|FUxjt4{b3eGrJ)&QjD-7~Q4XQzpi25IK}8NRjmu5??v8C3)VQ3%2d`nxT!vo|>8Ux~wlRdT{H1PQ`N ze=E8Hh-8PZnfDSsUR-%<63>TEsBy4+wRjG46+rErdy2tY6o_TS%dyT?7Mj}ezS;`?WBD2DX00>&)$76ljo|0Okw*Kq zI&;I0m>5?#=FYE|#C5FctxDeo7HO+)HZ{B$c~Bw$Y+U6q=u2eCL-XpJuK$h&^H2cr zOwkYBSkJ1iv}a)9P&#psg81~~)?!adUa_RPDV0sksC$WSXz4fubcFol>yP03I6SPI z`_Z>(yux$$^i#%VREzXAr2(hc`DQjXWv9;tUY~Q*nsq~z-}Esrzk*)-3XA=WPJu6i zB7NjF2+eVF36G=+3F9|sk0@j*e83Y}CZ9v9n=opAk|(cn#71{RAcUNU>unvQh_Z8= zZ3vl`NNg`WDxtN`s_|g^^l*oe7c^@88rcU;I@?xZhuj+q?irg<$F0`P^FWX^i*r)@pzpyYr-+l+Jsd$IqM=`edi_CDGMLu8w7pMB;rK>?U6C%e9Ot3 z^rbxk_j7#a;geAb{&Fs;9k-Ukn;(V*u%njb2!rCcj`g;k!4PPfL&;%6Wa6!)Y;x0Ko3>PY)BZ3S!8>!+1Gs?8*}3NL`V+vwg0DeEZM*Hy{lE!C8rtO>KO4Qjm{2K+Q$(v|6( zVqN$;x>mvw30qM&71s>`Aj4GC)vvC5cqt$W0f4Lg7s|=ymn=+NXO0<&=XLAxXdh?Y zEim%BaVRzG9nIi#GO(lK$Wh=E{NoE(xyCp9Y7NhdM7gtjY$8wcXk8(*&j4(?It^g( zKRbb^1(p-Qo(9CxL=u|mRJdE7&g5qp}mFQx3CIww^aEb)5j8MY6h`etZW+^S0Eu|M`a$cNq+nnXL(x zZojF{HB=~+n0}MOxk8(P(bUZ@VAd>t7i{MS?I$E?C(gDIgw)gFZa%3B@-cO->1gR8 zN-;0^f}^Q%2GknBqSjZCV9ExFBci+*IC7OnxDz)WuRrC@3gCwc#D%?ba=K*R7e?p- zO5~byQp-J#=F1}C?)(*M54@#q2HNH{Qd58cID(Bowj8t2AVQAvG?l?R!v(S-&!vI5 zI?B@o#MR-bxOxkO0BaL{j7>Os3tz0k%ItM%37#6w-tyCWm)O1U-ys12zal-MJ~z3{ z;|w%QjEZmJ64EzYsIRzL7+PmYWED&ahoL>dVdCrKbd=&{5?^TBpUuy_Ir5UY$XZ_0 z;Hf5p;Rggd#Dk5G2KqoC#Q1Iq#7o)Z;hXeEv6rW~pIXQO&yFE1<4&Qa5;|DBl0-SY ztJ3|7vD*YW(G{6arDJE)S3O&Xw&GuJT@CYEgkUavzosgnyPn8-1RFGqeDCT3WvktH z_n3WWrk7Tdh*)tvz$FNYTA2O-9zV+B1Gez7Fx5L~RecE@n%vi+lyA}2CnS?*wL)pR zv5`l~${PM1(xds`v%Fn~s#-e5TRIh?fjuAv8fqK~J_cXb=mFmYkHn}Ghd6ZC;P9~& zpa;^+oL*jJV4h7CzUMoI-@$=nm%B^xSVPF~PF*9rD1O?9ym=Uw0#f1LLa0439^Uo0%1W zXX+}M0DpC*cC(}X(cIv)H!UVB`_RkV z4SpxzAtxMfYhO*Z1b@S}%?ET#OE*rsn@M!)W1?8Ez`J)BTkN~3o=pf1&zuR`QAZim zU|UUCG*gV8#H)mlbr=oz%zAg98Ey82|e?F zz|_E4vr}SHtGeIf4?7-hx^v3{CC$Gs2c&u1C&!l^Mu~b#8!H{fx1~|iQTDx=f`EoH z_%j2K>BJ}Xfb*TE02HmQYyLnph;;^+)`h))yPO`TEW9xL$sgg+n@=6hgjJbOadL^P`faZUW`n*Te3@g;LU6Gl%ghwWK~=zCCe0+*9x(@ zPJjGwQq{TCLt5v<1B7~{mR2T$BRE;f+$>8uOlQ5_ZRaSG>`pZ+qQ$x8tPMXlLa`q8 zg^ZrA9=IWZ=&o&UF;=OwVpc`9=-N_OiZ-jug^GC zz%W(a{djSoV&4g0&C;YUdmR`Vzz=|b_2tu?^XDft)O+OQTG!#7&ojXl!apEw=yO}V zTUAxnir^kD!Cyi0VdL9{?CaXF%7Vz?UOR8w&1=uyB`SEmDdDr~z=|PgN)W+9(~5sb zI}{#(|1Ju_nLBMGlQm*fMdT(HpYXVp{*eJ05Y7hj>+z`ky2X+)#@`%my@Eoqg`sPf7^5Uq zj}bpyNpT>4p)?~Wm2Cs3yDIsZ2F-#3w zjE>ToYW1zQrWHGBtvN|$Uh%`V>^^A3#*+B@Z<`RHfok9- z@K^YxvvU*iDyjx!WL4DDMbdp~85lJHD+&&a*M~gZSuqMo40vMw_Hd$!w9MMEGoxD_ z5MscpjsDv?^fwR_jHGcJFi@n1dpe^a9%Rq{_j||)vRy)mx~_H)xOg_qxL74B*YG3G zYY$heiZ1frqvN4y;^BEi6bt9@!fnk92++7)HaD!E0~2^-2Frg|Yy~0)#`GjGQ*`l0 z5l^~iPVuOj@Rjf>W`y9{LZ@P+i;5OtM8+UHbl<%WVmtR6;}!(b(KbyYuFa#%&z=^)B}ABViBBtrY8q`PYQQ1nu5=)|AuRb$rTw# zw*9M%2N|K4ifp>dCiku2cqWDf%hq#g%{c{x)A6y-H^JRbU-291UY~W$Kt!!umV)~I z&Gw~LtHkoj61L*uPgz$f{*QGXq)&1jYCD&m{N47+!&uV`|T9w=ENEPJihU*VZMHJ1ZGw&BZJxx-3~+ zUY)vXh~N%b5$B-6$YMA!sKCYlRm*UUFzSs!J|5SWQb3wR@Hv9g7XwOb-wUiNfdq(4 zQ&Y@R3+bo<3{>cW&byAWCa3)R)ZEd$c)orP-GZT2hw<>`(DXtk(8XFaN$Z~dvn!yZkp{LcnE%XyfA=sO_1DxM_3Yc z)X1chzj*w;M4z`fa$k6ss(nVt9kQYbp?L-DZoI1;58U5{@L$Oq;?g|O%Fk*chJjjk zMxd4*%JPfA$R{U?3q0CN%2e6;y!HItvm1pdWGODME-pq6Z9^;S$@U?5>WN~n#PH6kH#Paf1SU0MyTr&|5_)J1!$fwj$<3C} z%^`~--3O1!VDa6d*TgBR@(2S91cD-C5=1Qjo~sN_`^po)?XEE?Qsf`|a1rQ_H5jGB zXY{MRtCK@2G9Xdf7g#)Wt;6#Ae>i`|(JdLI{&6+2(@&H}>o$wh2Fr@~JguqX8)j4# zEGC^DVA&_*UQ!qTzne`M^Nt`T9{c-6YHP>JED`eT*M|V9gyehg3LtY3Ou~-byvb7A z-r3RIL4YUqMV>*O8r}rYlH#mFp`>+n3DBYudkIa#SQO?#M~}&!abp`VkZB+`VoSaC z#oLVK>>b$?qUkFA6!?cDvUL}d#Ti~>oh$#T6P7cX<7a+c#L_5xTKUrcC`W0m(X5V2eW;H4PcG#Ci8;w}6dHt76`chVLVCpsF zCW!_u60U6cUdTqYN+MIRp}V~GtFv{4^Pq|l@#5%Hv2%sO*IesQ@U26VwyqHInCDhh zXs(j`UL zKwR5&95HF6tI&;S$U!?7G+c<-qWNTPsRQc8N0@QgI$s5)IMC=CB0?+eiN4?PdC3Nl zC%kUiXFWTe3BkUX*&b&i;yIU_MHgPQczx8a_TlY-V2gK_Gbg{Fhz4cF>)b+^0P=q! z<>TB&m*I%H@MO}+bZEiSrN$*kHhBhL4Z1DS4TPS|dI=pD4t;1NwX7Cs?f z&8EoddlpIq%tC<@=#7=QI8<`gYDzic<*q;0zTmE!j;9eRKTMFeBCG z$tnLDT80HW8DnwyPC2~QLs_D|O70gaDox0goA%05ZA*E?193iKA$rt1p-^_@|973Y{Z*VQDs@?qn;nFK;@`TFA`OsyP@?hFuIp zos$232g8sgebnnR^KunN+?l{bbbBcUMYSWq!yiVXitCfgAn3TcV4)TN6V$57 z2s#uE1rJVdmB18i>^x{DpAZciJ8=3ijj}Xv(Txq!BmZasAc_FjnK?1QmOm8{Kvi&0i1si$wi%*S$l>d2k0qLegvNpF7LCZ*lss}xA^286$X(xU^ zIq?~-DtEwY(RI_b4sMr?;&n4sO~)&$+er_dzm@31%9T;=1)D{f8p7|F_YGmreZT!7 zf*K~jdzsh(kLnf2pWmN|Gsd)Ob(k1ng31QlqV~3OBXw^wWx4Q5afGfQt<~#S1ouJk zEcc&6FM!A@%34sUlB)PKm?5nEHFkoTXw#B+qTA`B*nTN_{RpS&EV+h1f&%jGUsio< z%7FkfDqyDq9rYUl$r?anpZ>{>Z;k>Ml^g9NW!?s4kj7n*f3Bgh6+*KrbE=|Z$oWxH zJ2^0)5&>v`pd0=N1R&gpC7%VUXp&QbaVmxIZ?}Ql#;OM*1`1H0qVb@HiXj`1pK7b=GhBnE$kL zC$PV8U+}{_%T?a*T>j<#WRD(fA5CyVmFECWka90^?nK}#%(cM=RO1qkQ*ST$AOqV5 zI9Ei;<2@dIuTKMvMbBI&TKEXgVf+Ub>R0;WR%s@q|$^S>Peixl1nVk^*VKETBS$f0VfE5c?d!jjWiYU- z@T9R>#6$k`wiaAbNkeuH(1^NdX3GIhjum;9C|A%a0QK#SL+2Q3=Rju4F8B1dba(3~ zzKYaB98#jthKzd@q$N2C+#^xN$%#*3?*L*c=>F#x0yo=h$Ko<&u0kczSzPI>ahLR) zC4IQ@as=jrXud$SIWX3NBf1zpm#7$ZHsbELIsUE(`dHZh8T)^amw+5UDL8~UuiJ5) z^E{gWZXVRAUp6&PT^Z}H?xY^%b>DZykWsRKBRk(rE;?U3|2z&m1%cOTC2nThw-3~@ArDUfMBA#>P}GvybKN1PleE6=NXHm}DtB|f*WAhL$g2<<>87Gl!7rDa4p zz$HGaIE+hZPb#CTX+Xz6cWPvR`26vY%G&`yAcmLvE|t-iV=e<>mDjP{5Wk>j*fO-o zXNOx)#`h3iI;pLpHA>0bKd1>8nV(s_Ux%}RxjE;xh&J&~Xm230y?SA*7QxBkVKHFq_~;pastC{09atE=PD~JKu^#sm}QO64skz?0#JR z{cHWey#L_VJd_4)V7{1MWI0*F0<3->KMKJy8EZ~SnL3xSI2cR5yr@a?1PEp)txy1u zkXdYe1m&1%HuW>8{n;lH2oh=MnLGY~gjf1{r#kuYf9@ zMQf3`pgfPii}hvvk~@LGr4200|TuYS%Qa)a|acv>M1BLoR9*%=g>-r>IJx3 zvN3MXi$Xg*#+l7p4~ik0~`Z7FdHV?JqP4KhS(7kr|Q9c7`9u zEdUo;RvaLng{#S2$<|_->{Gg}2VXS=?DWv-{{HWEEUH-h2Ya_+mT=T^{ta)RoG}b_ zmK-!^k(N9-XwCUcm*l5^zkhe3CXHS_vAy-(c6gIUhH>HZ-gi<0ny>Qn01=C1MI+kc z-}MawpW@;yY8B<;;n%-tF|1v^2g_QFQvxH;xy;WPXs=v#cf0}U#ToGAhVnT3{awOj zPILt9F0tS|HC(TI_j)aF)D7!7Mm`=!d>`bNYwQ~>z#QF8ZDK?01hBm%*_ z$2kICX9IE({=_~vo#5wWtsDH)PuVl^WO*)G?we7@T`rlEMuc6zgZ|~0!Ow%Ui0SkC z&(i~zqz$_$Iu7?_Dh_q)>NiT$lU8PzR%$K#0V%RzC~s(W*PSnEenMUg8hMU<_o$b3tu+YIL`sr!c;h8$qQhxDE}Cl%fYu zy3R*in)C-6?l-HR3UUu;G`-FQ#Og~p(4ji4EH3pEt>8fBTQp1#Hh_cc!O)CpP1N|T z4-n|5ps+hdp#QhPDv@8WajfrkLwL{?!(m{pJ5cFTABj9q$=lZy|3J|9g1Q5F? zhur9gK_v{bQBLX%%3c^ukJR+$;;+)VF%tQ*MW0?pcw z04X&peP5}D>_Ijn{)a8DaTDu5Asj+;h*~f57%9xBoZ_kW@w7727h}%Xs$+3RA8B&0Cmi1x8F)EAUQLn14;z22Sv(TK9kFx1|Qp2v^IMAyQ zAVe5@Q3u}ngo6hj6KeIH{EXa_rEAO82!H^^EmWsC~o@7~O|6UZ4p zdfooH>73}JF_>3#vNfTmp%X&ndRQR&VR6RlN-qZ=gHbR4ACNigx1<@$47uVwt^pEs zr$}Hw>pCBd8jMwg3-Ko)I*i;yi>v2q$}>AvdQhTi&8LCxg^l{@{=wx{kYy28D+fN# zs4JRqr)!?cKa&37L{9p{woI+y*AGIA&6j(;mii*6Li#DgGLTOeim(l1#OhY47}I~%)784q9*7xiMxh5p zl#*V-fgj}0_eR`Kx$ubkJ$gLdRmS~?VMLSw3iLRAZkTct`0vIM{nkH>BOO-s7``r> zdZ5xsYkNkyP|<9bvRimXFpXR1}$o6HBSgjF?8QFmaj`oBCaj%j>A&jYs5*sO^Lw_NphXWk6;4HI4g)^NYC^0B zKeb(=_#BI#`h98Tdp84eDHB_t-8I$`IOVX(tfLwjPV=Rh;)KM)dbsB^A2I&3x=!=8 zWFzUA;l)$!{sX&uE(_Xt{c(sn)uWsw4*ZMhRgw%6)Vd@GVz`CgvBZCg4 zbkMEL-p-SnQq$KIacp^;@U>5_&`mC4nMI)DatcCRn_i00vDzcRPbHhksCBoaeHAb ztiOQ@{M)AY_^CdMm+W`Zi?*UH;*nJq<55nD*Hi7-?LPMcEp{KejNkE<%{efh6%77N zCa|9qzxqg3BS36e47_E;Ey6$&c-cW36jVdwx9uoP7%UIYZLoayG-^0-^+xrJ%JM`BL?4FZ;sQP z#G5JU4&~~jVLER|M@9`)73tPT6S}oKziMNtWfz5jS1lir;F7OA5}rRHK|8ovNycev zw{79qbYADa{+g3tp#P=NE$bDG^f(oFR>P2UmbWZXV+r?$IJ};GV{y#yCtw~r3S>(# z`%j<)Tu{@eZG=D$u?mzXDJwpw5N~rNL-|OOPVPeujV4WlBnpc2+JQ$;x32WC*o=*Sf;>Ej zE_y;<(LA>!7fq~rNQsV2BH?L`Y};ZkaR0<%@=(A#2(}fg4X%bC3~2tZ(`tj+kSXmX6SYTmrb^Mg-SDLTI}%t5D9$l<1NW z(328B_&mK3vmQeTJUU)veXsRNJ3vt&xx7WI*O2hut-SjI0TY`n;t=Mpm{QleYMDLl zbeAEabm!I=0^_L}#lwT$Dx`9f=vA~eKC((XD4iH`QfrBW<5_)`X_pHM2Er+p44Tg< zo53}~F#kjfM@h7QDqF=W&oH>T^>nx5z!q&*ctspQG14W#`P{56FyQp&$O-i z7IEy0KJN+obCua@1b|U#bR}MKX2r@|jUBkjC60fi@(`Sf^%eeu6&17W-BSrFnTrDD zO_t)zd2AX>vnywqmqZQ97+-eJ)GhC&XnLO2)NUg-(T8Hl1w=M+;^$=Fa2XrPTFvgz zw*p?2;uPJ$0^ngazP-LY?9Y#vQ+~Ea4DwysiB>SDXNhs)ex+92lmX>K_SVD5_Ik}9 z5VVH-{5k}uVtpd=OaeP{fu_88^9t~I*;Fk(XO~(81lSD<*yGG0xW=_ryZ%vp;uV0| z=JtDc@9`%ELbpQqD>c-s{yvMm(e(T1b<{cR-A{e8)w2W zjS0W%p5P2)V2V*v!kVr_AphV2x*djFuofnhtm+cDTy3|l{khw+F`L@Wz(O?$&Z4)^ z<3V?U;BF=j7V)N>mUXI6xhN;p>74$a66lNEcUr{IfjMwd40#1oMlYtRTNhrxT|mA# zs>3<|GEyc@CgXmwO+y*XrNnD6G@U9PZ3=o2j+w%am?drs`e=%H7_XjTv z-fCxF@iIpuI6Cn40b#MU<OOq>z3NFXSTUECj z2+;->2qgf(iMVnIAJHFtGpu2_i}^a&U-)MZnCFXQ_Kk6)ra&;l&A8t~Nf(If8 zy?Ftt_`SX78+776oQu3T~{?w;cgQD#we_5>PpBzli6&$Oagky;y+=gFtdBq zqTfpjnJdhUhG=ka4>D)nDYp&1J+w!|Qj^)L76E}4nVMH=j7``bG8Xs0k|;HfqL=Ie ztLG^HOgt<2X>26WchyR#zU!~+&02I!5&gF;B@ zj^BYTKzZWm7{)z$?ssabWevl&?$x5L*Z#K?eKYwc63#4pPWxR%lt{^!@K{F)ltVCl z0=&52e=_Df;4Q|XPcEwJt{qmY=G7Ow0?WtGPjWg2kQ33owJnHEc4a zZZQuubXV(!BCbU7FsnB`rb*5nL~zxHmO3f7g1)j|Iy91S>^hB}dM>QmjaYrFMvd2) znf5v0$ZMb7D`5n}81Y|-Eju7IKOmaBhl*?FO7EJd3&<>ya@x4+$7eMK?P$WD@oxd` zW#G95pe-n*;wOuj6T3==-1#`e6OBD?x{tWoB|9Z^-qs$v*4+TX`SY+$&;zFi01@$z zFZO#JqJ^$8-$ix=?C}pTz8&PY&CoL}-C^hgMGa!of=d>Wp5lThCyg1~)aHs(#xoI!~Z2*COTxAPY>?BGvbKcD8 zv}JPO-gEo8CFunQOAQ19U&}~~t8N@oB+}QO87Q;VlEzciG};$Jr1uxZ<*xun?8G{T zGG%fZBnn>_I*s(xV{zI!Yeb!FFOaF$?#aPv{-8)G)x~ty>FhhzyBLu&EcNdb`qqxv zg(XYHtp8h661)0KDxD}vup?%aQJejxMe&p!11`ITHfwYg_e*B25FP$uRWo`2%sJ2} z&}ob{yG8XW0LB~^ibYvZz?k!C!|R)qQjt)y#x47TVzX-yD%PtqH-)-r9_qS58R(60 zTzOW9Uc7i1vI_s=;D`y8`$Ob3BG0^H<(SaS)4IXIPOr5Y{rJA;69mOG`z)(LCM0&D zHJGIw(+@UH`M}cq5sai;t2fq`J{qU2yy)wK;E?BU-|AQzf7eR~gq7a1hzbWF(;8g;XR6z2 zk|Vv@%*eGy0<#mp5_}3)J_Z)h#Kl7-@5+`fl&r^ar3LwUY-*>tmTy^s{5l22Afl@PHJwFb&;0E52} z82o-_h2JGUH`k86z=8{tfKQ;;=#VPeD)y=msZuz$j6|{qKA~p zgN7;WPm4t@uoV352k(a*7v_ais|}h`e|$>qt|r$6^zv|si^EEQNhl2GhL$KtKvW9B z`6)NajNTLdrbdTvEXADfqgPF7liOQt6P>n@eK8~4#tf#BR^9xaLBox_uz^PY1kz)IUf z7a2G>XrBCxRNV;MybcV_e|kl48{s(1#2BT%QS@RV_#%qDV8yRgIv_6p8uo0E3IeIY z$hmptvlp^0GlBGezTc-)^jZXdogAp}{@kI{kc-x)#(dxK{q$MJ+iz%9A&kaTIK(;b zB!={~|8o0^6Ik^jHQ#u$dmKT*nGPQgmAf(Z+{2=RtFUQs%>zjf!S7l}*6z-Z-fSrQr!C)AXRd~$yGWHw0ahL@QD2`)u7Vwk64wSA7 z7nDOt^HQfjg-Rb=ZaH}NfwU5iX%?K}TCQZMVB9O>@Vb|=8|83qtX4yN@?G1c!aJw; zk0=O`Djv#&0up(zf?(6Sz6bZ8pCF7u0{Xt0G1}DWib6Z;XVx#74^W^CJ$Gc)mm#o= zA9h1>jePF-%yW6Nb|Ql4h27f%dxa%d@n%G9Q`1qj2s#ukf)9e#adZyzzK-0y@mz3` zQ*v3M)2-Hu7XukZOUb))UIc?Fipp(7t>DQ|Q2s%`3|VVJJ-gB}((;DUZNoyp(Ph?c z!vJV-MP^*C;aynn~IH82eHiOeR%Lr-Ob#0=?xy z7eq}5&eI`L0i&2LV$%uP#z1P%kKD|!6y)^18cSYF6i=G9R9_$(Q_SbeN>h`(cmK23 z*NCd0<>3qaw_v#GBAll0k(_PZ>MC2u9MDq&Fw=s@AT@K$;ttMK>~XNoZ;J?gf9hzx zg28XiHW8%Q^av*GPapgbqx8`8$TRb{ER)}v!Xg}mUMo*OZg)X6)U>!PD2ak)D$6a9 zBhQMagtYx+2bOvK*JVTwr|=|Ua| z&T8&P3fFM!X=)>fW8cWWrTv1^XdKZSdxT%OFQ%QluU<{Ngq+;hXeIUDI`WL*yqmBm zP~Q~PW|!&}0{Tm8u^w`<<|&X<(%BCos`>-Ff*9c9#N{*;+znU|&-M?lJpP1cSGfYx zfEmcY9LuwU9TmE~9b#76t7DH0ZcU~J)NO4jHIHXBj;9#dF4TVjO+%f=|E13mXbObI zeoSCOo=!7LF^mY8;mMcq4g#VOj{|NeyY>TV0nC8Ls(@HsQdovxh?aOYdiv9CIbjmg zyQ_w}&TNF!NM!qsk;`b`ZU?ZST6Qdj`BvP{#KP2D$7VS|f` zu1jZ5WbuS(t~iR6TYp0vBewOnfWI3C=2{vCDdE{h@0yeW;^kD`GIPLy zJYeK!@F#zE0^%MI85^>p>O3(dctX-;0;eKyNgg&nh#?APXiM01~mPWIYB&$twdQaAE7G@2xPWCA) z$-EbK=5{uq?W#{o`sV0-U~R{svWYH%N5SQVl_Q6zNfwV zs}Dc$`3PTd{!U2qozOSd9COQ!UxOh#^(m- zHfTL1E+?$L@D@Qnlh8JnJ>kY(rKjN{_-e?6e2{xN3iKo^R|vh)=JK9#u3 ze9|_LQ(`qz)d3reQ)r<8=H8K(6GJ^vr(X>3C?0qmb$myl?9hg~-RA`FDqv*`U?IZ7 z=jDU(D=>2)o&`8&cyd{aU+fGa1j@Tl8s+FtkhdOtq|1MG{hF$quIUK>Ozx-I<$NqY zp19TD5;Z7p{=Qd(Mdz1GGpFAUgmeef2$}9MdMK{@UDS**gYm0dhgZe*fL_5zOlRIEV~wnp$IGZxQEicfvt-o9q;n#*Nohmv3H< zKkpb_8)u?eX;U`9_Jsok5Kdtw0i-lqVkLSDen3gh&lCcdTi0XDGO6vA)+T3Cr86XN zg;K)yD3=awRLL*spV{_1ZTYu3tq68>O6kC7IvcAigmK!60Za7@Ij;{Tts>@c{^qB8#ht_c{Z~!eBJsoCn@w&lyr`}6^>G#VsbxyIfHG5fY6DZeg)Mv4Tg@VSG zB5^|EZZ3J>t?I@uk}?4%H?Gv;dw6PfNQY3g_}2xh9EUtVrH@^flgbhom-E+lxwNJ8 zS>rAEuL|fN=x}Sjr#s+m?rNkl_~P9<-}*T^W)~ht29nlsWHY}2f}(h6Tsa=J1RWVl zkd%n!C8}&a=QdyeWIp3|s?@w8bqPZTl!A4F`76{uPz#XU1x;k!;uq{`dzXSO%a%=- zz18!Y#o?`O%ZckCF9Y9YLLDf2kw)wYJ2C))?cA?w1DCnRA7)YaCFW8Jbao{!-<(bN=%Euf4v&;xlOKGoa5E~z` z9vvQ79S$RMJDpiUU9mQQSvmdXwS)@I$eiu_w(;4|Hn@U0r=nu&2gs&X|%{=uLM&uhjg}Ian zbyYRSHT-Wkn(RqbWU~Pi1XLnt*zuGDXO7qU>CCj&xeD>wG;VdL!sANF1-|}eTey5u z^;h+!@NE+1TI`6>Bp7J{!K zKzPle?bWUtKi1m4W;@!`QkdWfYs6LK_w~p_tmx5Lm}YW?~R9{{|NM_y9Ew^JlB%q zPPkR1>fUW(j(wX;yo^_6uV$Ndd}3!7X0$t(H0~ORb{0m5Wp}6@zXRCukP$+r9Hw&= z^M;GGMXFTW6|>hXFYTSa5zA(@#I_)5(UdWi|HUWurjODQCeZ2vOT_3jY+TJY_=3@s zTBnw-@7Jz0>y)l_261q-?)JhRkEd$cO=|PhkzbvYs@aKPH~la@oY=LxeLHszuc%h~ zE%e0pT*7WkF*Bn$%EUn^!oVf&md|HAHa=h+>O{3&<(G*(ccyhNMiEyN86V%C%oq}w zrdF-(A=TEXatX9y>0nu)yH#ETC<UE;=;z@TDhAt| zorXL89R6RVRRnw_TfO{YTi13V)=w$a^|F86QglT3nAdGVha6>%M7Bghjv>ny&JA%7#?La5X;qeq;G?n zQBbz*Hxp%9JA?t(X8e&cj)D~Jx~Be^_(D5cKwf0gZQ5~?siFiry3~WH`!O+Kr~qhO zjKxj3Hr(lO&$t^VRM+MAhMS~086bs(b4;ts$$`trBn&mR9N&{!gd&UFilUrGb5l^( zAf9im{i`XzIGi{`ZGo{`EnZQy;vXDH9jI_5QJji|IBMQa)rvZe3pwt*!LF_V(t*BW zEt5x%3E#*at~XvP6%UTEVv1aGFNm>SGh5@|Kz$RSUbdbrV8Ys`;SRZZdL+t%Rp=arKD z8XMxrN*%kMmjJ7d5y^cL?yS3Aky)hDL`~dk8=M^VSMAWYkwq36;OPAM1+92uep%VtYXjPhS@Ln{0Wl5a7bOfQ;!jmm8QIS>j5@_csq$=S zjz9cRE-uLni%RnxwemVe(}7n3>TDQVIzcsF`8nJeE_GL3fu0`_>W86fRfrqvX*8)C zjz`(9K`=}yIwZD)^e-zOpjdLMgT=T*Rmzl1Y)qOwqro-)aJM{rc7T^Q>0)XC|FujBs|~>z#pt}17yjfd;6Y@u(7a1PgL}2xKV+AEnRtkXX)xq3RHIiHx1BT0CC46 zPuE@oIsR@RTuOMUgII*E43wCu{flfuOg%LBQn(J#9RaT9i< zEir?*GZ{qOg`+7esBNhu|HTd%ex(Q^#wizI6x;Oa`=^JYcI8gl?6aD&gPJGf+pja~ z{ynP(GCDR{yH7O05WSkmE)Q34Sp>LIn!1ILoJgO5+(Q$~98)a*w5AbW!n#9wTW! zhWSbs(fVt>U2%_P@~Kyqb$ERc4Dl~Be?X$X(4-f_=3L)g{S47ReXv{*Om;Goa+$?Y zcSDi^Smqc8&K``$6k8{FC|2^Lw8|aBoiR8cbx*2d>1(0j5xJqgeQ(VP8zqdcWDdec z!mBsLOa@*utEmp=CGhoAKad!$yH*^m>V7~q!P5rUmwyRc?Vq^x0=-o!W8+r|C&eNU z<5p`~b}479u)H!wFubE5!|c1bKoAX1OQrehL}^uo5u#+(EpgtJ+{dD%h{wyOeZR8; z=Uy1*diyKp5A82~aKKh6@N>nl8d%_*G(s3h>UH#}3r$(`rN_zXrS}%XcY7=2A>HvC zBf0`l=DyAy+oD_k8BnoB51t@ws0m~HpXxzyvlsvQ{CfN{SFz%>{}s7>OJ8of%zeK_ z^TG`|5==p#hR+q1f4`23?T#Z~NTN3o@BRUy+oKT;zp@py zKINUeV0z2U!9BKQICSK-fUm;a>eALRldn+K5ta1k$iIe=Z& zy2M`3W=y@75mQ#Lr0}<< z7QjPshzygexVz|$H|qPTh5yB%hj6T8fJ|>Kb4K=N^V4G6%l2W|z=|4-j~O3i*gF(b)7`=2-+l;3Gj8Lv_Br{oQlHteVqe(c(O#+_Z8%-P-vZ%M^D3Z7GDHke;LS7QE> zw~Hb07?ot2HYaECNj|zl_5{Wj>V&A`Rk-~5e*8?R^;tCA0n^|BbqrIYPE1ZFN-s@E z*cYQb5f?<){knVvbTmBj5l!g|8#tMQ;s?FaUIHjz(()_2@ zLkru%4L@N6e%0epRwSp*r9Zxb{LI-5+kVCZ_yFsD_|-1Xf+b*`~UH`Yn-FA?O7J0Z5!^SWTp7fBTH6wYIwpa zS@)~%=XViEFk3-FY%|!WKER@9ti9$(HIPEq*IqU`cUq|g#nEW(f~oGFOLj4#p8E6y z^0eg*RPPX2)HBYiUb64a|1>!EBnc5~57R%lT{?9inSdz(VHm-vToq0D0dz%;mtu_fIH^tp z-W#Sunc>{wwPd#jK~urVJ#~3-H$iC}TbMrjoRpE+U&>%OKX~z@0ct=9Hg}D^t@zmd z^j-18Yd0Gm7*Q#~!-k9B1^VjpAG}gHb@1j&H~-hh9-v7=1wQw%L%H^Vq>{WS?**;L z#!Ecgl4n`niMFCnbmR|>dVn0!#3fBH;IA8@&@D6KHiUs^9_2=T$Y6C8fl_ zi%ZOKrL<4n+q=O5L8JGtsL=WU{!@!$d`m<@U{jr1EZFc8W&0;j58gWn=X}6Tj;%Mb z{FCjJtU~W&yb`QdopSiDdUxGFN=SgYJGcUV_-?;=nI^IB51J`r8Olo3cjqO%&+xr> z9SWKM0jaa=#&Z_|qas9#nD>x82)=dqF|M`L9xu6M<&!Hn#i5kV=5ohkhD5<7(G ziO%H-3NWQ@%;?--m3>PGjm(K}z6E1U)`S*ge0|tK7fcrUu1Zd(nN82qQ@y2D0(FiB zln(^h3;_J%^@8dU=zqYK#c0CG_0hDYGWjuvF7Wk}92q%L38N*fo?m=I3O*g#K~4h4 z5rM5jPX4Tdd}o+?(3L^dfb-1< z-9ZoqGW>~%{%^kAvGHg4JVwvO2_(10+>M|TNr1ocxZ{Dx7j#%$`*>irQ2&#re?n3T zG{0jD)2^qhxr@pN9_a9YvAp5fm7^LnJE76~gu z+c5;sU%T@#%NBe)&-kh>s`=WK1~)be?N8}EK2{|YH2*H~iF^f5XasH%gijdM*TEte z$o&(1aEVCNqJ^RwKpvF}3E%DFdqO&%de$&m{Zhx_>m)`rF5Ah1a^De@LHl}zN=;VH~cwFXiM^Er8q!3d!8B{k_?X46s3aH>?E z+Q2+;;$Wh8zm%&(pjz8@OfLs$IVKG&j5h$~TwN&LpbbUU;2z;|{g`0EsJyHmxLUvg ztheMG>!5paI2WGa`oNnTdx;o|B_j_Dvc{CM@D zDG(y5eCKUscR@kYr<{NlQJ!OyV`q~6?rg&7lrKoUT+Sa!)Z{IAHs%ocC8u|C3Y2?KQK$p2YQclTNMn^ln;^+6wwKIa}X_0Aq-Rg(mb6pJQ zK=hgL3Z8;Bd3m#mf@FSS$%1P(scp=vD0ES`O-VGms}vuPa#Gq*l5jvQEO<6YGsIx(&IZirl(lSpJN@#fR-kYdM1~} zFUG^Lj$~09T-PWqg2Y>xL-&N65Kw5#tV_Nu1@!W0y64@^?+Lu_|H|WK5W!1bes115 zNt!i1>1mp_)H{>!BcL*0G&IDS-N)HT(1qZP_!Uv+!{0QvU!t1jBh4bUZC#Wr=FKhp z2;kKO_6G%D0ukb$F0_Yz8@d-F+AimGlUx9uz`ei5lQL*tF3$%%t&JBkalf||wtdpH zjZ<|!ad_f6Uh=!Me3iKR?@%#epx-e;_veByc}f4lz3xm`F&gWFsc37dMITB%-@xnK ze~o=P8gT{U@%)z^?)vns8N+!5+S$7bU>~e{3hlV0J)b2!$qEv7jilYrunnt%X#kDX zynk-htGbiD+Xxk#3bM@k*OZMK;5ZL!yCNJC)RS!^5BMhHL#tq_mdON1olJ)XmeF5W z?_v?gRii!qaPc>U&-_nl3DwTg?^Q9u7b0Iq-uic)419#wA^Ui|;FA4&RHVN!dtXFx zcHlQk@mVi;A@2dVeFM%9^#2_A0Z1H2OK++@27jqoQsn+?LeO6Tg-E|06^IY2=<5@j z$;GH#v8%3G@ThFIn$^)W58;%9kntX$Q>Tnfh1%3{-cm2%xc<{|Kr!D`&PEJ4Hahw6 z1K&Rdu^c^~5x$qbD^y}H-WLdKx+xZ7Emm$wN!>K+IWGrJGz-KJoqscs{=1?6_#*l7 z67r+LiT6(_6vu9%zb2CHj0``&>n&$1<4^;8nm^z6WVl0bj@uT%Y;x2As2ipl zy$P>t{%|W3+{HT_XcQyG`5rx`i{nt#XCvvE^SjWSHxW8JC?{-ddFv8zNJo_xwSTU< zz+;B9-u^Q7-gzTSM7BD8E2eLi_%bB8=lmzgZ=znD@$XE&2N!OV>-Od6x01@bil>M`{*)6AE9|s)M_Nu4TPFqH~bX`KeT}a!w%{L)N(O9%txZX-v#g4Zs%#Yq7 z7&!B~X)HZwbFpCjq$dOTB%=k)~W%EEdK%Qw(ZfSUFvnlG@KgRV?wQTxyfQAaa>5{&p? zMDZ{@JDT*JlEa^1hKPUN0=m7wW-0{vWK%l&;6&i$_E^l$ek1$MA@&Pect&4h^lc|> zG5yLjQ6?DwdAj3;vc+$Be zy@5F%T-uKw24DhErzEVTb z_S7GM6w3DB6S{wO=@8>?KMk8t6ppgal>n9l5YW?|(TL{-LSdfb9D6;3zs&5%$rzEk z{|*uGTL%SqeP=9@rJWWxmdHsGdlAQU<$P;4fENW~U2TG|CLUdL?xcSrXK+DzQ_!V6 zw@Rwuve*AqsZB2OuTg&XH&zYYjWqWmS1xDScvotz_B*Je3|e2oRj|rF|L^b;((A?m zUc#LtKOOt}WqWsz(8M)ph9_o+q6ffh#A?=axN`>(+E{OaL>3hi=IcT-k021fKiiT%4;8eA*Mpb69EpiLf-);fBBoRBo)w!AD|-( z8F%vJT}{I?MM>(oQI?kH=U&&V!!@QK20)VmL)59;AcCV2QhW1g8<@!r{ZUc=zWvvN z^_}0v0T%V&ZLg<=eiH+JAp;41=JWh~NjeVD-PpU}G=~j$z3SdGeQn$kh_vykeZ?of zC>DahZUu^t*2oB#%;&FJni)v==cFWM-9kB0m6XV!0vnu64Nufn!t$R!=!?0I4_H~i z`;gjy`@+Bb>pvU@Ng5+Eq=(AT{2e^c?&c zEh+Zoms}(I+rblX&J)pU)XM7WB;(G_?X#5+wGo>e1WYvPTucb=Qavg6wy1jkvV>HdL#UqekO^8d%3wuu_ zg7XCksH)t8iiK62J=f+S6=1%#v37Y>A zW%PUTAB9m%h)ef_=gUdr5ECV(uUCEoK&qO4_Z-LuluEHwdCyqJ{2?E#{VFXY*2ViG zq?ona)1R~Io|Xevel|6nUi4Jn1Fnc|1&m6c$b*nue=OjD3Lbz!g50O9F_{$alM&oH zrJ=gLw2SaM=HV2;PtcU^vZZnxbe0Kk_Tm4?F-A5{_voV!$v{kJZ&&FT>7{z$h(6T4 zH3j{V6y)xfG18W)tcEPzeU=W5SO@xR_8rn#JbHi}@2@C6cjE2MU+F3#_%Gq>1HCt= zTo|jTNtM#5vw#qJY&pMdBj?Hd84Z*RL<2z+k7tKg`>x#e%z{-W$)yb1Zd9NJ-1b_4 zgT;*bOsdvn!5{IS9<%EN#!LUk$`WXbw=!M(701RvQ3hN}fB*U`(JKFX$0M&y%U7Zv zYQ!3Zrl+Y|Vy&^p{Hq)S&T_$R>>I}_q&M&aVYe%YUKKAI|3yLdKl(VAH3d*tu0V~! z*}h0n5jjuJl#P_1+Is!%;`-T5a0%$IG%I`r3L`(M_=tH@&Fk9ZMI4~!!5ev3DlvTe zv|}-SG3zry^Y^;Q85eI20@`%P5cd@Y0+N>2y9Q8P^OGxj3+VMTnz>GIBWU^YU z^OUBRsc>lQMtHU2d!6W?Q3CNlqXb-!7u|D^0Z#Ny`XY^0O8_#xWdu{K43W@8+4;;g z9bO&11TvAAAFe~_4vQPl>TTH!3~%%Yx-^B_{?w(z%e>n{4!ldCrcnk;KyaP3r<%#Gya?j`B|FUGXefDb|NgIxaLI4LAn$ zgHfCG?Cs=FOXL^v_6zz;Aj=znP0#>-)6rV2))6Z7L|i22=~{E^aFUFyrfbAZ^W^4% zFAC{@(jwRZTEv-I&@s>jjG_FeX+9TyjTO!Jc#UmCNngnf3E33>l$kqD|7G8lAKyb>;diuhfJP(^K8qqDsH=FyKQRchsp6W6$S#h3g%}N|BFh&Am>rp9@I+n-rqQQ$mgzEH3c{$;@dKQ1a$ds4s7B-gHd? zh!bdP=X`~Sr0?n0&4vC5UsH;^gA)CRSwX-FISmsP{uvP@{s84h9$rswKfOc@<>Esa^9fMx*aMXv>-pk_lGYX{>8I*G z8>6#U6sg-ZVxAO0Km@cmZ+8m26DNSIB)>6+k6ggTXMZi7fE{>*86&x1pr-H_eCIA3H-Orc}8Eax{0 z<+x|v_Zp=tUtRgl`C&3<7S?zE=fFXbZ{T|cKB0S)M9;%zidC~P+jZ6NDYV+Ejvlon z_V_>U*0C5P_WN714a_6P=f+e!E?g1+_#whTC6!=G9nj-LQJ|A2jdeU6vD$NOaireu zYt$dhO`7)eAIH7dxq{w>$Nt5U{Hf<%=fxs5;YbQ&&1bH>G+qGVqug^q4xBwv$j0sw zv-9U3^WS&>ZGr!`z<*odf4c=f`;Cc?yz+B!!GKGV+LI`i4m;}?Fy9YXt5{hX5oGo7 zM~%buODg;513P%IHf6DOL(^xIHmR?N@kzJasz%|{OYEkUQFh7Hz#s5zOp99YRO`0) z=(LHaWKU3S{LIku&i2f~%lfZk&Tl7p4p!VQIr`dOEz2)V3;b#qZ8La(Xe57WudlCX z@?x&jjT!j%=+m^Ui@^nz*IX{F8g}(ujw*S#h4nZEPAX|oQWhGkO9sLsoD2z^Ip=Gz zkja>#J7Ly6>(>kUGj!-rXwpzGs=Kdu@n+rEA?48c)No);S7q8*Q~!XY#2-sa_khE8 zj86S5cbbwA#6cRSkwd~C7z6ajpSgh zyVrf=OuE-8o$h|?dHVaWqGSYLU!JX-f@jWtCu&)MEKTbZfP7~oLM~Lj4tb@wtEWw8V-at@4zfbM{A6# zbX>xoWDQ=2)=;Qw7O>Afm~QgPHD>4XtaZ7F$i6>~C__gZ1tOQ)xiYhFpfs(Jo}i=Cd#9J^ZLRTYAQk zy}QBXotL3ba&!1QPc}PFErF?9J&-oxjC+~S_$r=1q}pSWl=a+FLoc`*)JBTEKC57R z4ixstp~v+))_*N{kdo0oGfh0F?agQ9$bQK?@f*wT<~Tbsp$YUDSw6AgvcsfrsEm`R zm>)O7-Biev>Dm?f&|)YJ@uO#cme;3r%fDo|P+K6fBc3GDutuj=&@Nc8h{oLsjl1tJ zeF~wWe#V$_T0$Vj3V=!gT;|CRAK6s3c^W#IxbVB-I{kIt>^7mxiYHpv!6{%4)|A3) zrA}u615mP_>zbD~$pHFgpst}JDfc_;xz;_R;<)4(W^*oG54vZ)xTO)pC|l9{sf|_c zu3*1210Di&*zdk)S0<(CaPzXaa`Kk9cG58#O{^DKCU^7-Z+7N&NHrw~&}e&Dtn4X3 z6*oaM0?C$LjBgpACU{RICW1cKb8GrVY-1vkta=Pt+OsUPLM_D4f;8T5{-&`R4{**d)&@B{4lRgoP_x-CiwHuS_bUt5e?X6BF`DWk=&k zP>4Xn=pr2qE5X1YdGcePPRki-R!hQroJinVx2lR}Uaac<)FuhAqE@nSn~IxkJE;t@ zUzd@R)#1l5%gm^!cHWFroKx|{BwRey#s+S7y1{_uoV${RwsxY2jIrPz-IwlyJ>M3+ zyd-SFV>D7?roZN<+6Ymu`o`eJEBc}{d1G~9Pl={!(p9YlGVg8tY*bqUN&~5GN263a z1@gugK4QQ_R zv}ZI40S}+hbetk+OMBJx`l07L!CevODPI@Q#@;bbd9%C%?zPZT2QFGE3juLmQcfsZ zH=v?wn^OQXeV8O{8VW}sLMX*cmc*xUZTc+CGZMs+C3(i$EU#+K*1c=pJPuA_l?}={ zHrXlLcj|?U#;|!;OY&pgyBXP9myu+#=P%P$*cr}Kb!&qoWbPZ)a?0@EDwZ7RkCSPk zaZ93joZ_OcgEo}lkjEId1%*-7nqg*0ub$wsl4BTL{yNeonSnFjDqWjNd2Ml~|1ESm z@;+LOySl)jOVE9dNr}mK(2uO?RYP`#of&%wA8mj{dIebu!Q@JVjXW)r!Sl~+P-vC6 zQ1OaHO^M`N24vfl(v2?N!O<0Iq`p<8gayjKQ@&IxfI}o~Wl}muyt0%bfFveCPi#Ov zTDL-enth`F#;jDPV=G9l&01?(CATG}o(b({#tYwF%# zz`S)cV*+QU9sCcyw4P-_nJVSTJ5t`jt!|E{^uW!lD7%89XUwrpTb#P;%W;nuxY=^H zFE$F)H;n2iLcaS@*Oh6rljz_gp@tHQC)f=Hcqb8I2(hSC@f1sDpFK%T1sn!5eo`;^ z#wZ;deClFnL~6nEz~w?rk<{cZTW#L$Oovt@+^1CgAE1Uy>I2y(b=n;7&}bjIK+gGZ zP7W6g#~VMVt$jHB0eVvX@U_sa<0!JBhv(_j?}vNbEUZ~NdG@hv?Z_!@e_V>{u9|ue zCyoj&Y=Nv2T3j-_oiX-i5p4AIW3W-e?&-T9vXQGdrGzXXy$4=vk>rSruGYKZ=c;wj zKE90CLuhBOXBVl~k0_G(`JV+w7NbOUWZ|exx8{5SSO|HRI)W={z!}}}pbBvjozLjH zgQffd3KUx{{zT0pq^kR+j9!OWp7{hCg`9O@H5qtOn2QcJj=iH5H;%Q8KwbF8AiAWt zw?LnEJ|FSQ6CTa@!l1c$jsfj1LJRfvT|epBW1XevXi(HF;czWdn)C6P1#ft3RF;RO z#s2%DzPlpPGVf!yL%!j(E%pUJl@~l=TIQ`jr1u#genWGIIBkz1w~X?Qm`d4y;eNS;|>lrgf>gMz64woUkxTp#3J+hmfTMM3fh}IOV#u0DvuY|b3Gm$X= zE9pFx4>WKQQBGub_G?#X?0+8q(qA1uK*_Q6>*w=zz)Tlq4;5s^+lIWVW-UP#$W6&< z#M@(x7)|1eSM+!>)V?yOgrH$9Bl3-@dFmH=59{gOLt~a$Z^k_M0s1z|S{Affh7+S! zHGfOQTz07?tcd(Q%ExRtf9m(S%pip2E*JbohtjR*|0yhSx?BN;C=aHx1MsGaf-=SHZ zgl#+wq33Mvml(5@zv|^l7gM?O-q>csBqc|hyfle$G)f%7YQ>@bN>RUhg_B8iJ`XI*x{{T%4TJ<*<3rGkp|q|5!BTrmUeZ(fO2 zs@nCfJm`CY#HyytpVxY>nSTZRTng#9yC0jbz}y07jD0?POZg)sZTPwYQlrXS9~ajh zykPZmKzidv$qmN))Mctg_5E4Qw{mjayO7zAOi+2)wTQ@(DCx#B%&cLu^^zh@Yud|lAAWelJ!HwUZM?=l2!_GM!d_RK_=sa|d>%gqx1_ke zd|S?6Y4lPgz3NURY>L`TJkstHqSR%L-hXyiP{mWcv6XNWrd3qmDwn9cauRi>p$$VT z5TDfJz`@D2%puH_Zu*`Ib_>pVIKwsIO_I)^*^cGTL_BPbjkWq{zhpO*vES4fx}=t$ z7oX5AVb+PAiS=^232k^;F0BbQ^@QX&WjrOw6uRo%){r%3Ggsuuoje#`TZ$1$V>+e}rL6^FWr{j!EvLj&8=KsP0xH8cl3jA{Zdocw zol;vEROU`Ohh?PKJ(GoPvZw1PESoLz5sU3?!y@47?NaFOSVp3SC6K^DX zGCd+&OY^G3?DKMy@%5>7#Pn_pMUQmuOb^o!&?bc!-S==x>UUfIgA9&se$YihFQw^7 zlS-4A3^@_Y$#vuI-U+gU(H&4cZL(5~YQegg|mPIVNT2 zA7Mm6r-+1$U%eI{l&1oM`grXrAa*DNL-Cwz_iMH%Z9(9?>tT8amiMV!vu;nn{{a$8 zOP-7tA1M$3VAfRz3ONZqd!>8I4t7TaJ-<%c8}j# zrAVWWgT&cgusF$J*-9W#?aC$_&K04re?YF%1ZzsBnO7TWTpu~JuDZry?D8dHLXq2Z zqmTr;jKnOi<~**>sG@7^s^;VV$f!1EfB@6Qe5h$X{VXy`_Dfk(R8Ue1GuE|$JHt7C z&Tgur`8zP%tKVR{I#Ird5>7+8_TP%T;6wX1(zeA{NdT4H?x;wuxosA)6I7eFHMVmZ zy`-*VslY=+BmRIk)r$2c99O{ofhO@L@6f?)QNg?%v>F@C+Z7)w-q&Y~>qg5e`N0sI3EyDm;2I2j@wy}3lPJ!8NxR%C;}yr>)F-*~|qo9Pv2J~2Pz z$`aB`5jks6j%>-G)`lj_6hSwL>U2J!hz}I>CqRBMu$+^<7kX^I&?5jJ6IO zMEzHTQd3h#V{zJIk z>qeUR?BBPVMeLHi>k|W~6uCuOUtj(ziLLHNFb$}q{4(?j?@eD39kw;ztFx8ZlSrul z5_XwXPuF7w_vjNP!?29X?F`Dhh>I8?t$s|A{{$)wb7LytZoYD5&4eM^wsxs(xe>y~ zo}M!Ve@6ELycFge$p#zDk}lmXC_c*_;kJ+FQ+3fFTGLV`P)C(hWpucXWzbW$qk5=F;(*rPLW1)m%48(pr*qv&2Lg{lUzq}G}>?GYlh3`S$1m~>agyV1N6 zF9+SlhLZ^MCY#3_a7oEXIe*3ac?lYB_KDiDiTS%I-QmyuUH(@*o3PGUA*ak}z*pJy z=5VP)atG<&7T9gr@>trA--&WbS(yiS*#pI>`h5+U1HLLU?kRuUkgFikqJ^6UzrF6|#f{DI~eX~5|)pH)vPLznWd0iRj>JBEzF(k@t z3V$(`Sjvk40~L7=9~g5+E;xxz788Dy9#}n5Tm}P}OY;=qt2Wvs3RXk1oC>k5`qe?y zVt>B%oBxYjKb5>UN3U#b-R{(|6bA=WQkIx{GD+o&y65#1)Gh%z`qf_;GI&WAgn_w% zqd5AHxxrwFgRt2Lo7t#kGY*prCIj9COhF^oZ?PQG83oI_=`^G|6bi-DH?N^_=ps|? zWw$E0p@2n#Ys2goQ32}mpsm*0^@YT2lE#>($r0N=8ygz{RT&u>p6`JLfq4J7YePIrCNC%_!Z%^ zCZs!DFy*hA7XD7XOi@}vwQIXtZ0OnV^$tzm-(muJMnA`t4~(8Sfxl>h+>Pt|5>K2zow50k6lOwHsAc`{Z7_&()vb>1xAA--p=t6?3YBHrdLSMq5KE731YKETD=G?eUTLRfIo_T(*urREZP3jCImdax?i>sP=n zrHQ;V(pzPV#ya*HyTRakP^$)q!qI+=b5lvRiFr~oSF$Fnpjacustm%Goa^YFmodrJ zb#IJn<}B&81XXk?Mp1~60S6Ml64gjj$;48^8*-DoB34x1lw)4|6s!nKn+7Hch?7f2 zMwH%QQEbQa0tsvZ@Sr+y^MTQwGr3CE4VDE5vKNY0B0Ke&9~ExaCfOGtP+k;%q{=p? zSMALB#J{Ywkqwi)k>EAgx7qYk$(RwT^tA12BgoZ_f55J~5&qMF8)K3qoy?0%(5UVg z*zlW|x|1kl9Fs>m`PJbEXmIM&tjO-O@2r& zjMXwyy)oAPs+OLqX85()3bq5{r}LI~tJuqY+`sY{WTlF+%~5m7i6aLqGDcg$n0PEO z`IQHGXz@b!nCQ$a@>abD`PfLrh}`&^9&%TVIZm5AVJg{YoBob<8XLf$zcFs)NEQotlAVxGf;Cpu|)2KuW zjvL{C;8Jdv){?g=;An;7s+~UU5gH3drCDk5kB)j4FKIuNtW^ppdxnE2GGF}hbBg4 zji&JF%eF`=18C}l=mZWo4cqsi6TIw8`bI5CaV=JLTy@Ppbm9$=N1<^JEQ5%7(84>5 zyq5_dWOu968takn22-Vn$iAutgCT1&(o`J-=W4Ymo@VG*y{JhP8wQA3|D<`;;_`R@q89&8%)<>d5fB&_w0jj~nk+JRMNu z`$0+KTdFVF?tkVFY6;Js-Y%Pc*lJ z3bBZWhean2Efak=8EX{|Q|}Whf+xv$+1wU+hLsv*UK-xG_3AQ{SZenrUoE$NCqY=# z4Nrr71UTbLhy&Zk7vdWy6)fFfUyY?2Tk&ig1>0@$j>8lANv;VD*f)}wVWBB+cn7(h z+|v1H$YT7|u)z^B^&Ghx4y7^o)YSiS1^G!jMxNPY?$jU3VVE3GvXrqthtUc&3el4RV9E9KE89sV%Ea=^ZYH zT3m}xz20fb&5$hwrn^|mN#SJByQ>X$l1qXG@B-7#ux)1nFxZ*empG@U<X*30FW zOr6O(s;DCXHe0{#{U+3^AvRfS73pEDrN-sjU1iH)G8vsHMTVxoDY%0?%w11+OQ%Y` zWcCf+Yh<`%jQ&Qsyt!ozEAug&sE0O#AuQB(5wLanRzKoXoy*{;8ynYO(?8Y5fN@qQM3c9`Zx# zdiH(5B46=<=TyB*{0%gJns*jTgVj_`cTh=Q%@`A_#(DX5Z~tv794JZFC`RB3<(Ti? zllM_Z#zI869+Md-?)+p4hXS|y65LkFM*|7*nfFZ3v?RY_Vj4x#Bw9X zWEX|V6J0K+yI}@KYY=mSGkkO%92xAU%eJK0iX0K%dQ;o4Rn9Fb9LBs6KE~SVTwRQ0 z1EEz0tPh~rE=y{i(b5l;SCmJ=w*p? Date: Mon, 23 Mar 2020 11:18:15 +0800 Subject: [PATCH 171/523] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1f2306336..7a6c936a0 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ ## 技术交流 - > * 招聘大数据平台开发工程师，想了解岗位详细信息可以添加本人微信号ysqwhiletrue,注明招聘，如有意者发送简历至sishu@dtstack.com + * 招聘大数据平台开发工程师，想了解岗位详细信息可以添加本人微信号ysqwhiletrue,注明招聘，如有意者发送简历至sishu@dtstack.com - > * 可以搜索群号30537511或者可以扫下面的二维码进入钉钉群 + * 可以搜索群号30537511或者可以扫下面的二维码进入钉钉群

- +
From 14bc810e44bfeec911dce8249e8ed7d7dcda4bf1 Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 23 Mar 2020 15:04:54 +0800 Subject: [PATCH 172/523] =?UTF-8?q?=E5=88=9D=E6=AD=A5=E9=87=8D=E6=9E=84=20?= =?UTF-8?q?sql=E8=AF=AD=E5=8F=A5=E8=A7=A3=E6=9E=90=E9=80=BB=E8=BE=91?= =?UTF-8?q?=EF=BC=9B=20=E5=AD=97=E6=AE=B5=E6=9B=BF=E6=8D=A2=E5=92=8C?= =?UTF-8?q?=E9=87=8D=E5=91=BD=E5=90=8D=E7=9B=B8=E5=85=B3=E7=9A=84=E9=80=BB?= =?UTF-8?q?=E8=BE=91=E5=9C=A8=E8=A7=A3=E6=9E=90=E9=98=B6=E6=AE=B5=E5=AE=8C?= =?UTF-8?q?=E6=88=90=EF=BC=9B=20=E7=9B=AE=E6=A0=87=EF=BC=9A=E7=BB=8F?= =?UTF-8?q?=E8=BF=87=E8=A7=A3=E6=9E=90sql=E4=B9=8B=E5=90=8E=E7=9A=84?= =?UTF-8?q?=E6=89=80=E6=9C=89sql=E6=98=AF=E7=A1=AE=E5=AE=9A=E7=9A=84?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/Main.java | 8 +- .../com/dtstack/flink/sql/side/FieldInfo.java | 9 + .../com/dtstack/flink/sql/side/JoinInfo.java | 2 +- .../flink/sql/side/JoinNodeDealer.java | 54 ++- .../dtstack/flink/sql/side/SideSQLParser.java | 73 +--- .../dtstack/flink/sql/side/SideSqlExec.java | 68 +--- .../flink/sql/util/FieldReplaceUtil.java | 339 ++++++++++++++++++ 7 files changed, 397 insertions(+), 156 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index fed359c2f..443033b3b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -149,7 +149,13 @@ public static void main(String[] args) throws Exception { env.execute(name); } - private static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironment tableEnv,SqlTree sqlTree,Map sideTableMap,Map registerTableCache, StreamQueryConfig queryConfig) throws Exception { + private static void sqlTranslation(String localSqlPluginPath, + StreamTableEnvironment tableEnv, + SqlTree sqlTree, + Map sideTableMap, + Map registerTableCache, + StreamQueryConfig queryConfig) throws Exception { + SideSqlExec sideSqlExec = new SideSqlExec(); sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { diff --git a/core/src/main/java/com/dtstack/flink/sql/side/FieldInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/FieldInfo.java index 85bad8c2c..1259ddecf 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/FieldInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/FieldInfo.java @@ -85,4 +85,13 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(table, fieldName); } + + @Override + public String toString() { + return "FieldInfo{" + + "table='" + table + '\'' + + ", fieldName='" + fieldName + '\'' + + ", typeInformation=" + typeInformation + + '}'; + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java index 8254b763e..8854ff4ec 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java @@ -41,7 +41,7 @@ public class JoinInfo implements Serializable { private static final long serialVersionUID = -1L; //左表是否是维表 - private boolean leftIsSideTable; + private boolean leftIsSideTable = false; //右表是否是维表 private boolean rightIsSideTable; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index 33bd2b293..17bc91b40 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -74,7 +74,7 @@ public JoinNodeDealer(SideSQLParser sideSQLParser){ * 解析 join 操作 * @param joinNode * @param sideTableSet 标明哪些表名是维表 - * @param queueInfo + * @param queueInfo sql执行队列 * @param parentWhere join 关联的最上层的where 节点 * @param parentSelectList join 关联的最上层的select 节点 * @param joinFieldSet @@ -99,18 +99,13 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, String rightTableName = ""; String rightTableAlias = ""; - //TODO 含义需要更明确 - HashBiMap fieldReplaceRef = HashBiMap.create(); - - //如果是连续join 判断是否已经处理过添加到执行队列 + //抽取join中的的条件 extractJoinField(joinNode.getCondition(), joinFieldSet); - if(leftNode.getKind() == IDENTIFIER){ - leftTbName = leftNode.toString(); - } else if (leftNode.getKind() == JOIN) { + if (leftNode.getKind() == JOIN) { //处理连续join dealNestJoin(joinNode, sideTableSet, - queueInfo, parentWhere, parentSelectList, joinFieldSet, tableRef, fieldRef, parentSelectList, fieldReplaceRef); + queueInfo, parentWhere, parentSelectList, joinFieldSet, tableRef, fieldRef, parentSelectList); leftNode = joinNode.getLeft(); } @@ -135,24 +130,16 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, JoinInfo tableInfo = new JoinInfo(); tableInfo.setLeftTableName(leftTbName); tableInfo.setRightTableName(rightTableName); - if (StringUtils.isEmpty(leftTbAlias)){ - tableInfo.setLeftTableAlias(leftTbName); - } else { - tableInfo.setLeftTableAlias(leftTbAlias); - } - if (StringUtils.isEmpty(rightTableAlias)){ - tableInfo.setRightTableAlias(rightTableName); - } else { - tableInfo.setRightTableAlias(rightTableAlias); - } + leftTbAlias = StringUtils.isEmpty(leftTbAlias) ? leftTbName : leftTbAlias; + rightTableAlias = StringUtils.isEmpty(rightTableAlias) ? rightTableName : rightTableAlias; - tableInfo.setLeftIsSideTable(leftIsSide); + tableInfo.setLeftTableAlias(leftTbAlias); + tableInfo.setRightTableAlias(rightTableAlias); tableInfo.setRightIsSideTable(rightIsSide); tableInfo.setLeftNode(leftNode); tableInfo.setRightNode(rightNode); tableInfo.setJoinType(joinType); - tableInfo.setCondition(joinNode.getCondition()); TableUtils.replaceJoinFieldRefTableName(joinNode.getCondition(), fieldRef); @@ -216,6 +203,15 @@ public void extractJoinNeedSelectField(SqlNode leftNode, tableInfo.setRightSelectFieldInfo(rightTbSelectField); } + /** + * 指定的节点关联到的 select 中的字段和 where中的字段 + * @param sqlNode + * @param parentWhere + * @param parentSelectList + * @param tableRef + * @param joinFieldSet + * @return + */ public Set extractField(SqlNode sqlNode, SqlNode parentWhere, SqlNodeList parentSelectList, @@ -250,8 +246,7 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, Set> joinFieldSet, Map tableRef, Map fieldRef, - SqlNodeList parentSelectList, - HashBiMap fieldReplaceRef){ + SqlNodeList parentSelectList){ SqlJoin leftJoinNode = (SqlJoin) joinNode.getLeft(); SqlNode parentRightJoinNode = joinNode.getRight(); @@ -267,7 +262,7 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, SqlBasicCall buildAs = TableUtils.buildAsNodeByJoinInfo(joinInfo, null, null); if(rightIsSide){ - addSideInfoToExeQueue(queueInfo, joinInfo, joinNode, parentSelectList, parentWhere, fieldReplaceRef, tableRef); + addSideInfoToExeQueue(queueInfo, joinInfo, joinNode, parentSelectList, parentWhere, tableRef); } SqlNode newLeftNode = joinNode.getLeft(); @@ -280,7 +275,7 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, //替换leftNode 为新的查询 joinNode.setLeft(buildAs); - replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, parentSelectList, parentWhere, fieldReplaceRef); + replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, parentSelectList, parentWhere); } return joinInfo; @@ -294,7 +289,6 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, * @param joinNode * @param parentSelectList * @param parentWhere - * @param fieldReplaceRef * @param tableRef */ public void addSideInfoToExeQueue(Queue queueInfo, @@ -302,7 +296,6 @@ public void addSideInfoToExeQueue(Queue queueInfo, SqlJoin joinNode, SqlNodeList parentSelectList, SqlNode parentWhere, - HashBiMap fieldReplaceRef, Map tableRef){ //只处理维表 if(!joinInfo.isRightIsSideTable()){ @@ -315,7 +308,7 @@ public void addSideInfoToExeQueue(Queue queueInfo, //替换左表为新的表名称 joinNode.setLeft(buildAs); - replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, parentSelectList, parentWhere, fieldReplaceRef); + replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, parentSelectList, parentWhere); } /** @@ -325,14 +318,12 @@ public void addSideInfoToExeQueue(Queue queueInfo, * @param tableRef * @param parentSelectList * @param parentWhere - * @param fieldReplaceRef */ public void replaceSelectAndWhereField(SqlBasicCall buildAs, SqlNode leftJoinNode, Map tableRef, SqlNodeList parentSelectList, - SqlNode parentWhere, - HashBiMap fieldReplaceRef){ + SqlNode parentWhere){ String newLeftTableName = buildAs.getOperands()[1].toString(); Set fromTableNameSet = Sets.newHashSet(); @@ -343,6 +334,7 @@ public void replaceSelectAndWhereField(SqlBasicCall buildAs, } //替换select field 中的对应字段 + HashBiMap fieldReplaceRef = HashBiMap.create(); for(SqlNode sqlNode : parentSelectList.getList()){ for(String tbTmp : fromTableNameSet) { TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldReplaceRef); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index 061fe52a2..a94d218a2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -90,48 +90,16 @@ public Queue getExeQueue(String exeSql, Set sideTableSet) throws return queueInfo; } - private void checkAndReplaceMultiJoin(SqlNode sqlNode, Set sideTableSet) { - SqlKind sqlKind = sqlNode.getKind(); - switch (sqlKind) { - case WITH: { - SqlWith sqlWith = (SqlWith) sqlNode; - SqlNodeList sqlNodeList = sqlWith.withList; - for (SqlNode withAsTable : sqlNodeList) { - SqlWithItem sqlWithItem = (SqlWithItem) withAsTable; - checkAndReplaceMultiJoin(sqlWithItem.query, sideTableSet); - } - checkAndReplaceMultiJoin(sqlWith.body, sideTableSet); - break; - } - case INSERT: - SqlNode sqlSource = ((SqlInsert) sqlNode).getSource(); - checkAndReplaceMultiJoin(sqlSource, sideTableSet); - break; - case SELECT: - SqlNode sqlFrom = ((SqlSelect) sqlNode).getFrom(); - if (sqlFrom.getKind() != IDENTIFIER) { - checkAndReplaceMultiJoin(sqlFrom, sideTableSet); - } - break; - case JOIN: - convertSideJoinToNewQuery((SqlJoin) sqlNode, sideTableSet); - break; - case AS: - SqlNode info = ((SqlBasicCall) sqlNode).getOperands()[0]; - if (info.getKind() != IDENTIFIER) { - checkAndReplaceMultiJoin(info, sideTableSet); - } - break; - case UNION: - SqlNode unionLeft = ((SqlBasicCall) sqlNode).getOperands()[0]; - SqlNode unionRight = ((SqlBasicCall) sqlNode).getOperands()[1]; - checkAndReplaceMultiJoin(unionLeft, sideTableSet); - checkAndReplaceMultiJoin(unionRight, sideTableSet); - break; - } - } - + /** + * 解析 sql 根据维表 join关系重新组装新的sql + * @param sqlNode + * @param sideTableSet + * @param queueInfo + * @param parentWhere + * @param parentSelectList + * @return + */ public Object parseSql(SqlNode sqlNode, Set sideTableSet, Queue queueInfo, SqlNode parentWhere, SqlNodeList parentSelectList){ SqlKind sqlKind = sqlNode.getKind(); switch (sqlKind){ @@ -175,7 +143,9 @@ public Object parseSql(SqlNode sqlNode, Set sideTableSet, Queue JoinNodeDealer joinNodeDealer = new JoinNodeDealer(this); Set> joinFieldSet = Sets.newHashSet(); Map tableRef = Maps.newHashMap(); - return joinNodeDealer.dealJoinNode((SqlJoin) sqlNode, sideTableSet, queueInfo, parentWhere, parentSelectList, joinFieldSet, tableRef); + Map fieldRef = Maps.newHashMap(); + return joinNodeDealer.dealJoinNode((SqlJoin) sqlNode, sideTableSet, queueInfo, + parentWhere, parentSelectList, joinFieldSet, tableRef, fieldRef); case AS: SqlNode info = ((SqlBasicCall)sqlNode).getOperands()[0]; SqlNode alias = ((SqlBasicCall) sqlNode).getOperands()[1]; @@ -207,26 +177,7 @@ public Object parseSql(SqlNode sqlNode, Set sideTableSet, Queue return ""; } - private AliasInfo getSqlNodeAliasInfo(SqlNode sqlNode) { - SqlNode info = ((SqlBasicCall) sqlNode).getOperands()[0]; - SqlNode alias = ((SqlBasicCall) sqlNode).getOperands()[1]; - String infoStr = info.getKind() == IDENTIFIER ? info.toString() : null; - - AliasInfo aliasInfo = new AliasInfo(); - aliasInfo.setName(infoStr); - aliasInfo.setAlias(alias.toString()); - return aliasInfo; - } - /** - * 将和维表关联的join 替换为一个新的查询 - * @param sqlNode - * @param sideTableSet - */ - private void convertSideJoinToNewQuery(SqlJoin sqlNode, Set sideTableSet) { - checkAndReplaceMultiJoin(sqlNode.getLeft(), sideTableSet); - checkAndReplaceMultiJoin(sqlNode.getRight(), sideTableSet); - } public void setLocalTableCache(Map localTableCache) { diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 0a9668916..920dec07c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -84,8 +84,12 @@ public class SideSqlExec { private Map localTableCache = Maps.newHashMap(); - public void exec(String sql, Map sideTableMap, StreamTableEnvironment tableEnv, - Map tableCache, StreamQueryConfig queryConfig, CreateTmpTableParser.SqlParserResult createView) throws Exception { + public void exec(String sql, + Map sideTableMap, + StreamTableEnvironment tableEnv, + Map tableCache, + StreamQueryConfig queryConfig, + CreateTmpTableParser.SqlParserResult createView) throws Exception { if(localSqlPluginPath == null){ throw new RuntimeException("need to set localSqlPluginPath"); } @@ -192,66 +196,6 @@ private FieldReplaceInfo parseAsQuery(SqlBasicCall asSqlNode, Map } - /** - * 添加字段别名 - * @param pollSqlNode - * @param fieldList - * @param mappingTable - */ - private void addAliasForFieldNode(SqlNode pollSqlNode, List fieldList, HashBasedTable mappingTable) { - SqlKind sqlKind = pollSqlNode.getKind(); - switch (sqlKind) { - case INSERT: - SqlNode source = ((SqlInsert) pollSqlNode).getSource(); - addAliasForFieldNode(source, fieldList, mappingTable); - break; - case AS: - addAliasForFieldNode(((SqlBasicCall) pollSqlNode).getOperands()[0], fieldList, mappingTable); - break; - case SELECT: - SqlNodeList selectList = ((SqlSelect) pollSqlNode).getSelectList(); - selectList.getList().forEach(node -> { - if (node.getKind() == IDENTIFIER) { - SqlIdentifier sqlIdentifier = (SqlIdentifier) node; - if (sqlIdentifier.names.size() == 1) { - return; - } - // save real field - String fieldName = sqlIdentifier.names.get(1); - if (!fieldName.endsWith("0") || fieldName.endsWith("0") && mappingTable.columnMap().containsKey(fieldName)) { - fieldList.add(fieldName); - } - - } - }); - for (int i = 0; i < selectList.getList().size(); i++) { - SqlNode node = selectList.get(i); - if (node.getKind() == IDENTIFIER) { - SqlIdentifier sqlIdentifier = (SqlIdentifier) node; - if (sqlIdentifier.names.size() == 1) { - return; - } - String name = sqlIdentifier.names.get(1); - // avoid real field pv0 convert pv - if (name.endsWith("0") && !fieldList.contains(name) && !fieldList.contains(name.substring(0, name.length() - 1))) { - SqlOperator operator = new SqlAsOperator(); - SqlParserPos sqlParserPos = new SqlParserPos(0, 0); - - SqlIdentifier sqlIdentifierAlias = new SqlIdentifier(name.substring(0, name.length() - 1), null, sqlParserPos); - SqlNode[] sqlNodes = new SqlNode[2]; - sqlNodes[0] = sqlIdentifier; - sqlNodes[1] = sqlIdentifierAlias; - SqlBasicCall sqlBasicCall = new SqlBasicCall(operator, sqlNodes, sqlParserPos); - - selectList.set(i, sqlBasicCall); - } - } - } - break; - } - } - - public AliasInfo parseASNode(SqlNode sqlNode) throws SqlParseException { SqlKind sqlKind = sqlNode.getKind(); if(sqlKind != AS){ diff --git a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java new file mode 100644 index 000000000..10919ca5b --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java @@ -0,0 +1,339 @@ +package com.dtstack.flink.sql.util; + +import com.dtstack.flink.sql.side.FieldReplaceInfo; +import com.google.common.collect.Lists; +import org.apache.calcite.sql.*; +import org.apache.calcite.sql.fun.SqlCase; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.commons.collections.CollectionUtils; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import static org.apache.calcite.sql.SqlKind.*; + +/** + * 替换 字段 + */ +public class FieldReplaceUtil { + + /** + * 需要考虑更多的情况 + */ + public static void replaceFieldName(SqlNode sqlNode, + String oldTbName, + String newTbName, + Map mappingField) { + SqlKind sqlKind = sqlNode.getKind(); + switch (sqlKind) { + case INSERT: + SqlNode sqlSource = ((SqlInsert) sqlNode).getSource(); + replaceFieldName(sqlSource, oldTbName, newTbName, mappingField); + break; + case AS: + SqlNode asNode = ((SqlBasicCall) sqlNode).getOperands()[0]; + replaceFieldName(asNode, oldTbName, newTbName, mappingField); + break; + case SELECT: + SqlSelect sqlSelect = (SqlSelect) sqlNode; + SqlNodeList sqlSelectList = sqlSelect.getSelectList(); + SqlNode whereNode = sqlSelect.getWhere(); + SqlNodeList sqlGroup = sqlSelect.getGroup(); + + //TODO 抽取,暂时使用使用单个join条件作为测试 + if(sqlSelect.getFrom().getKind().equals(JOIN)){ + SqlJoin joinNode = (SqlJoin) sqlSelect.getFrom(); + SqlNode joinCondition = joinNode.getCondition(); + replaceFieldName(((SqlBasicCall)joinCondition).operands[0], oldTbName, newTbName, mappingField); + replaceFieldName(((SqlBasicCall)joinCondition).operands[1], oldTbName, newTbName, mappingField); + } + + //TODO 暂时不处理having + SqlNode sqlHaving = sqlSelect.getHaving(); + + List newSelectNodeList = Lists.newArrayList(); + for( int i=0; i replaceNodeList = replaceSelectStarFieldName(selectNode, replaceInfo); + //newSelectNodeList.addAll(replaceNodeList); + throw new RuntimeException("not support table.* now"); + } + + SqlNode replaceNode = replaceSelectFieldName(selectNode, oldTbName, newTbName, mappingField); + if(replaceNode == null){ + continue; + } + + newSelectNodeList.add(replaceNode); + } + + SqlNodeList newSelectList = new SqlNodeList(newSelectNodeList, sqlSelectList.getParserPosition()); + sqlSelect.setSelectList(newSelectList); + + //where + if(whereNode != null){ + SqlNode[] sqlNodeList = ((SqlBasicCall)whereNode).getOperands(); + for(int i =0; i mappingField) { + if(orderNode.getKind() == IDENTIFIER){ + return createNewIdentify((SqlIdentifier) orderNode, oldTbName, newTbName, mappingField); + } else if (orderNode instanceof SqlBasicCall) { + SqlBasicCall sqlBasicCall = (SqlBasicCall) orderNode; + for(int i=0; i mappingField){ + if(groupNode.getKind() == IDENTIFIER){ + return createNewIdentify((SqlIdentifier) groupNode, oldTbName, newTbName, mappingField); + }else if(groupNode instanceof SqlBasicCall){ + SqlBasicCall sqlBasicCall = (SqlBasicCall) groupNode; + for(int i=0; i mappingField){ + + if (sqlIdentifier.names.size() == 1) { + return sqlIdentifier; + } + + String tableName = sqlIdentifier.names.get(0); + String fieldName = sqlIdentifier.names.get(1); + if(!tableName.equalsIgnoreCase(oldTbName)){ + return sqlIdentifier; + } + + String mappingFieldName = mappingField.get(fieldName); + if(mappingFieldName == null){ + return sqlIdentifier; + } + + sqlIdentifier = sqlIdentifier.setName(0, newTbName); + sqlIdentifier = sqlIdentifier.setName(1, mappingFieldName); + return sqlIdentifier; + } + + public static boolean filterNodeWithTargetName(SqlNode sqlNode, String targetTableName) { + + SqlKind sqlKind = sqlNode.getKind(); + switch (sqlKind){ + case SELECT: + SqlNode fromNode = ((SqlSelect)sqlNode).getFrom(); + if(fromNode.getKind() == AS && ((SqlBasicCall)fromNode).getOperands()[0].getKind() == IDENTIFIER){ + if(((SqlBasicCall)fromNode).getOperands()[0].toString().equalsIgnoreCase(targetTableName)){ + return true; + }else{ + return false; + } + }else{ + return filterNodeWithTargetName(fromNode, targetTableName); + } + case AS: + SqlNode aliasName = ((SqlBasicCall)sqlNode).getOperands()[1]; + return aliasName.toString().equalsIgnoreCase(targetTableName); + case JOIN: + SqlNode leftNode = ((SqlJoin)sqlNode).getLeft(); + SqlNode rightNode = ((SqlJoin)sqlNode).getRight(); + boolean leftReturn = filterNodeWithTargetName(leftNode, targetTableName); + boolean rightReturn = filterNodeWithTargetName(rightNode, targetTableName); + + return leftReturn || rightReturn; + + default: + return false; + } + } + + public static SqlNode replaceSelectFieldName(SqlNode selectNode, + String oldTbName, + String newTbName, + Map mappingField) { + + if (selectNode.getKind() == AS) { + SqlNode leftNode = ((SqlBasicCall) selectNode).getOperands()[0]; + SqlNode replaceNode = replaceSelectFieldName(leftNode, oldTbName, newTbName, mappingField); + if (replaceNode != null) { + ((SqlBasicCall) selectNode).getOperands()[0] = replaceNode; + } + + return selectNode; + }else if(selectNode.getKind() == IDENTIFIER){ + return createNewIdentify((SqlIdentifier) selectNode, oldTbName, newTbName, mappingField); + }else if(selectNode.getKind() == LITERAL || selectNode.getKind() == LITERAL_CHAIN){//字面含义 + return selectNode; + }else if( AGGREGATE.contains(selectNode.getKind()) + || AVG_AGG_FUNCTIONS.contains(selectNode.getKind()) + || COMPARISON.contains(selectNode.getKind()) + || selectNode.getKind() == OTHER_FUNCTION + || selectNode.getKind() == DIVIDE + || selectNode.getKind() == CAST + || selectNode.getKind() == TRIM + || selectNode.getKind() == TIMES + || selectNode.getKind() == PLUS + || selectNode.getKind() == NOT_IN + || selectNode.getKind() == OR + || selectNode.getKind() == AND + || selectNode.getKind() == MINUS + || selectNode.getKind() == TUMBLE + || selectNode.getKind() == TUMBLE_START + || selectNode.getKind() == TUMBLE_END + || selectNode.getKind() == SESSION + || selectNode.getKind() == SESSION_START + || selectNode.getKind() == SESSION_END + || selectNode.getKind() == HOP + || selectNode.getKind() == HOP_START + || selectNode.getKind() == HOP_END + || selectNode.getKind() == BETWEEN + || selectNode.getKind() == IS_NULL + || selectNode.getKind() == IS_NOT_NULL + || selectNode.getKind() == CONTAINS + || selectNode.getKind() == TIMESTAMP_ADD + || selectNode.getKind() == TIMESTAMP_DIFF + || selectNode.getKind() == LIKE + + ){ + SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; + for(int i=0; i replaceSelectStarFieldName(SqlNode selectNode, FieldReplaceInfo replaceInfo){ + SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; + List sqlNodes = Lists.newArrayList(); + if(sqlIdentifier.isStar()){//处理 [* or table.*] + int identifierSize = sqlIdentifier.names.size(); + Collection columns = null; + if(identifierSize == 1){ + columns = replaceInfo.getMappingTable().values(); + }else{ + columns = replaceInfo.getMappingTable().row(sqlIdentifier.names.get(0)).values(); + } + + for(String colAlias : columns){ + SqlParserPos sqlParserPos = new SqlParserPos(0, 0); + List columnInfo = Lists.newArrayList(); + columnInfo.add(replaceInfo.getTargetTableAlias()); + columnInfo.add(colAlias); + SqlIdentifier sqlIdentifierAlias = new SqlIdentifier(columnInfo, sqlParserPos); + sqlNodes.add(sqlIdentifierAlias); + } + + return sqlNodes; + }else{ + throw new RuntimeException("is not a star select field." + selectNode); + } + } + +} From a0fb9b3729a80d74dd74d100e2f85c1cf0756316 Mon Sep 17 00:00:00 2001 From: huangxiaowei Date: Tue, 24 Mar 2020 07:21:08 +0800 Subject: [PATCH 173/523] 1. one wrong spelling 2. change some public method to private. --- .../flink/sql/exec/ExecuteProcessHelper.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 058ef6097..571d504cc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -137,7 +137,7 @@ public static ParamsInfo parseParams(String[] args) throws Exception { * @param pluginLoadMode * @return */ - public static boolean checkRemoteSqlPluginPath(String remoteSqlPluginPath, String deployMode, String pluginLoadMode) { + private static boolean checkRemoteSqlPluginPath(String remoteSqlPluginPath, String deployMode, String pluginLoadMode) { if (StringUtils.isEmpty(remoteSqlPluginPath)) { return StringUtils.equalsIgnoreCase(pluginLoadMode, EPluginLoadMode.SHIPFILE.name()) || StringUtils.equalsIgnoreCase(deployMode, ClusterMode.local.name()); @@ -174,7 +174,7 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf } - public static List getExternalJarUrls(String addJarListStr) throws java.io.IOException { + private static List getExternalJarUrls(String addJarListStr) throws java.io.IOException { List jarUrlList = Lists.newArrayList(); if (Strings.isNullOrEmpty(addJarListStr)) { return jarUrlList; @@ -187,7 +187,7 @@ public static List getExternalJarUrls(String addJarListStr) throws java.io. } return jarUrlList; } - + private static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironment tableEnv, SqlTree sqlTree,Map sideTableMap, @@ -238,7 +238,7 @@ private static void sqlTranslation(String localSqlPluginPath, } } - public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrlList, TableEnvironment tableEnv) + private static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrlList, TableEnvironment tableEnv) throws IllegalAccessException, InvocationTargetException { // udf和tableEnv须由同一个类加载器加载 ClassLoader levelClassLoader = tableEnv.getClass().getClassLoader(); @@ -266,9 +266,9 @@ public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrl * @return * @throws Exception */ - public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, + private static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { - Set pluginClassPatshSets = Sets.newHashSet(); + Set pluginClassPathSets = Sets.newHashSet(); WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); for (TableInfo tableInfo : sqlTree.getTableInfoMap().values()) { @@ -306,7 +306,7 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment registerTableCache.put(tableInfo.getName(), regTable); URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), SourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); - pluginClassPatshSets.add(sourceTablePathUrl); + pluginClassPathSets.add(sourceTablePathUrl); } else if (tableInfo instanceof TargetTableInfo) { TableSink tableSink = StreamSinkFactory.getTableSink((TargetTableInfo) tableInfo, localSqlPluginPath); @@ -314,18 +314,18 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), TargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); - pluginClassPatshSets.add(sinkTablePathUrl); + pluginClassPathSets.add(sinkTablePathUrl); } else if (tableInfo instanceof SideTableInfo) { String sideOperator = ECacheType.ALL.name().equals(((SideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; sideTableMap.put(tableInfo.getName(), (SideTableInfo) tableInfo); URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, SideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); - pluginClassPatshSets.add(sideTablePathUrl); + pluginClassPathSets.add(sideTablePathUrl); } else { throw new RuntimeException("not support table type:" + tableInfo.getType()); } } - return pluginClassPatshSets; + return pluginClassPathSets; } /** @@ -333,7 +333,7 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment * @param env * @param classPathSet */ - public static void registerPluginUrlToCachedFile(StreamExecutionEnvironment env, Set classPathSet) { + private static void registerPluginUrlToCachedFile(StreamExecutionEnvironment env, Set classPathSet) { int i = 0; for (URL url : classPathSet) { String classFileName = String.format(CLASS_FILE_NAME_FMT, i); @@ -342,7 +342,7 @@ public static void registerPluginUrlToCachedFile(StreamExecutionEnvironment env, } } - public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperties, String deployMode) throws Exception { + private static StreamExecutionEnvironment getStreamExeEnv(Properties confProperties, String deployMode) throws Exception { StreamExecutionEnvironment env = !ClusterMode.local.name().equals(deployMode) ? StreamExecutionEnvironment.getExecutionEnvironment() : new MyLocalStreamEnvironment(); From 05723687299f0c00b1ef33dec7179e9e10512a3a Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 24 Mar 2020 10:43:32 +0800 Subject: [PATCH 174/523] oracle char type --- .../java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java | 1 + .../java/com/dtstack/flink/sql/sink/oracle/OracleSink.java | 4 ---- .../src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java | 1 + 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java index d51e33056..bc248f8b7 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.sink.oracle; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index db526ce79..ee0239a1f 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -31,10 +31,6 @@ */ public class OracleSink extends RdbSink implements IStreamSinkGener { - private final String SQL_DEFAULT_PLACEHOLDER = " ? "; - private final String DEAL_CHAR_KEY = "char"; - private String RPAD_FORMAT = " rpad(?, %d, ' ') "; - public OracleSink() { super(new OracleDialect()); } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java index a03bc81b6..ddd929d2c 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java @@ -106,6 +106,7 @@ public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { this.sqlTypes = JDBCTypeConvertUtils.buildSqlTypes(fieldTypeArray); this.allReplace = rdbTableInfo.isAllReplace(); this.updateMode = rdbTableInfo.getUpdateMode(); + rdbTableInfo.getFieldList(); return this; } From a78738963037df956cacf51bdca24f58b5f3165e Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 24 Mar 2020 15:51:38 +0800 Subject: [PATCH 175/523] =?UTF-8?q?=E8=A1=A5=E5=85=85=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E8=B6=85=E6=97=B6=E8=BE=93=E5=87=BAleft=20join?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/BaseAsyncReqRow.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 221bf2c85..8713b541a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -145,11 +145,15 @@ public void timeout(CRow input, ResultFuture resultFuture) throws Exceptio LOG.info("Async function call has timed out. input:{}, timeOutNum:{}",input.toString(), timeOutNum); } timeOutNum ++; - if(timeOutNum > sideInfo.getSideTableInfo().getAsyncTimeoutNumLimit()){ - resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); - } else { + if(sideInfo.getJoinType() == JoinType.LEFT){ resultFuture.complete(null); + return; + } + if(timeOutNum > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Integer.MAX_VALUE)){ + resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); + return; } + resultFuture.complete(null); } protected void preInvoke(CRow input, ResultFuture resultFuture){ From 23b410c04682342f37be1b785e10f14640e97b1e Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 24 Mar 2020 16:03:52 +0800 Subject: [PATCH 176/523] =?UTF-8?q?=E8=BE=93=E5=87=BA=E5=A0=86=E6=A0=88?= =?UTF-8?q?=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 6b4298f24..10b9d93db 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -30,6 +30,7 @@ import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import com.google.common.collect.Lists; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; @@ -45,6 +46,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; /** * Date: 2018/11/26 @@ -90,6 +92,7 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { AtomicInteger failCounter = new AtomicInteger(0); + AtomicReference connErrMsg = new AtomicReference<>(); while(true){ AtomicBoolean connectFinish = new AtomicBoolean(false); rdbSqlClient.getConnection(conn -> { @@ -98,17 +101,20 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul if(failCounter.getAndIncrement() % 1000 == 0){ logger.error("getConnection error", conn.cause()); } + connErrMsg.set(conn.cause()); conn.result().close(); } ScheduledFuture timerFuture = registerTimer(input, resultFuture); cancelTimerWhenComplete(resultFuture, timerFuture); handleQuery(conn.result(), inputParams, input, resultFuture); }); + while(!connectFinish.get()){ Thread.sleep(50); } + if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ - resultFuture.completeExceptionally(new RuntimeException("connection fail")); + resultFuture.completeExceptionally(connErrMsg.get()); return; } } From bfce8c88b479658f9ab02d7b303fb178b6089815 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 24 Mar 2020 16:26:47 +0800 Subject: [PATCH 177/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=80=80=E5=87=BA?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 10b9d93db..c48635bf3 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -93,30 +93,32 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { AtomicInteger failCounter = new AtomicInteger(0); AtomicReference connErrMsg = new AtomicReference<>(); - while(true){ + AtomicBoolean finishFlag = new AtomicBoolean(false); + while(!finishFlag.get()){ AtomicBoolean connectFinish = new AtomicBoolean(false); rdbSqlClient.getConnection(conn -> { - connectFinish.set(true); if(conn.failed()){ + connectFinish.set(true); if(failCounter.getAndIncrement() % 1000 == 0){ logger.error("getConnection error", conn.cause()); } + if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ + resultFuture.completeExceptionally(connErrMsg.get()); + finishFlag.set(true); + } connErrMsg.set(conn.cause()); conn.result().close(); + return; } ScheduledFuture timerFuture = registerTimer(input, resultFuture); cancelTimerWhenComplete(resultFuture, timerFuture); handleQuery(conn.result(), inputParams, input, resultFuture); + finishFlag.set(true); }); while(!connectFinish.get()){ Thread.sleep(50); } - - if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ - resultFuture.completeExceptionally(connErrMsg.get()); - return; - } } } From 267511a2659e2fbc7ba162293af75f6333f7f964 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 24 Mar 2020 16:27:36 +0800 Subject: [PATCH 178/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=80=80=E5=87=BA?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index c48635bf3..827d94dad 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -92,7 +92,6 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { AtomicInteger failCounter = new AtomicInteger(0); - AtomicReference connErrMsg = new AtomicReference<>(); AtomicBoolean finishFlag = new AtomicBoolean(false); while(!finishFlag.get()){ AtomicBoolean connectFinish = new AtomicBoolean(false); @@ -103,10 +102,9 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul logger.error("getConnection error", conn.cause()); } if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ - resultFuture.completeExceptionally(connErrMsg.get()); + resultFuture.completeExceptionally(conn.cause()); finishFlag.set(true); } - connErrMsg.set(conn.cause()); conn.result().close(); return; } From 4e8a00e4ccf12b865f163e2d87a923be53cb9fce Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 24 Mar 2020 17:39:43 +0800 Subject: [PATCH 179/523] =?UTF-8?q?=E9=98=BB=E5=A1=9E=E9=80=BB=E8=BE=91?= =?UTF-8?q?=E9=87=8D=E6=96=B0=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/rdb/async/RdbAsyncReqRow.java | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 827d94dad..011920afe 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -43,6 +43,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.Future; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -77,6 +78,8 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private transient SQLClient rdbSqlClient; + private final static AtomicBoolean CONN_STATUS = new AtomicBoolean(true); + private Logger logger = LoggerFactory.getLogger(getClass()); public RdbAsyncReqRow(BaseSideInfo sideInfo) { @@ -91,13 +94,32 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { + + while (!CONN_STATUS.get()){//network is unhealth + Thread.sleep(100); + } + rdbSqlClient.getConnection(conn -> { + if(conn.failed()){ + CONN_STATUS.set(false); + connectWithRetry(inputParams, input, resultFuture, rdbSqlClient); + return; + } + CONN_STATUS.set(true); + ScheduledFuture timerFuture = registerTimer(input, resultFuture); + cancelTimerWhenComplete(resultFuture, timerFuture); + handleQuery(conn.result(), inputParams, input, resultFuture); + }); + + } + + private void connectWithRetry(Map inputParams, CRow input, ResultFuture resultFuture, SQLClient rdbSqlClient) { AtomicInteger failCounter = new AtomicInteger(0); AtomicBoolean finishFlag = new AtomicBoolean(false); while(!finishFlag.get()){ - AtomicBoolean connectFinish = new AtomicBoolean(false); + AtomicBoolean connFinish = new AtomicBoolean(false); rdbSqlClient.getConnection(conn -> { + connFinish.set(true); if(conn.failed()){ - connectFinish.set(true); if(failCounter.getAndIncrement() % 1000 == 0){ logger.error("getConnection error", conn.cause()); } @@ -108,16 +130,22 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul conn.result().close(); return; } + CONN_STATUS.set(true); ScheduledFuture timerFuture = registerTimer(input, resultFuture); cancelTimerWhenComplete(resultFuture, timerFuture); handleQuery(conn.result(), inputParams, input, resultFuture); finishFlag.set(true); }); + while(!connFinish.get()){ + try { + Thread.sleep(50); + } catch (InterruptedException e){ + logger.error("", e); + } - while(!connectFinish.get()){ - Thread.sleep(50); } } + } @Override From e0af118f6beca7965960f06341a9d9f672a181e8 Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Tue, 24 Mar 2020 21:46:17 +0800 Subject: [PATCH 180/523] =?UTF-8?q?oceanbase-sink=E6=8F=92=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/pom.xml | 6 -- oceanbase/oceanbase-sink/pom.xml | 90 +++++++++++++++++++ .../sql/sink/ocean/OceanbaseDialect.java | 59 ++++++++++++ .../flink/sql/sink/ocean/OceanbaseSink.java | 42 +++++++++ .../sink/ocean/table/OceanbaseSinkParser.java | 25 ++++++ oceanbase/pom.xml | 39 ++++++++ pom.xml | 1 + 7 files changed, 256 insertions(+), 6 deletions(-) create mode 100644 oceanbase/oceanbase-sink/pom.xml create mode 100644 oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java create mode 100644 oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java create mode 100644 oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java create mode 100644 oceanbase/pom.xml diff --git a/core/pom.xml b/core/pom.xml index eb470a3c0..f57c1ea47 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -116,12 +116,6 @@ flink-statebackend-rocksdb_2.11 ${flink.version} - - - junit - junit - 4.12 - com.aiweiergou tools-logger diff --git a/oceanbase/oceanbase-sink/pom.xml b/oceanbase/oceanbase-sink/pom.xml new file mode 100644 index 000000000..02514f0d1 --- /dev/null +++ b/oceanbase/oceanbase-sink/pom.xml @@ -0,0 +1,90 @@ + + + + sql.oceanbase + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.sink.oceanbase + jar + oceanbase-sink + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.sink.rdb + ${sql.sink.rdb.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java new file mode 100644 index 000000000..b10f4a3b3 --- /dev/null +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java @@ -0,0 +1,59 @@ +package com.dtstack.flink.sql.sink.ocean; + +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; + +import java.util.Arrays; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * @author : tiezhu + * @date : 2020/3/24 + */ +public class OceanbaseDialect implements JDBCDialect { + @Override + public boolean canHandle(String url) { + return url.startsWith("jdbc:mysql:"); + } + + @Override + public Optional defaultDriverName() { + return Optional.of("com.mysql.jdbc.Driver"); + } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } + + @Override + public Optional getUpsertStatement(String schema, + String tableName, + String[] fieldNames, + String[] uniqueKeyFields, + boolean allReplace) { + return allReplace ? + buildReplaceIntoStatement(tableName, fieldNames) : + buildDuplicateUpsertStatement(tableName, fieldNames); + } + + private Optional buildDuplicateUpsertStatement(String tableName, String[] fieldsName) { + String updateClause = Arrays.stream(fieldsName).map(f -> quoteIdentifier(f) + + "IFNULL(VALUES(" + quoteIdentifier(f) + ")," + quoteIdentifier(f) + ")") + .collect(Collectors.joining(",")); + return Optional.of(getInsertIntoStatement("", tableName, fieldsName, null) + + " ON DUPLICATE KEY UPDATE " + updateClause + ); + } + + private Optional buildReplaceIntoStatement(String tableName, String[] fieldsNames) { + String columns = Arrays.stream(fieldsNames) + .map(this::quoteIdentifier) + .collect(Collectors.joining(",")); + String placeholders = Arrays.stream(fieldsNames) + .map(f -> "?") + .collect(Collectors.joining(",")); + return Optional.of("REPLACE INTO " + quoteIdentifier(tableName) + + "(" + columns + ") VALUES (" + placeholders + ")"); + } +} diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java new file mode 100644 index 000000000..c08cf9bb9 --- /dev/null +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java @@ -0,0 +1,42 @@ +package com.dtstack.flink.sql.sink.ocean; + +import com.dtstack.flink.sql.sink.IStreamSinkGener; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.JDBCOptions; +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; + +/** + * @author : tiezhu + * @date : 2020/3/24 + */ +public class OceanbaseSink extends AbstractRdbSink implements IStreamSinkGener { + + private static final String OCEANBESE_DRIVER = "com.mysql.jdbc.Driver"; + + public OceanbaseSink() { + super(new OceanbaseDialect()); + } + + @Override + public JDBCUpsertOutputFormat getOutputFormat() { + JDBCOptions oceanbaseOptions = JDBCOptions.builder() + .setDbUrl(dbUrl) + .setDialect(jdbcDialect) + .setUsername(userName) + .setPassword(password) + .setTableName(tableName) + .build(); + + return JDBCUpsertOutputFormat.builder() + .setOptions(oceanbaseOptions) + .setFieldNames(fieldNames) + .setFlushMaxSize(batchNum) + .setFlushIntervalMills(batchWaitInterval) + .setFieldTypes(sqlTypes) + .setKeyFields(primaryKeys) + .setAllReplace(allReplace) + .setUpdateMode(updateMode) + .build(); + } +} diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java new file mode 100644 index 000000000..f9f3fb42a --- /dev/null +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java @@ -0,0 +1,25 @@ +package com.dtstack.flink.sql.sink.ocean.table; + +import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; + +import java.util.Map; + +/** + * @author : tiezhu + * @date : 2020/3/24 + */ +public class OceanbaseSinkParser extends RdbSinkParser { + + private static final String CURRENT_TYPE = "oceanbase"; + + @Override + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + + AbstractTableInfo oceanbaseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + + oceanbaseTableInfo.setType(CURRENT_TYPE); + + return oceanbaseTableInfo; + } +} diff --git a/oceanbase/pom.xml b/oceanbase/pom.xml new file mode 100644 index 000000000..231665282 --- /dev/null +++ b/oceanbase/pom.xml @@ -0,0 +1,39 @@ + + + + flink.sql + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.oceanbase + pom + + oceanbase-sink + + + + 5.1.40 + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.core + ${sql.core.version} + provided + + + + mysql + mysql-connector-java + ${mysql.connector.version} + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index de963240c..87c8e97e6 100644 --- a/pom.xml +++ b/pom.xml @@ -34,6 +34,7 @@ impala db2 polardb + oceanbase From 201dd7480e045d628cc3cf5ca5eb7779c0c5b1a7 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 24 Mar 2020 21:50:57 +0800 Subject: [PATCH 181/523] fix --- .../java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java | 1 - 1 file changed, 1 deletion(-) diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java index c08cf9bb9..8aede349b 100644 --- a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java @@ -3,7 +3,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** From 04028d5914fb245301f4faec51cb4fb4a2131783 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 25 Mar 2020 12:38:45 +0800 Subject: [PATCH 182/523] deal oracle char type --- .../flink/sql/sink/oracle/OracleDialect.java | 88 +++++++++++++++---- .../flink/sql/sink/oracle/OracleSink.java | 4 + .../dtstack/flink/sql/sink/rdb/RdbSink.java | 9 +- 3 files changed, 83 insertions(+), 18 deletions(-) diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java index bc248f8b7..a687aa012 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java @@ -35,6 +35,14 @@ */ public class OracleDialect implements JDBCDialect { + private final String SQL_DEFAULT_PLACEHOLDER = " ? "; + private final String DEAL_CHAR_KEY = "char"; + private String RPAD_FORMAT = " rpad(?, %d, ' ') "; + + private List fieldList; + private List fieldTypeList; + private List fieldExtraInfoList; + @Override public boolean canHandle(String url) { return url.startsWith("jdbc:oracle:"); @@ -48,27 +56,31 @@ public Optional defaultDriverName() { @Override public Optional getUpsertStatement(String schema, String tableName, String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { tableName = DtStringUtil.getTableFullPath(schema, tableName); - StringBuilder sb = new StringBuilder(); - sb.append("MERGE INTO " + tableName + " T1 USING " - + "(" + buildDualQueryStatement(fieldNames) + ") T2 ON (" - + buildConnectionConditions(uniqueKeyFields) + ") "); + StringBuilder mergeIntoSql = new StringBuilder(); + mergeIntoSql.append("MERGE INTO " + tableName + " T1 USING (") + .append(buildDualQueryStatement(fieldNames)) + .append(") T2 ON (") + .append(buildConnectionConditions(uniqueKeyFields) + ") "); String updateSql = buildUpdateConnection(fieldNames, uniqueKeyFields, allReplace); if (StringUtils.isNotEmpty(updateSql)) { - sb.append(" WHEN MATCHED THEN UPDATE SET "); - sb.append(updateSql); + mergeIntoSql.append(" WHEN MATCHED THEN UPDATE SET "); + mergeIntoSql.append(updateSql); } - sb.append(" WHEN NOT MATCHED THEN " - + "INSERT (" + Arrays.stream(fieldNames).map(this::quoteIdentifier).collect(Collectors.joining(",")) + ") VALUES (" - + Arrays.stream(fieldNames).map(col -> "T2." + quoteIdentifier(col)).collect(Collectors.joining(",")) + ")"); + mergeIntoSql.append(" WHEN NOT MATCHED THEN ") + .append("INSERT (") + .append(Arrays.stream(fieldNames).map(this::quoteIdentifier).collect(Collectors.joining(","))) + .append(") VALUES (") + .append(Arrays.stream(fieldNames).map(col -> "T2." + quoteIdentifier(col)).collect(Collectors.joining(","))) + .append(")"); - return Optional.of(sb.toString()); + return Optional.of(mergeIntoSql.toString()); } /** - * build T1."A"=T2."A" or T1."A"=nvl(T2."A",T1."A") + * build T1."A"=T2."A" or T1."A"=nvl(T2."A",T1."A") * @param fieldNames * @param uniqueKeyFields * @param allReplace @@ -76,11 +88,18 @@ public Optional getUpsertStatement(String schema, String tableName, Stri */ private String buildUpdateConnection(String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { List uniqueKeyList = Arrays.asList(uniqueKeyFields); - return Arrays.stream(fieldNames).filter(col -> !uniqueKeyList.contains(col)).map(col -> { - return allReplace ? quoteIdentifier("T1") + "." + quoteIdentifier(col) + " = " + quoteIdentifier("T2") + "." + quoteIdentifier(col) : - quoteIdentifier("T1") + "." + quoteIdentifier(col) + " =nvl(" + quoteIdentifier("T2") + "." + quoteIdentifier(col) + "," - + quoteIdentifier("T1") + "." + quoteIdentifier(col) + ")"; - }).collect(Collectors.joining(",")); + String updateConnectionSql = Arrays.stream(fieldNames). + filter(col -> !uniqueKeyList.contains(col)) + .map(col -> buildConnectionByAllReplace(allReplace, col)) + .collect(Collectors.joining(",")); + return updateConnectionSql; + } + + private String buildConnectionByAllReplace(boolean allReplace, String col) { + String conncetionSql = allReplace ? quoteIdentifier("T1") + "." + quoteIdentifier(col) + " = " + quoteIdentifier("T2") + "." + quoteIdentifier(col) : + quoteIdentifier("T1") + "." + quoteIdentifier(col) + " =nvl(" + quoteIdentifier("T2") + "." + quoteIdentifier(col) + "," + + quoteIdentifier("T1") + "." + quoteIdentifier(col) + ")"; + return conncetionSql; } @@ -96,8 +115,43 @@ private String buildConnectionConditions(String[] uniqueKeyFields) { */ public String buildDualQueryStatement(String[] column) { StringBuilder sb = new StringBuilder("SELECT "); - String collect = Arrays.stream(column).map(col -> " ? " + quoteIdentifier(col)).collect(Collectors.joining(", ")); + String collect = Arrays.stream(column) + .map(col -> wrapperPlaceholder(col) + quoteIdentifier(col)) + .collect(Collectors.joining(", ")); sb.append(collect).append(" FROM DUAL"); return sb.toString(); } + + + /** + * char type is wrapped with rpad + * @param fieldName + * @return + */ + public String wrapperPlaceholder(String fieldName) { + int pos = fieldList.indexOf(fieldName); + String type = fieldTypeList.get(pos); + + if (StringUtils.contains(type.toLowerCase(), DEAL_CHAR_KEY)) { + TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfoList.get(pos); + int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); + if (charLength > 0) { + return String.format(RPAD_FORMAT, charLength); + } + } + return SQL_DEFAULT_PLACEHOLDER; + } + + + public void setFieldList(List fieldList) { + this.fieldList = fieldList; + } + + public void setFieldTypeList(List fieldTypeList) { + this.fieldTypeList = fieldTypeList; + } + + public void setFieldExtraInfoList(List fieldExtraInfoList) { + this.fieldExtraInfoList = fieldExtraInfoList; + } } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index ee0239a1f..7a37aa6cd 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -37,6 +37,10 @@ public OracleSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { + ((OracleDialect) jdbcDialect).setFieldList(fieldList); + ((OracleDialect) jdbcDialect).setFieldTypeList(fieldTypeList); + ((OracleDialect) jdbcDialect).setFieldExtraInfoList(fieldExtraInfoList); + JDBCOptions jdbcOptions = JDBCOptions.builder() .setDBUrl(dbURL) .setDialect(jdbcDialect) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java index ddd929d2c..84eea68ff 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java @@ -20,6 +20,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -83,6 +84,10 @@ public abstract class RdbSink implements RetractStreamTableSink, Serializab protected String updateMode; + public List fieldList; + public List fieldTypeList; + public List fieldExtraInfoList; + public RdbSink(JDBCDialect jdbcDialect) { this.jdbcDialect = jdbcDialect; } @@ -106,7 +111,9 @@ public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { this.sqlTypes = JDBCTypeConvertUtils.buildSqlTypes(fieldTypeArray); this.allReplace = rdbTableInfo.isAllReplace(); this.updateMode = rdbTableInfo.getUpdateMode(); - rdbTableInfo.getFieldList(); + this.fieldList = rdbTableInfo.getFieldList(); + this.fieldTypeList = rdbTableInfo.getFieldTypeList(); + this.fieldExtraInfoList = rdbTableInfo.getFieldExtraInfoList(); return this; } From ad272dee5efa5225fc0d8e1183144396b70f1315 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 25 Mar 2020 15:07:30 +0800 Subject: [PATCH 183/523] fix sonarqube scan --- .../side/cassandra/CassandraAsyncReqRow.java | 3 +-- .../cassandra/CassandraAsyncSideInfo.java | 8 +++++-- .../sql/sink/console/ConsoleOutputFormat.java | 2 +- .../sink/console/table/TablePrintUtil.java | 2 +- .../flink/sql/exec/ExecuteProcessHelper.java | 9 ++------ .../DtNestRowDeserializationSchema.java | 3 ++- .../flink/sql/option/OptionParser.java | 21 +++++++------------ .../dtstack/flink/sql/side/BaseAllReqRow.java | 6 +++++- .../flink/sql/side/JoinNodeDealer.java | 11 +++++----- .../dtstack/flink/sql/side/SideSQLParser.java | 18 +--------------- .../dtstack/flink/sql/side/SideSqlExec.java | 10 ++------- .../com/dtstack/flink/sql/util/MathUtil.java | 2 +- .../Elasticsearch6AsyncReqRow.java | 7 ++++++- .../flink/sql/side/hbase/HbaseAllReqRow.java | 14 ++++++++++--- .../PreRowKeyModeDealerDealer.java | 3 +-- .../rowkeydealer/RowKeyEqualModeDealer.java | 3 +-- .../sink/kafka/CustomerFlinkPartition.java | 8 +++---- .../sql/launcher/ClusterClientFactory.java | 4 ++-- .../perjob/PerJobClusterClientBuilder.java | 3 ++- .../sql/launcher/perjob/PerJobSubmitter.java | 1 - .../sql/side/rdb/all/RdbAllSideInfo.java | 5 ++++- .../sql/side/rdb/async/RdbAsyncSideInfo.java | 6 +++++- .../rdb/format/JDBCUpsertOutputFormat.java | 1 - .../sink/rdb/writer/AbstractUpsertWriter.java | 3 --- 24 files changed, 71 insertions(+), 82 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index e1ed6d8e1..7eee3cb5f 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -216,7 +216,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce connCassandraDB(cassandraSideTableInfo); String sqlCondition = sideInfo.getSqlCondition() + " " + sqlWhere + " ALLOW FILTERING "; - System.out.println("sqlCondition:" + sqlCondition); + LOG.info("sqlCondition:{}" + sqlCondition); ListenableFuture resultSet = Futures.transformAsync(session, new AsyncFunction() { @@ -265,7 +265,6 @@ public void onSuccess(List rows) { public void onFailure(Throwable t) { LOG.error("Failed to retrieve the data: %s%n", t.getMessage()); - System.out.println("Failed to retrieve the data: " + t.getMessage()); cluster.closeAsync(); resultFuture.completeExceptionally(t); } diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java index 82055b94c..4bee5648b 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java @@ -30,6 +30,8 @@ import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; @@ -42,6 +44,8 @@ public class CassandraAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = -4403313049809013362L; + private static final Logger LOG = LoggerFactory.getLogger(CassandraAsyncSideInfo.class.getSimpleName()); + public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); @@ -63,9 +67,9 @@ public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInf } sqlCondition = "select ${selectField} from ${tableName}"; - sqlCondition = sqlCondition.replace("${tableName}", cassandraSideTableInfo.getDatabase()+"."+cassandraSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields); - System.out.println("---------side_exe_sql-----\n" + sqlCondition); + + LOG.info("---------side_exe_sql-----\n{}" + sqlCondition); } diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java index e99c76d03..fc5a768c4 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java @@ -56,7 +56,7 @@ public void open(int taskNumber, int numTasks) throws IOException { @Override public void writeRecord(Tuple2 tuple2) throws IOException { - System.out.println("received oriainal data:" + tuple2); + LOG.info("received oriainal data:{}" + tuple2); Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); if (!retract) { diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java index 6ddc60386..12bb2a5d4 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java @@ -95,7 +95,7 @@ public static TablePrintUtil build(List data) { try { value = obj.getClass().getMethod(colList.get(j).getMethodName).invoke(data.get(i)).toString(); } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { - e.printStackTrace(); + LOG.error("", e); } item[j] = value == null ? "null" : value; } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index c02c3b142..dbe38044e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -95,11 +95,8 @@ public class ExecuteProcessHelper { public static ParamsInfo parseParams(String[] args) throws Exception { LOG.info("------------program params-------------------------"); - System.out.println("------------program params-------------------------"); Arrays.stream(args).forEach(arg -> LOG.info("{}", arg)); - Arrays.stream(args).forEach(System.out::println); LOG.info("-------------------------------------------"); - System.out.println("----------------------------------------"); OptionParser optionParser = new OptionParser(args); Options options = optionParser.getOptions(); @@ -228,12 +225,10 @@ private static void sqlTranslation(String localSqlPluginPath, //sql-dimensional table contains the dimension table of execution sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null); } else { - System.out.println("----------exec sql without dimension join-----------"); - System.out.println("----------real sql exec is--------------------------"); - System.out.println(result.getExecSql()); + LOG.info("----------exec sql without dimension join-----------"); + LOG.info("----------real sql exec is--------------------------\n{}", result.getExecSql()); FlinkSQLExec.sqlUpdate(tableEnv, result.getExecSql(), queryConfig); if (LOG.isInfoEnabled()) { - System.out.println(); LOG.info("exec sql: " + result.getExecSql()); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 4c0d68eb2..5f1b1c6f3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -28,8 +28,9 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.*; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.NullNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.types.Row; import java.io.IOException; diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java index e49adfd93..0fd057c5a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java @@ -31,7 +31,10 @@ import java.io.File; import java.io.FileInputStream; import java.net.URLEncoder; +import java.util.stream.Stream; + import org.apache.commons.codec.Charsets; +import org.apache.flink.util.FileUtils; /** @@ -92,19 +95,16 @@ public Options getOptions(){ } public List getProgramExeArgList() throws Exception { - Map mapConf = PluginUtil.objectToMap(properties); + Map mapConf = PluginUtil.objectToMap(properties); List args = Lists.newArrayList(); - for(Map.Entry one : mapConf.entrySet()){ + for (Map.Entry one : mapConf.entrySet()) { String key = one.getKey(); Object value = one.getValue(); - if(value == null){ + if (value == null) { continue; - }else if(OPTION_SQL.equalsIgnoreCase(key)){ + } else if (OPTION_SQL.equalsIgnoreCase(key)) { File file = new File(value.toString()); - FileInputStream in = new FileInputStream(file); - byte[] filecontent = new byte[(int) file.length()]; - in.read(filecontent); - String content = new String(filecontent, Charsets.UTF_8.name()); + String content = FileUtils.readFile(file, "UTF-8"); value = URLEncoder.encode(content, Charsets.UTF_8.name()); } args.add("-" + key); @@ -112,9 +112,4 @@ public List getProgramExeArgList() throws Exception { } return args; } - - public static void main(String[] args) throws Exception { - OptionParser optionParser = new OptionParser(args); - System.out.println(optionParser.getOptions()); - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java index 674239449..0b7fe15b3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java @@ -27,6 +27,8 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.sql.SQLException; import java.util.concurrent.ScheduledExecutorService; @@ -42,6 +44,8 @@ public abstract class BaseAllReqRow extends RichFlatMapFunction implements ISideReqRow { + private static final Logger LOG = LoggerFactory.getLogger(BaseAllReqRow.class); + protected BaseSideInfo sideInfo; private ScheduledExecutorService es; @@ -59,7 +63,7 @@ public BaseAllReqRow(BaseSideInfo sideInfo){ public void open(Configuration parameters) throws Exception { super.open(parameters); initCache(); - System.out.println("----- all cacheRef init end-----"); + LOG.info("----- all cacheRef init end-----"); //start reload cache thread AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index a1c3cecb1..411de9380 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.config.CalciteConfig; import com.dtstack.flink.sql.util.TableUtils; +import com.esotericsoftware.minlog.Log; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -264,11 +265,10 @@ private void extractTemporaryQuery(SqlNode node, String tableAlias, SqlBasicCall SqlBasicCall sqlBasicCall = buildAsSqlNode(tableAlias, sqlNode); queueInfo.offer(sqlBasicCall); - System.out.println("-------build temporary query-----------"); - System.out.println(tmpSelectSql); - System.out.println("---------------------------------------"); - }catch (Exception e){ - e.printStackTrace(); + Log.info("-------build temporary query-----------\n{}", tmpSelectSql); + Log.info("---------------------------------------"); + } catch (Exception e) { + Log.error("", e); throw new RuntimeException(e); } } @@ -389,7 +389,6 @@ private void extractSelectField(SqlNode selectNode, } }else if(selectNode.getKind() == CASE){ - System.out.println("selectNode"); SqlCase sqlCase = (SqlCase) selectNode; SqlNodeList whenOperands = sqlCase.getWhenOperands(); SqlNodeList thenOperands = sqlCase.getThenOperands(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index 9be1551aa..d21effb0c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -22,46 +22,32 @@ import com.dtstack.flink.sql.config.CalciteConfig; import com.dtstack.flink.sql.util.TableUtils; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Queues; import com.google.common.collect.Sets; -import org.apache.calcite.sql.JoinType; -import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlBinaryOperator; -import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlInsert; import org.apache.calcite.sql.SqlJoin; import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlWith; import org.apache.calcite.sql.SqlWithItem; -import org.apache.calcite.sql.fun.SqlCase; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.table.api.Table; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; import java.util.Map; import java.util.Queue; import java.util.Set; -import static org.apache.calcite.sql.SqlKind.*; +import static org.apache.calcite.sql.SqlKind.IDENTIFIER; /** * Parsing sql, obtain execution information dimension table @@ -76,8 +62,6 @@ public class SideSQLParser { private Map localTableCache = Maps.newHashMap(); public Queue getExeQueue(String exeSql, Set sideTableSet) throws SqlParseException { - System.out.println("----------exec original Sql----------"); - System.out.println(exeSql); LOG.info("----------exec original Sql----------"); LOG.info(exeSql); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index e828bec03..1a7b05416 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -143,11 +143,9 @@ public void exec(String sql, Map sideTableMap, St } if(pollSqlNode.getKind() == INSERT){ - System.out.println("----------real exec sql-----------" ); - System.out.println(pollSqlNode.toString()); FlinkSQLExec.sqlUpdate(tableEnv, pollSqlNode.toString(), queryConfig); if(LOG.isInfoEnabled()){ - LOG.info("exec sql: " + pollSqlNode.toString()); + LOG.info("----------real exec sql-----------\n{}", pollSqlNode.toString()); } }else if(pollSqlNode.getKind() == AS){ @@ -178,8 +176,7 @@ public void exec(String sql, Map sideTableMap, St } }else if (pollObj instanceof JoinInfo){ - System.out.println("----------exec join info----------"); - System.out.println(pollObj.toString()); + LOG.info("----------exec join info----------\n{}", pollObj.toString()); preIsSideJoin = true; joinFun(pollObj, localTableCache, sideTableMap, tableEnv, replaceInfoList); } @@ -408,8 +405,6 @@ private void replaceFieldName(SqlNode sqlNode, FieldReplaceInfo replaceInfo) { } } }else{ - //TODO - System.out.println(sqlNode); throw new RuntimeException("---not deal type:" + sqlNode); } @@ -644,7 +639,6 @@ private SqlNode replaceSelectFieldName(SqlNode selectNode, FieldReplaceInfo repl return selectNode; }else if(selectNode.getKind() == CASE){ - System.out.println("selectNode"); SqlCase sqlCase = (SqlCase) selectNode; SqlNodeList whenOperands = sqlCase.getWhenOperands(); SqlNodeList thenOperands = sqlCase.getThenOperands(); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index 8708f86d7..966b82e3e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -219,7 +219,7 @@ public static BigDecimal getBigDecimal(Object obj) { } else if (obj instanceof BigInteger) { return new BigDecimal((BigInteger) obj); } else if (obj instanceof Number) { - return new BigDecimal(((Number) obj).doubleValue()); + return BigDecimal.valueOf(((Number) obj).doubleValue()); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to BigDecimal."); } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index e8164edb2..1f4bd1bf1 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -18,6 +18,12 @@ package com.dtstack.flink.sql.side.elasticsearch6; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.PredicateInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -26,7 +32,6 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 6f7720472..ed2931fff 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -182,9 +182,17 @@ private void loadData(Map> tmpCache) throws SQLExcep LOG.error("", e); } finally { try { - conn.close(); - table.close(); - resultScanner.close(); + if (null != conn && !conn.isClosed()) { + conn.close(); + } + + if (null != table) { + table.close(); + } + + if (null != resultScanner) { + resultScanner.close(); + } } catch (IOException e) { LOG.error("", e); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java index 3fad216b2..dcd50131d 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java @@ -110,8 +110,7 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, for (String key : colNames) { Object val = sideMap.get(key); if (val == null) { - System.out.println("can't get data with column " + key); - LOG.error("can't get data with column " + key); + LOG.error("can't get data with column {}", key); } sideVal.add(val); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index 01f43b246..41208c7f3 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -86,8 +86,7 @@ public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultF for(String key : colNames){ Object val = sideMap.get(key); if(val == null){ - System.out.println("can't get data with column " + key); - LOG.error("can't get data with column " + key); + LOG.error("can't get data with column {}", key); } sideVal.add(val); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index e212d1f57..90dfe996b 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -11,13 +11,13 @@ public class CustomerFlinkPartition extends FlinkKafkaPartitioner { public CustomerFlinkPartition() { } - + @Override public void open(int parallelInstanceId, int parallelInstances) { Preconditions.checkArgument(parallelInstanceId >= 0, "Id of this subtask cannot be negative."); Preconditions.checkArgument(parallelInstances > 0, "Number of subtasks must be larger than 0."); this.parallelInstanceId = parallelInstanceId; } - + @Override public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); if(key == null){ @@ -25,11 +25,11 @@ public int partition(T record, byte[] key, byte[] value, String targetTopic, int } return partitions[Math.abs(new String(key).hashCode()) % partitions.length]; } - + @Override public boolean equals(Object o) { return this == o || o instanceof CustomerFlinkPartition; } - + @Override public int hashCode() { return CustomerFlinkPartition.class.hashCode(); } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 7f4ded520..8f0090db5 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -21,11 +21,11 @@ import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.util.PluginUtil; +import com.esotericsoftware.minlog.Log; import org.apache.commons.io.Charsets; import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.MiniClusterClient; -import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; @@ -109,7 +109,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { applicationId = getYarnClusterApplicationId(yarnClient); } - System.out.println("applicationId=" + applicationId.toString()); + Log.info("applicationId={}", applicationId.toString()); if (StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 3de1cdbc5..0e5089bc9 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.enums.EPluginLoadMode; import com.dtstack.flink.sql.launcher.YarnConfLoader; import com.dtstack.flink.sql.option.Options; +import com.esotericsoftware.minlog.Log; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.cache.DistributedCache; import org.apache.flink.configuration.Configuration; @@ -79,7 +80,7 @@ public void init(String yarnConfDir, Configuration flinkConfig, Properties userC yarnClient.init(yarnConf); yarnClient.start(); - System.out.println("----init yarn success ----"); + Log.info("----init yarn success ----"); } public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJarPath, Options launcherOptions, JobGraph jobGraph) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index 29cc4890a..ca2d5bdd7 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -73,7 +73,6 @@ public static String submit(Options launcherOptions, JobGraph jobGraph, Configur String flinkJobId = jobGraph.getJobID().toString(); String tips = String.format("deploy per_job with appId: %s, jobId: %s", applicationId, flinkJobId); - System.out.println(tips); LOG.info(tips); return applicationId; diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 08dc56149..143017133 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -31,6 +31,8 @@ import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; @@ -47,6 +49,7 @@ public class RdbAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = -5858335638589472159L; + private static final Logger LOG = LoggerFactory.getLogger(RdbAllSideInfo.class.getSimpleName()); public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { @@ -57,7 +60,7 @@ public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); @@ -74,7 +78,7 @@ public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInf sqlCondition = getSelectFromStatement(getTableName(rdbSideTableInfo), Arrays.asList(StringUtils.split(sideSelectFields, ",")), equalFieldList, sqlJoinCompareOperate, sideTableInfo.getPredicateInfoes()); - System.out.println("----------dimension sql query-----------\n" + sqlCondition); + LOG.info("----------dimension sql query-----------\n{}", sqlCondition); } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index b01595bb9..e8b6dc8a4 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -114,7 +114,6 @@ public void open(int taskNumber, int numTasks) throws IOException { if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.APPEND.name()) || keyFields == null || keyFields.length == 0) { String insertSql = dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields); LOG.info("execute insert sql： {}", insertSql); - System.out.println("execute insert sql :" + insertSql); jdbcWriter = new AppendOnlyWriter(insertSql, fieldTypes, this); } else { jdbcWriter = AbstractUpsertWriter.create( diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java index 9ca0fd754..2c070b680 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -70,11 +70,9 @@ public static AbstractUpsertWriter create( String deleteSql = dialect.getDeleteStatement(schema, tableName, keyFields); LOG.info("deleteSQL is :{}", deleteSql); - System.out.println("deleteSQL is :" + deleteSql); Optional upsertSql = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); LOG.info("execute UpsertStatement: {}", upsertSql.orElse("use UsingInsertUpdateStatement")); - System.out.println("execute UpsertStatement: " + upsertSql.orElse("use UsingInsertUpdateStatement")); return upsertSql.map((Function) sql -> new UpsertWriterUsingUpsertStatement( @@ -171,7 +169,6 @@ public void executeUpdate(Connection connection) throws SQLException { } connection.commit(); } catch (Exception e) { - System.out.println(e.getCause()); // deal pg error: current transaction is aborted, commands ignored until end of transaction block connection.rollback(); connection.commit(); From 7bcb8ac449c61656cb3eb2542c5d7a23a2a1fce6 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 25 Mar 2020 15:25:09 +0800 Subject: [PATCH 184/523] remove unuse class --- .../flink/sql/side/redis/RedisAllReqRow.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index f484c12b3..5d09c8f3d 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -36,21 +36,27 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.*; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.JedisCommands; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisSentinelPool; import java.io.Closeable; import java.io.IOException; import java.sql.SQLException; -import java.util.*; + +import java.util.Calendar; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; /** * @author yanxi */ From 2833176a6402c911c57e93ad81c10320fba45360 Mon Sep 17 00:00:00 2001 From: xuchao Date: Wed, 25 Mar 2020 19:07:46 +0800 Subject: [PATCH 185/523] =?UTF-8?q?=E9=87=8D=E6=9E=84=20sql=E8=AF=AD?= =?UTF-8?q?=E5=8F=A5=E8=A7=A3=E6=9E=90=E9=80=BB=E8=BE=91=EF=BC=9B=20?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E6=9B=BF=E6=8D=A2=E5=92=8C=E9=87=8D=E5=91=BD?= =?UTF-8?q?=E5=90=8D=E7=9B=B8=E5=85=B3=E7=9A=84=E9=80=BB=E8=BE=91=E5=9C=A8?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E9=98=B6=E6=AE=B5=E5=AE=8C=E6=88=90=EF=BC=9B?= =?UTF-8?q?=20=E7=9B=AE=E6=A0=87=EF=BC=9A=E7=BB=8F=E8=BF=87=E8=A7=A3?= =?UTF-8?q?=E6=9E=90sql=E4=B9=8B=E5=90=8E=E7=9A=84=E6=89=80=E6=9C=89sql?= =?UTF-8?q?=E6=98=AF=E7=A1=AE=E5=AE=9A=E7=9A=84=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/parser/InsertSqlParser.java | 1 + .../flink/sql/side/JoinNodeDealer.java | 4 +-- .../com/dtstack/flink/sql/side/SideInfo.java | 36 ++++++++++++++++--- .../dtstack/flink/sql/side/SideSqlExec.java | 1 - .../dtstack/flink/sql/util/TableUtils.java | 10 +++++- .../sql/side/rdb/async/RdbAsyncReqRow.java | 4 +-- 6 files changed, 45 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java index a7c6db9eb..e2940c4f7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java @@ -57,6 +57,7 @@ public void parseSql(String sql, SqlTree sqlTree) { .configBuilder() .setLex(Lex.MYSQL) .build(); + SqlParser sqlParser = SqlParser.create(sql,config); SqlNode sqlNode = null; try { diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index 17bc91b40..d321130cc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -731,9 +731,9 @@ private boolean checkAndRemoveWhereCondition(Set fromTableNameSet, Set conditionRefTableNameSet = Sets.newHashSet(); fieldInfos.forEach(fieldInfo -> { - String[] splitInfo = StringUtils.split(fieldInfo, ","); + String[] splitInfo = StringUtils.split(fieldInfo, "."); if(splitInfo.length == 2){ - conditionRefTableNameSet.add(splitInfo[1]); + conditionRefTableNameSet.add(splitInfo[0]); } }); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java index df41e1663..029c86e25 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java @@ -55,6 +55,8 @@ public abstract class SideInfo implements Serializable{ protected String sideSelectFields = ""; + protected Map sideSelectFieldsType = Maps.newHashMap(); + protected JoinType joinType; //key:Returns the value of the position, value: the ref field index​in the input table @@ -84,15 +86,17 @@ public void parseSelectFields(JoinInfo joinInfo){ String sideTableName = joinInfo.getSideTableName(); String nonSideTableName = joinInfo.getNonSideTable(); List fields = Lists.newArrayList(); + int sideTableFieldIndex = 0; - int sideIndex = 0; for( int i=0; i getSideFieldNameIndex() { public void setSideFieldNameIndex(Map sideFieldNameIndex) { this.sideFieldNameIndex = sideFieldNameIndex; } + + public Map getSideSelectFieldsType() { + return sideSelectFieldsType; + } + + public void setSideSelectFieldsType(Map sideSelectFieldsType) { + this.sideSelectFieldsType = sideSelectFieldsType; + } + + public String getSelectSideFieldType(int index){ + return sideSelectFieldsType.get(index); + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 920dec07c..906149a96 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -394,7 +394,6 @@ private void joinFun(Object pollObj, HashBasedTable mappingTable = ((JoinInfo) pollObj).getTableFieldRef(); //获取两个表的所有字段 - //TODO 抽取 List sideJoinFieldInfo = ParserJoinField.getRowTypeInfo(joinInfo.getSelectNode(), joinScope, true); //通过join的查询字段信息过滤出需要的字段信息 sideJoinFieldInfo.removeIf(tmpFieldInfo -> mappingTable.get(tmpFieldInfo.getTable(), tmpFieldInfo.getFieldName()) == null); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 39d79c065..e62f3fcfa 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -26,6 +26,7 @@ import com.google.common.collect.HashBasedTable; import com.google.common.collect.HashBiMap; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.typesafe.config.ConfigException; import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.SqlBasicCall; @@ -200,7 +201,14 @@ public static SqlBasicCall buildAsNodeByJoinInfo(JoinInfo joinInfo, SqlNode sqlN String joinLeftTableAlias = joinInfo.getLeftTableAlias(); joinLeftTableName = Strings.isNullOrEmpty(joinLeftTableName) ? joinLeftTableAlias : joinLeftTableName; String newTableName = buildInternalTableName(joinLeftTableName, SPLIT, joinInfo.getRightTableName()); - String newTableAlias = !StringUtils.isEmpty(tableAlias) ? tableAlias : buildInternalTableName(joinInfo.getLeftTableAlias(), SPLIT, joinInfo.getRightTableAlias()); + String lefTbAlias = joinInfo.getLeftTableAlias(); + if(Strings.isNullOrEmpty(lefTbAlias)){ + Set fromTableSet = Sets.newHashSet(); + TableUtils.getFromTableInfo(joinInfo.getLeftNode(), fromTableSet); + lefTbAlias = StringUtils.join(fromTableSet, "_"); + } + + String newTableAlias = !StringUtils.isEmpty(tableAlias) ? tableAlias : buildInternalTableName(lefTbAlias, SPLIT, joinInfo.getRightTableAlias()); if (null == sqlNode0) { sqlNode0 = new SqlIdentifier(newTableName, null, sqlParserPos); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index a48e5eb0f..dd89d1671 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -201,7 +201,6 @@ protected List getRows(Row inputRow, List cacheContent, List entry : sideInfo.getInFieldIndex().entrySet()) { Object obj = input.getField(entry.getValue()); boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); @@ -216,7 +215,8 @@ public Row fillData(Row input, Object line) { if (jsonArray == null) { row.setField(entry.getKey(), null); } else { - Object object = SwitchUtil.getTarget(jsonArray.getValue(entry.getValue()), fields[entry.getValue()]); + String fieldType = sideInfo.getSelectSideFieldType(entry.getValue()); + Object object = SwitchUtil.getTarget(jsonArray.getValue(entry.getValue()), fieldType); row.setField(entry.getKey(), object); } } From e433093a9d989db6da7a4e5e27ff0f0d16ee0587 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 25 Mar 2020 19:17:46 +0800 Subject: [PATCH 186/523] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E5=B8=B8=E7=94=A8?= =?UTF-8?q?=E4=BB=93=E5=BA=93=E9=87=8C=E9=9D=A2=E4=B8=8D=E5=AD=98=E5=9C=A8?= =?UTF-8?q?=E7=9A=84=E4=BE=9D=E8=B5=96=E5=8C=85:http://redmine.prod.dtstac?= =?UTF-8?q?k.cn/issues/23710?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- launcher/pom.xml | 5 ----- .../com/dtstack/flink/sql/launcher/LauncherMain.java | 11 ----------- pom.xml | 1 - 3 files changed, 17 deletions(-) diff --git a/launcher/pom.xml b/launcher/pom.xml index 8c1ce53d5..1771df1ec 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -34,11 +34,6 @@ fastjson 1.2.7 - - com.aiweiergou - tools-logger - ${logger.tool.version} - diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 95afc86cd..c098bc27e 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -20,7 +20,6 @@ package com.dtstack.flink.sql.launcher; -import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; import com.dtstack.flink.sql.constrant.ConfigConstrant; import com.google.common.collect.Lists; import com.alibaba.fastjson.JSON; @@ -84,8 +83,6 @@ public static void main(String[] args) throws Exception { confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - setLogLevel(confProperties.getProperty(ConfigConstrant.LOG_LEVEL_KEY)); - if(mode.equals(ClusterMode.local.name())) { String[] localArgs = argList.toArray(new String[argList.size()]); Main.main(localArgs); @@ -149,12 +146,4 @@ private static String[] parseJson(String[] args) { String[] array = list.toArray(new String[list.size()]); return array; } - - private static void setLogLevel(String logLevel){ - if(org.apache.commons.lang3.StringUtils.isBlank(logLevel)){ - return; - } - ChangeLogLevelProcess logLevelProcess = new ChangeLogLevelProcess(); - logLevelProcess.process(logLevel); - } } diff --git a/pom.xml b/pom.xml index a48b7e1d1..2794f9004 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,6 @@ UTF-8 1.8.1 - 1.0.0-SNAPSHOT From 2353b7fdcd52f1f96ccb9ae06480954222285bfb Mon Sep 17 00:00:00 2001 From: xuchao Date: Thu, 26 Mar 2020 10:41:47 +0800 Subject: [PATCH 187/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0sonar=20=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E8=B4=A8=E9=87=8F=E6=A3=80=E6=9F=A5=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 10 ++++++++++ ci/sonar_notify.sh | 14 ++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 ci/sonar_notify.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..029098c87 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,10 @@ +build: + stage: test + script: + - mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q + - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar + - sh ci/sonar_notify.sh + only: + - 1.8_dev + tags: + - dt-insight-engine \ No newline at end of file diff --git a/ci/sonar_notify.sh b/ci/sonar_notify.sh new file mode 100644 index 000000000..41f8a3c0e --- /dev/null +++ b/ci/sonar_notify.sh @@ -0,0 +1,14 @@ +#!/bin/bash +#参考钉钉文档 https://open-doc.dingtalk.com/microapp/serverapi2/qf2nxq + sonarreport=$(curl -s http://172.16.100.198:8082/?projectname=dt-insight-engine/flinkStreamSQL) + curl -s "https://oapi.dingtalk.com/robot/send?access_token=71555061297a53d3ac922a6f4d94285d8e23bccdca0c00b4dc6df0a2d49da724" \ + -H "Content-Type: application/json" \ + -d "{ + \"msgtype\": \"markdown\", + \"markdown\": { + \"title\":\"sonar代码质量\", + \"text\": \"## sonar代码质量报告: \n +> [sonar地址](http://172.16.100.198:9000/dashboard?id=dt-insight-engine/flinkStreamSQL) \n +> ${sonarreport} \n\" + } + }" \ No newline at end of file From b169a349e4ab14461731c64104932005f7f52fea Mon Sep 17 00:00:00 2001 From: xuchao Date: Thu, 26 Mar 2020 10:52:33 +0800 Subject: [PATCH 188/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index 462eed30b..a8d69597d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -37,7 +37,7 @@ /** * - * Reason: TODO ADD REASON(可选) + * 日期工具 * Date: 2017年03月10日 下午1:16:37 * Company: www.dtstack.com * @author sishu.yss From 45013fab4abb379a6043a7bc02a877b966c33b8b Mon Sep 17 00:00:00 2001 From: xuchao Date: Thu, 26 Mar 2020 11:01:23 +0800 Subject: [PATCH 189/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9sonar=20=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 029098c87..39e0d4746 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,6 @@ build: - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar - sh ci/sonar_notify.sh only: - - 1.8_dev + - v1.8.0_dev tags: - dt-insight-engine \ No newline at end of file From adcf84727685e82b83aaa41e1d8a660a3b27d667 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 26 Mar 2020 16:28:40 +0800 Subject: [PATCH 190/523] oceanbase dimension --- .../flink/sql/exec/ExecuteProcessHelper.java | 6 +- .../dtstack/flink/sql/side/SideSqlExec.java | 2 +- .../sql/watermarker/WaterMarkerAssigner.java | 6 +- .../flink/sql/launcher/LauncherMain.java | 19 ++-- .../oceanbase-side/oceanbase-all-side/pom.xml | 94 +++++++++++++++++++ .../side/oceanbase/OceanbaseAllReqRow.java | 71 ++++++++++++++ .../side/oceanbase/OceanbaseAllSideInfo.java | 40 ++++++++ .../oceanbase-async-side/pom.xml | 92 ++++++++++++++++++ .../side/oceanbase/OceanbaseAsyncReqRow.java | 73 ++++++++++++++ .../oceanbase/OceanbaseAsyncSideInfo.java | 40 ++++++++ .../oceanbase-side-core/pom.xml | 18 ++++ .../oceanbase/table/OceanbaseSideParser.java | 39 ++++++++ oceanbase/oceanbase-side/pom.xml | 34 +++++++ .../sql/sink/ocean/OceanbaseDialect.java | 17 ++++ .../flink/sql/sink/ocean/OceanbaseSink.java | 19 +++- .../sink/ocean/table/OceanbaseSinkParser.java | 17 ++++ oceanbase/pom.xml | 1 + .../side/rdb/all/AbstractRdbAllReqRow.java | 11 +-- .../sql/side/rdb/all/RdbAllSideInfo.java | 3 +- 19 files changed, 570 insertions(+), 32 deletions(-) create mode 100644 oceanbase/oceanbase-side/oceanbase-all-side/pom.xml create mode 100644 oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllReqRow.java create mode 100644 oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllSideInfo.java create mode 100644 oceanbase/oceanbase-side/oceanbase-async-side/pom.xml create mode 100644 oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncReqRow.java create mode 100644 oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncSideInfo.java create mode 100644 oceanbase/oceanbase-side/oceanbase-side-core/pom.xml create mode 100644 oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java create mode 100644 oceanbase/oceanbase-side/pom.xml diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index c02c3b142..01e0f9d6e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -94,11 +94,9 @@ public class ExecuteProcessHelper { public static ParamsInfo parseParams(String[] args) throws Exception { - LOG.info("------------program params-------------------------"); System.out.println("------------program params-------------------------"); Arrays.stream(args).forEach(arg -> LOG.info("{}", arg)); Arrays.stream(args).forEach(System.out::println); - LOG.info("-------------------------------------------"); System.out.println("----------------------------------------"); OptionParser optionParser = new OptionParser(args); @@ -288,9 +286,7 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment RowTypeInfo typeInfo = new RowTypeInfo(adaptTable.getSchema().getFieldTypes(), adaptTable.getSchema().getFieldNames()); DataStream adaptStream = tableEnv.toRetractStream(adaptTable, typeInfo) - .map((Tuple2 f0) -> { - return f0.f1; - }) + .map((Tuple2 f0) -> f0.f1) .returns(typeInfo); String fields = String.join(",", typeInfo.getFieldNames()); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index e828bec03..4a90e7fb1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -121,7 +121,7 @@ public void exec(String sql, Map sideTableMap, St SideSQLParser sideSQLParser = new SideSQLParser(); sideSQLParser.setLocalTableCache(localTableCache); Queue exeQueue = sideSQLParser.getExeQueue(sql, sideTableMap.keySet()); - Object pollObj = null; + Object pollObj; //need clean boolean preIsSideJoin = false; diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java index b0ca0335f..784aa0b29 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java @@ -41,11 +41,7 @@ public class WaterMarkerAssigner { public boolean checkNeedAssignWaterMarker(AbstractSourceTableInfo tableInfo){ - if(Strings.isNullOrEmpty(tableInfo.getEventTimeField())){ - return false; - } - - return true; + return !Strings.isNullOrEmpty(tableInfo.getEventTimeField()); } public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo typeInfo, AbstractSourceTableInfo sourceTableInfo){ diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 0b036c87c..0943eed79 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -48,6 +48,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -67,8 +68,7 @@ public class LauncherMain { private static String getLocalCoreJarPath(String localSqlRootJar) throws Exception { String jarPath = PluginUtil.getCoreJarFileName(localSqlRootJar, CORE_JAR); - String corePath = localSqlRootJar + SP + jarPath; - return corePath; + return localSqlRootJar + SP + jarPath; } public static void main(String[] args) throws Exception { @@ -85,14 +85,14 @@ public static void main(String[] args) throws Exception { Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); if(mode.equals(ClusterMode.local.name())) { - String[] localArgs = argList.toArray(new String[argList.size()]); + String[] localArgs = argList.toArray(new String[0]); Main.main(localArgs); return; } String pluginRoot = launcherOptions.getLocalSqlPluginPath(); File jarFile = new File(getLocalCoreJarPath(pluginRoot)); - String[] remoteArgs = argList.toArray(new String[argList.size()]); + String[] remoteArgs = argList.toArray(new String[0]); PackagedProgram program = new PackagedProgram(jarFile, Lists.newArrayList(), remoteArgs); String savePointPath = confProperties.getProperty(ConfigConstrant.SAVE_POINT_PATH_KEY); @@ -116,14 +116,14 @@ public static void main(String[] args) throws Exception { private static String[] parseJson(String[] args) { BufferedReader reader = null; - String lastStr = ""; + StringBuilder lastStr = new StringBuilder(); try{ FileInputStream fileInputStream = new FileInputStream(args[0]); - InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, "UTF-8"); + InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, StandardCharsets.UTF_8); reader = new BufferedReader(inputStreamReader); String tempString = null; while((tempString = reader.readLine()) != null){ - lastStr += tempString; + lastStr.append(tempString); } reader.close(); }catch(IOException e){ @@ -137,14 +137,13 @@ private static String[] parseJson(String[] args) { } } } - Map map = JSON.parseObject(lastStr, new TypeReference>(){} ); + Map map = JSON.parseObject(lastStr.toString(), new TypeReference>(){} ); List list = new LinkedList<>(); for (Map.Entry entry : map.entrySet()) { list.add("-" + entry.getKey()); list.add(entry.getValue().toString()); } - String[] array = list.toArray(new String[list.size()]); - return array; + return list.toArray(new String[0]); } } diff --git a/oceanbase/oceanbase-side/oceanbase-all-side/pom.xml b/oceanbase/oceanbase-side/oceanbase-all-side/pom.xml new file mode 100644 index 000000000..544476129 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-all-side/pom.xml @@ -0,0 +1,94 @@ + + + + sql.side.oceanbase + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.side.all.oceanbase + oceanbase-all-side + jar + 1.0-SNAPSHOT + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.side.oceanbase.core + ${sql.side.oceanbase.core.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + false + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllReqRow.java b/oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllReqRow.java new file mode 100644 index 000000000..30317a302 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllReqRow.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstatck.flink.sql.side.oceanbase; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.google.common.collect.Maps; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.List; +import java.util.Map; + +/** + * @author : tiezhu + * @date : 2020/3/26 + */ +public class OceanbaseAllReqRow extends AbstractRdbAllReqRow { + + private static final Logger LOG = LoggerFactory.getLogger(OceanbaseAllReqRow.class); + + private static final String OCEAN_BASE_DRIVER = "com.mysql.jdbc.Driver"; + + public OceanbaseAllReqRow(RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, + List outFieldInfoList, + AbstractSideTableInfo sideTableInfo) { + super(new OceanbaseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + } + + @Override + public Connection getConn(String dbUrl, String userName, String password) { + try { + Class.forName(OCEAN_BASE_DRIVER); + Map addParams = Maps.newHashMap(); + addParams.put("useCursorFetch", "true"); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); + return DriverManager.getConnection(targetDbUrl, userName, password); + } catch (Exception e) { + LOG.error("oceanbase get connect error", e); + throw new RuntimeException(e); + } + } + + @Override + public int getFetchSize() { + return Integer.MIN_VALUE; + } +} diff --git a/oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllSideInfo.java b/oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllSideInfo.java new file mode 100644 index 000000000..a983a82fd --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllSideInfo.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstatck.flink.sql.side.oceanbase; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; + +import java.util.List; + +/** + * @author : tiezhu + * @date : 2020/3/26 + */ +public class OceanbaseAllSideInfo extends RdbAllSideInfo { + public OceanbaseAllSideInfo(RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, + List outFieldInfoList, + AbstractSideTableInfo sideTableInfo) { + super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + } +} diff --git a/oceanbase/oceanbase-side/oceanbase-async-side/pom.xml b/oceanbase/oceanbase-side/oceanbase-async-side/pom.xml new file mode 100644 index 000000000..dd5158ce3 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-async-side/pom.xml @@ -0,0 +1,92 @@ + + + + sql.side.oceanbase + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.side.async.oceanbase + oceanbase-async-side + jar + 1.0-SNAPSHOT + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.side.oceanbase.core + ${sql.side.oceanbase.core.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + false + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncReqRow.java b/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncReqRow.java new file mode 100644 index 000000000..fd76b56b9 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncReqRow.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.flink.sql.side.oceanbase; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import io.vertx.core.Vertx; +import io.vertx.core.VertxOptions; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.jdbc.JDBCClient; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; + +import java.util.List; + +/** + * @author : tiezhu + * @date : 2020/3/26 + */ +public class OceanbaseAsyncReqRow extends RdbAsyncReqRow { + + private static final String OCEAN_BASE_DRIVER = "com.mysql.jdbc.Driver"; + + public OceanbaseAsyncReqRow(RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, + List outFieldInfoList, + AbstractSideTableInfo sideTableInfo) { + super(new OceanbaseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + } + + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + JsonObject oceanbaseClientConfig = new JsonObject(); + RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + oceanbaseClientConfig.put("url", rdbSideTableInfo.getUrl()) + .put("driver_class", OCEAN_BASE_DRIVER) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("user", rdbSideTableInfo.getUserName()) + .put("password", rdbSideTableInfo.getPassword()) + .put("provider_class", DT_PROVIDER_CLASS) + .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) + .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) + .put("test_conncetion_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); + + System.setProperty("vertx.disableFileCPResolving", "true"); + + VertxOptions vo = new VertxOptions(); + vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setFileResolverCachingEnabled(false); + Vertx vertx = Vertx.vertx(vo); + setRdbSqlClient(JDBCClient.createNonShared(vertx, oceanbaseClientConfig)); + } +} diff --git a/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncSideInfo.java b/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncSideInfo.java new file mode 100644 index 000000000..c3e79a573 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncSideInfo.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.flink.sql.side.oceanbase; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; + +import java.util.List; + +/** + * @author : tiezhu + * @date : 2020/3/26 + */ +public class OceanbaseAsyncSideInfo extends RdbAsyncSideInfo { + + public OceanbaseAsyncSideInfo(RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, + List outfieldsInfoList, + AbstractSideTableInfo sideTableInfo){ + super(rowTypeInfo, joinInfo, outfieldsInfoList, sideTableInfo); + } +} diff --git a/oceanbase/oceanbase-side/oceanbase-side-core/pom.xml b/oceanbase/oceanbase-side/oceanbase-side-core/pom.xml new file mode 100644 index 000000000..397b62063 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-side-core/pom.xml @@ -0,0 +1,18 @@ + + + + sql.side.oceanbase + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.side.oceanbase.core + oceanbase-side-core + 1.0-SNAPSHOT + jar + + \ No newline at end of file diff --git a/oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java b/oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java new file mode 100644 index 000000000..a5c6324a9 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.flink.sql.side.oceanbase.table; + +import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; + +import java.util.Map; + +/** + * @author : tiezhu + * @date : 2020/3/26 + */ +public class OceanbaseSideParser extends RdbSideParser { + + private static final String CURRENT_TYPE = "oceanbase"; + + @Override + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo oceanbaseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + oceanbaseTableInfo.setType(CURRENT_TYPE); + return oceanbaseTableInfo; + } +} diff --git a/oceanbase/oceanbase-side/pom.xml b/oceanbase/oceanbase-side/pom.xml new file mode 100644 index 000000000..2f25c3d24 --- /dev/null +++ b/oceanbase/oceanbase-side/pom.xml @@ -0,0 +1,34 @@ + + + + sql.oceanbase + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.side.oceanbase + 1.0-SNAPSHOT + + oceanbase-all-side + oceanbase-async-side + oceanbase-side-core + + oceanbase-side + pom + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.side.rdb + ${rdb.side.version} + + + + \ No newline at end of file diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java index b10f4a3b3..6a157ea23 100644 --- a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.sink.ocean; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java index 8aede349b..537a2b3ea 100644 --- a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.sink.ocean; import com.dtstack.flink.sql.sink.IStreamSinkGener; @@ -11,8 +28,6 @@ */ public class OceanbaseSink extends AbstractRdbSink implements IStreamSinkGener { - private static final String OCEANBESE_DRIVER = "com.mysql.jdbc.Driver"; - public OceanbaseSink() { super(new OceanbaseDialect()); } diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java index f9f3fb42a..29f2e67da 100644 --- a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.sink.ocean.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; diff --git a/oceanbase/pom.xml b/oceanbase/pom.xml index 231665282..8b7c8bc36 100644 --- a/oceanbase/pom.xml +++ b/oceanbase/pom.xml @@ -13,6 +13,7 @@ pom oceanbase-sink + oceanbase-side diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 2e9986742..5a1158125 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -43,10 +43,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -107,7 +104,7 @@ public void flatMap(CRow value, Collector out) throws Exception { List equalValIndex = sideInfo.getEqualValIndex(); ArrayList inputParams = equalValIndex.stream() .map(value.row()::getField) - .filter(object -> null != object) + .filter(Objects::nonNull) .collect(Collectors.toCollection(ArrayList::new)); if (inputParams.size() != equalValIndex.size() && sideInfo.getJoinType() == JoinType.LEFT) { @@ -124,7 +121,7 @@ public void flatMap(CRow value, Collector out) throws Exception { out.collect(new CRow(fillData(value.row(), null), value.change())); } - cacheList.stream().forEach(one -> out.collect(new CRow(fillData(value.row(), one), value.change()))); + cacheList.forEach(one -> out.collect(new CRow(fillData(value.row(), one), value.change()))); } @Override @@ -218,7 +215,7 @@ private void queryAndFillData(Map>> tmpCache, C } String cacheKey = sideInfo.getEqualFieldList().stream() - .map(equalField -> oneRow.get(equalField)) + .map(oneRow::get) .map(Object::toString) .collect(Collectors.joining("_")); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 08dc56149..7140849e2 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -68,8 +68,7 @@ private String getSelectFromStatement(String tableName, List selectField String fromClause = selectFields.stream().map(this::quoteIdentifier).collect(Collectors.joining(", ")); String predicateClause = predicateInfoes.stream().map(this::buildFilterCondition).collect(Collectors.joining(" AND ")); String whereClause = buildWhereClause(predicateClause); - String sql = "SELECT " + fromClause + " FROM " + tableName + whereClause; - return sql; + return "SELECT " + fromClause + " FROM " + tableName + whereClause; } private String buildWhereClause(String predicateClause) { From 2d9a65efe53461e9c772284bd79c414565891114 Mon Sep 17 00:00:00 2001 From: xuchao Date: Fri, 27 Mar 2020 13:55:04 +0800 Subject: [PATCH 191/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9sonar=20=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=EF=BC=8C=E6=B7=BB=E5=8A=A0login-token?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 39e0d4746..242b309a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ build: stage: test script: - mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q - - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar + - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.login= 11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar - sh ci/sonar_notify.sh only: - v1.8.0_dev From 0ae4bb331c6d081dd8ced9c8f77fbfa89223f98b Mon Sep 17 00:00:00 2001 From: xuchao Date: Fri, 27 Mar 2020 15:04:06 +0800 Subject: [PATCH 192/523] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=B1=BB=E5=90=8D=E7=A7=B0=E9=94=99=E8=AF=AFpackage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test/java/com/dtstack/flinkx/AppTest.java | 58 ------------------- .../test/java/com/dtstack/flinkx/AppTest.java | 58 ------------------- .../test/java/com/dtstack/flinkx/AppTest.java | 58 ------------------- 3 files changed, 174 deletions(-) delete mode 100644 cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java delete mode 100644 mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java delete mode 100644 mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java diff --git a/cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java deleted file mode 100644 index 33a0233ac..000000000 --- a/cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.dtstack.flinkx; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -/** - * Unit test for simple App. - */ -public class AppTest - extends TestCase -{ - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest( String testName ) - { - super( testName ); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() - { - return new TestSuite( AppTest.class ); - } - - /** - * Rigourous Test :-) - */ - public void testApp() - { - assertTrue( true ); - } -} diff --git a/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java deleted file mode 100644 index 33a0233ac..000000000 --- a/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.dtstack.flinkx; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -/** - * Unit test for simple App. - */ -public class AppTest - extends TestCase -{ - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest( String testName ) - { - super( testName ); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() - { - return new TestSuite( AppTest.class ); - } - - /** - * Rigourous Test :-) - */ - public void testApp() - { - assertTrue( true ); - } -} diff --git a/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java deleted file mode 100644 index 33a0233ac..000000000 --- a/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.dtstack.flinkx; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -/** - * Unit test for simple App. - */ -public class AppTest - extends TestCase -{ - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest( String testName ) - { - super( testName ); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() - { - return new TestSuite( AppTest.class ); - } - - /** - * Rigourous Test :-) - */ - public void testApp() - { - assertTrue( true ); - } -} From eafed59c1ab647d636d0d4113a12211cdc98aa55 Mon Sep 17 00:00:00 2001 From: xuchao Date: Fri, 27 Mar 2020 17:00:21 +0800 Subject: [PATCH 193/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9sonar=20login=20?= =?UTF-8?q?=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 242b309a9..981a47148 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ build: stage: test script: - mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q - - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.login= 11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar + - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar - sh ci/sonar_notify.sh only: - v1.8.0_dev From 59b996b53d3b164565cbdd683a343c8915c1e52c Mon Sep 17 00:00:00 2001 From: xuchao Date: Fri, 27 Mar 2020 17:35:53 +0800 Subject: [PATCH 194/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9sonar=20=E5=88=86?= =?UTF-8?q?=E6=94=AF=20=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 981a47148..2952551ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ build: stage: test script: - mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q - - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar + - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.branch.name="v1.8.0_dev" -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar - sh ci/sonar_notify.sh only: - v1.8.0_dev From c895ba2d0961d5ec07e632bc8f2de6fe06c3fcab Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 30 Mar 2020 09:56:58 +0800 Subject: [PATCH 195/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dkafka-source=20?= =?UTF-8?q?=E4=BD=9C=E4=B8=BA=E6=95=B0=E6=8D=AE=E6=BA=90=E9=94=99=E8=AF=AF?= =?UTF-8?q?=EF=BC=8Ctype=3D'kakfa'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../source/kafka/KafkaDeserializationMetricWrapper.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java index afa950c5b..d0d76747d 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java @@ -76,8 +76,13 @@ protected void beforeDeserialize() throws IOException { } protected void registerPtMetric(AbstractFetcher fetcher) throws Exception { + Field consumerThreadField = null; + if(fetcher.getClass().getDeclaredField("consumerThread") != null){ + consumerThreadField = fetcher.getClass().getDeclaredField("consumerThread"); + } else { + consumerThreadField = fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); + } - Field consumerThreadField = fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); consumerThreadField.setAccessible(true); KafkaConsumerThread consumerThread = (KafkaConsumerThread) consumerThreadField.get(fetcher); From a8947ec9f4740494432542ec0ac8c947de006ef4 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 30 Mar 2020 11:46:44 +0800 Subject: [PATCH 196/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=98=BB=E5=A1=9E?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/rdb/async/RdbAsyncReqRow.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 011920afe..da6cb3876 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -43,6 +43,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; @@ -116,9 +117,8 @@ private void connectWithRetry(Map inputParams, CRow input, Resul AtomicInteger failCounter = new AtomicInteger(0); AtomicBoolean finishFlag = new AtomicBoolean(false); while(!finishFlag.get()){ - AtomicBoolean connFinish = new AtomicBoolean(false); + CountDownLatch latch = new CountDownLatch(1); rdbSqlClient.getConnection(conn -> { - connFinish.set(true); if(conn.failed()){ if(failCounter.getAndIncrement() % 1000 == 0){ logger.error("getConnection error", conn.cause()); @@ -127,22 +127,27 @@ private void connectWithRetry(Map inputParams, CRow input, Resul resultFuture.completeExceptionally(conn.cause()); finishFlag.set(true); } + latch.countDown(); conn.result().close(); return; } - CONN_STATUS.set(true); - ScheduledFuture timerFuture = registerTimer(input, resultFuture); - cancelTimerWhenComplete(resultFuture, timerFuture); - handleQuery(conn.result(), inputParams, input, resultFuture); - finishFlag.set(true); - }); - while(!connFinish.get()){ try { - Thread.sleep(50); - } catch (InterruptedException e){ + CONN_STATUS.set(true); + ScheduledFuture timerFuture = registerTimer(input, resultFuture); + cancelTimerWhenComplete(resultFuture, timerFuture); + handleQuery(conn.result(), inputParams, input, resultFuture); + finishFlag.set(true); + } catch (Exception e) { logger.error("", e); + } finally { + latch.countDown(); } - + }); + //主线程阻塞 + try { + latch.wait(); + } catch (InterruptedException e) { + logger.error("", e); } } From b5c6b4a17282b4f20f9c941114b29ffd45807d5d Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 30 Mar 2020 12:02:30 +0800 Subject: [PATCH 197/523] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E7=9A=84=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/kafka/AbstractKafkaProducerFactory.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java index 9dcaf222b..9958a2544 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java @@ -26,12 +26,9 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.formats.avro.AvroRowSerializationSchema; -import org.apache.flink.formats.csv.CsvRowSerializationSchema; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; From ba3680f3ef4142826d7314b958689ddb5417f2cf Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 30 Mar 2020 13:40:52 +0800 Subject: [PATCH 198/523] csv sink default delimiter --- docs/kafkaSink.md | 2 +- .../com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/kafkaSink.md b/docs/kafkaSink.md index 3c6eb1dc6..5d7c7b2a7 100644 --- a/docs/kafkaSink.md +++ b/docs/kafkaSink.md @@ -41,7 +41,7 @@ CREATE TABLE tableName( |partitionKeys | 用来分区的字段|否|| |updateMode | 回溯流数据下发模式，append,upsert.upsert模式下会将是否为回溯信息以字段形式进行下发。|否|append| |sinkdatatype | 写入kafka数据格式，json,avro,csv|否|json| -|fieldDelimiter | csv数据分隔符|否| \ | +|fieldDelimiter | csv数据分隔符|否| , | **kafka相关参数可以自定义，使用kafka.开头即可。** diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index 6740ea867..4ad8947a8 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -49,7 +49,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Mon, 30 Mar 2020 14:44:55 +0800 Subject: [PATCH 199/523] =?UTF-8?q?hbase=20=E7=BB=93=E6=9E=9C=E8=A1=A8?= =?UTF-8?q?=E6=94=AF=E6=8C=81=20rowKey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sink/hbase/AbstractReplaceOperator.java | 58 +++++++ .../sql/sink/hbase/HbaseOutputFormat.java | 37 ++--- .../sql/sink/hbase/Md5ReplaceOperator.java | 43 +++++ .../flink/sql/sink/hbase/ReplaceInfo.java | 74 +++++++++ .../flink/sql/sink/hbase/RowKeyBuilder.java | 148 ++++++++++++++++++ .../sql/sink/hbase/enums/EReplaceOpType.java | 39 +++++ .../sql/sink/hbase/enums/EReplaceType.java | 43 +++++ 7 files changed, 419 insertions(+), 23 deletions(-) create mode 100644 hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/AbstractReplaceOperator.java create mode 100644 hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/Md5ReplaceOperator.java create mode 100644 hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/ReplaceInfo.java create mode 100644 hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java create mode 100644 hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceOpType.java create mode 100644 hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceType.java diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/AbstractReplaceOperator.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/AbstractReplaceOperator.java new file mode 100644 index 000000000..c3e5701aa --- /dev/null +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/AbstractReplaceOperator.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flink.sql.sink.hbase; + + +import com.dtstack.flink.sql.sink.hbase.enums.EReplaceOpType; + +import java.util.Map; + +/** + * Reason: + * Date: 2018/8/23 + * Company: www.dtstack.com + * @author xuchao + */ + +public abstract class AbstractReplaceOperator { + + private EReplaceOpType opType; + + public AbstractReplaceOperator(EReplaceOpType opType){ + this.opType = opType; + } + + public String doOperator(Map refData){ + String replaceStr = replaceStr(refData); + return doFunc(replaceStr); + } + + public String replaceStr(Map refData){ + return ""; + } + + /** + * The processing function to provide custom + * @param replaceStr + * @return + */ + abstract String doFunc(String replaceStr); +} diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 382e8f975..1769fbda7 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -23,6 +23,7 @@ import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; @@ -40,7 +41,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.text.SimpleDateFormat; import java.util.List; import java.util.Map; import java.util.Set; @@ -69,8 +69,6 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private transient Connection conn; private transient Table table; - public final SimpleDateFormat ROWKEY_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss"); - @Override public void configure(Configuration parameters) { LOG.warn("---configure---"); @@ -178,30 +176,23 @@ private String buildRowKey(Row record) { private List getRowKeyValues(Row record) { List rowKeyValues = Lists.newArrayList(); - for (int i = 0; i < rowkey.length; ++i) { - String colName = rowkey[i]; - int rowKeyIndex = 0; - for (; rowKeyIndex < columnNames.length; ++rowKeyIndex) { - if (columnNames[rowKeyIndex].equals(colName)) { - break; - } - } - - if (rowKeyIndex != columnNames.length && record.getField(rowKeyIndex) != null) { - Object field = record.getField(rowKeyIndex); - if (field == null) { - continue; - } else if (field instanceof java.util.Date) { - java.util.Date d = (java.util.Date) field; - rowKeyValues.add(ROWKEY_DATE_FORMAT.format(d)); - } else { - rowKeyValues.add(field.toString()); - } - } + Map row = rowConvertMap(record); + for (String key : rowkey) { + RowKeyBuilder rowKeyBuilder = new RowKeyBuilder(); + rowKeyBuilder.init(key); + rowKeyValues.add(rowKeyBuilder.getRowKey(row)); } return rowKeyValues; } + private Map rowConvertMap(Row record){ + Map rowValue = Maps.newHashMap(); + for(int i = 0; i < columnNames.length; i++){ + rowValue.put(columnNames[i], record.getField(i)); + } + return rowValue; + } + @Override public void close() throws IOException { if (conn != null) { diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/Md5ReplaceOperator.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/Md5ReplaceOperator.java new file mode 100644 index 000000000..e9474220a --- /dev/null +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/Md5ReplaceOperator.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flink.sql.sink.hbase; + +import com.dtstack.flink.sql.sink.hbase.enums.EReplaceOpType; +import com.dtstack.flink.sql.util.MD5Utils; + +/** + * Reason: + * Date: 2018/8/23 + * Company: www.dtstack.com + * @author xuchao + */ + +public class Md5ReplaceOperator extends AbstractReplaceOperator { + + public Md5ReplaceOperator(EReplaceOpType opType) { + super(opType); + } + + @Override + String doFunc(String replaceStr) { + return MD5Utils.getMD5String(replaceStr); + } +} diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/ReplaceInfo.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/ReplaceInfo.java new file mode 100644 index 000000000..c233bfe6b --- /dev/null +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/ReplaceInfo.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flink.sql.sink.hbase; + + +import com.dtstack.flink.sql.sink.hbase.enums.EReplaceType; + +import java.io.Serializable; +import java.util.List; + +/** + * Reason: + * Date: 2018/8/23 + * Company: www.dtstack.com + * + * @author xuchao + */ + +public class ReplaceInfo implements Serializable { + + private static final long serialVersionUID = 2058635242957737717L; + + private EReplaceType type; + + private String param; + + private List subReplaceInfos; + + public ReplaceInfo(EReplaceType type){ + this.type = type; + } + + public EReplaceType getType() { + return type; + } + + public void setType(EReplaceType type) { + this.type = type; + } + + public String getParam() { + return param; + } + + public void setParam(String param) { + this.param = param; + } + + public List getSubReplaceInfos() { + return subReplaceInfos; + } + + public void setSubReplaceInfos(List subReplaceInfos) { + this.subReplaceInfos = subReplaceInfos; + } +} diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java new file mode 100644 index 000000000..89667ee28 --- /dev/null +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flink.sql.sink.hbase; + +import com.dtstack.flink.sql.sink.hbase.enums.EReplaceType; +import com.dtstack.flink.sql.util.MD5Utils; +import com.google.common.collect.Lists; +import org.apache.commons.collections.CollectionUtils; + +import javax.xml.crypto.Data; +import java.io.Serializable; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * rowkey rule + * Date: 2018/8/23 + * Company: www.dtstack.com + * @author xuchao + */ + +public class RowKeyBuilder implements Serializable{ + + private static final long serialVersionUID = 2058635242857937717L; + + private static Pattern Md5Operator = Pattern.compile("(?i)^md5\\(\\s*(.*)\\s*\\)$"); + + private List operatorChain = Lists.newArrayList(); + + public void init(String rowKeyTempl){ + operatorChain.addAll(makeFormula(rowKeyTempl)); + } + + /** + * + * @param refData + * @return + */ + public String getRowKey(Map refData){ + return buildStr(operatorChain, refData); + } + + + + private String buildStr(List fieldList, Map refData){ + if(CollectionUtils.isEmpty(fieldList)){ + return ""; + } + StringBuffer sb = new StringBuffer(""); + for(ReplaceInfo replaceInfo : fieldList){ + + if(replaceInfo.getType() == EReplaceType.CONSTANT){ + sb.append(replaceInfo.getParam()); + continue; + } + + if(replaceInfo.getType() == EReplaceType.FUNC){ + sb.append(MD5Utils.getMD5String(buildStr(replaceInfo.getSubReplaceInfos(), refData))); + continue; + } + String replaceName = replaceInfo.getParam(); + if(!refData.containsKey(replaceName)){ + throw new RuntimeException(String.format("build rowKey with field %s which value not found.", replaceName)); + } + Object value = refData.get(replaceName); + if(value instanceof Date){ + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss"); + java.util.Date d = (java.util.Date) value; + sb.append(sdf.format(d)); + } else { + sb.append(value); + } + } + + return sb.toString(); + } + + public static String[] splitIgnoreQuotaBrackets(String str, String delimiter){ + String splitPatternStr = delimiter + "(?![^()]*+\\))(?![^{}]*+})(?![^\\[\\]]*+\\])"; + return str.split(splitPatternStr); + } + + /** + * + * @param field + * @return + */ + public ReplaceInfo getReplaceInfo(String field){ + + field = field.trim(); + if(field.length() <= 2){ + throw new RuntimeException(field + " \n" + + "Format defined exceptions"); + } + + //判断是不是常量==>''包裹的标识 + if(field.startsWith("'") && field.endsWith("'")){ + ReplaceInfo replaceInfo = new ReplaceInfo(EReplaceType.CONSTANT); + field = field.substring(1, field.length() - 1); + replaceInfo.setParam(field); + return replaceInfo; + } + + ReplaceInfo replaceInfo = new ReplaceInfo(EReplaceType.PARAM); + replaceInfo.setParam(field); + return replaceInfo; + } + + private List makeFormula(String formula){ + if(formula == null || formula.length() <= 0){ + Lists.newArrayList(); + } + List result = Lists.newArrayList(); + for(String meta: splitIgnoreQuotaBrackets(formula, "\\+")){ + Matcher matcher = Md5Operator.matcher(meta.trim()); + if(matcher.find()){ + ReplaceInfo replaceInfo = new ReplaceInfo(EReplaceType.FUNC); + replaceInfo.setSubReplaceInfos(makeFormula(matcher.group(1))); + result.add(replaceInfo); + } else { + result.add(getReplaceInfo(meta)); + } + } + return result; + } +} diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceOpType.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceOpType.java new file mode 100644 index 000000000..8621f352d --- /dev/null +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceOpType.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flink.sql.sink.hbase.enums; + +/** + * Reason: + * Date: 2018/8/23 + * Company: www.dtstack.com + * @author xuchao + */ + +public enum EReplaceOpType { + /** + * 没有func + */ + NO_FUNC, + /** + * md5 func + */ + MD5_FUNC; +} diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceType.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceType.java new file mode 100644 index 000000000..ad6bbcc4e --- /dev/null +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceType.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flink.sql.sink.hbase.enums; + +/** + * Reason: + * Date: 2018/8/23 + * Company: www.dtstack.com + * + * @author xuchao + */ +public enum EReplaceType { + /** + * 参数 + */ + PARAM, + /** + * 函数 + */ + FUNC, + /** + * 常量 + */ + CONSTANT; +} From a7967f7a5edb50433959f90c5af93f648767f737 Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 30 Mar 2020 17:21:28 +0800 Subject: [PATCH 200/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=AF=B9groupby?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E5=AF=B9=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/JoinNodeDealer.java | 77 ++++++++++++++----- .../dtstack/flink/sql/side/SideSQLParser.java | 28 ++++--- .../dtstack/flink/sql/util/TableUtils.java | 18 +++-- 3 files changed, 85 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index d321130cc..806d00afb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -86,6 +86,7 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, Queue queueInfo, SqlNode parentWhere, SqlNodeList parentSelectList, + SqlNodeList parentGroupByList, Set> joinFieldSet, Map tableRef, Map fieldRef) { @@ -105,12 +106,12 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, if (leftNode.getKind() == JOIN) { //处理连续join dealNestJoin(joinNode, sideTableSet, - queueInfo, parentWhere, parentSelectList, joinFieldSet, tableRef, fieldRef, parentSelectList); + queueInfo, parentWhere, parentSelectList, parentGroupByList, joinFieldSet, tableRef, fieldRef); leftNode = joinNode.getLeft(); } if (leftNode.getKind() == AS) { - AliasInfo aliasInfo = (AliasInfo) sideSQLParser.parseSql(leftNode, sideTableSet, queueInfo, parentWhere, parentSelectList); + AliasInfo aliasInfo = (AliasInfo) sideSQLParser.parseSql(leftNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); leftTbName = aliasInfo.getName(); leftTbAlias = aliasInfo.getAlias(); } @@ -118,7 +119,7 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, boolean leftIsSide = checkIsSideTable(leftTbName, sideTableSet); Preconditions.checkState(!leftIsSide, "side-table must be at the right of join operator"); - Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, parentWhere, parentSelectList); + Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); rightTableName = rightTableNameAndAlias.f0; rightTableAlias = rightTableNameAndAlias.f1; @@ -145,7 +146,7 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, //extract 需要查询的字段信息 if(rightIsSide){ - extractJoinNeedSelectField(leftNode, rightNode, parentWhere, parentSelectList, tableRef, joinFieldSet, fieldRef, tableInfo); + extractJoinNeedSelectField(leftNode, rightNode, parentWhere, parentSelectList, parentGroupByList, tableRef, joinFieldSet, fieldRef, tableInfo); } if(tableInfo.getLeftNode().getKind() != AS){ @@ -168,13 +169,14 @@ public void extractJoinNeedSelectField(SqlNode leftNode, SqlNode rightNode, SqlNode parentWhere, SqlNodeList parentSelectList, + SqlNodeList parentGroupByList, Map tableRef, Set> joinFieldSet, Map fieldRef, JoinInfo tableInfo){ - Set extractSelectField = extractField(leftNode, parentWhere, parentSelectList, tableRef, joinFieldSet); - Set rightExtractSelectField = extractField(rightNode, parentWhere, parentSelectList, tableRef, joinFieldSet); + Set extractSelectField = extractField(leftNode, parentWhere, parentSelectList, parentGroupByList, tableRef, joinFieldSet); + Set rightExtractSelectField = extractField(rightNode, parentWhere, parentSelectList, parentGroupByList, tableRef, joinFieldSet); //重命名right 中和 left 重名的 Map leftTbSelectField = Maps.newHashMap(); @@ -208,6 +210,7 @@ public void extractJoinNeedSelectField(SqlNode leftNode, * @param sqlNode * @param parentWhere * @param parentSelectList + * @param parentGroupByList * @param tableRef * @param joinFieldSet * @return @@ -215,6 +218,7 @@ public void extractJoinNeedSelectField(SqlNode leftNode, public Set extractField(SqlNode sqlNode, SqlNode parentWhere, SqlNodeList parentSelectList, + SqlNodeList parentGroupByList, Map tableRef, Set> joinFieldSet){ Set fromTableNameSet = Sets.newHashSet(); @@ -225,8 +229,11 @@ public Set extractField(SqlNode sqlNode, Set extractSelectField = extractSelectFields(parentSelectList, fromTableNameSet, tableRef); Set fieldFromJoinCondition = extractSelectFieldFromJoinCondition(joinFieldSet, fromTableNameSet, tableRef); + Set extractGroupByField = extractFieldFromGroupByList(parentGroupByList, fromTableNameSet, tableRef); + extractSelectField.addAll(extractCondition); extractSelectField.addAll(fieldFromJoinCondition); + extractSelectField.addAll(extractGroupByField); return extractSelectField; } @@ -242,27 +249,27 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, Set sideTableSet, Queue queueInfo, SqlNode parentWhere, - SqlNodeList selectList, + SqlNodeList parentSelectList, + SqlNodeList parentGroupByList, Set> joinFieldSet, Map tableRef, - Map fieldRef, - SqlNodeList parentSelectList){ + Map fieldRef){ SqlJoin leftJoinNode = (SqlJoin) joinNode.getLeft(); SqlNode parentRightJoinNode = joinNode.getRight(); SqlNode rightNode = leftJoinNode.getRight(); - Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, parentWhere, selectList); - Tuple2 parentRightJoinInfo = parseRightNode(parentRightJoinNode, sideTableSet, queueInfo, parentWhere, selectList); + Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + Tuple2 parentRightJoinInfo = parseRightNode(parentRightJoinNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); boolean parentRightIsSide = checkIsSideTable(parentRightJoinInfo.f0, sideTableSet); - JoinInfo joinInfo = dealJoinNode(leftJoinNode, sideTableSet, queueInfo, parentWhere, selectList, joinFieldSet, tableRef, fieldRef); + JoinInfo joinInfo = dealJoinNode(leftJoinNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, joinFieldSet, tableRef, fieldRef); String rightTableName = rightTableNameAndAlias.f0; boolean rightIsSide = checkIsSideTable(rightTableName, sideTableSet); SqlBasicCall buildAs = TableUtils.buildAsNodeByJoinInfo(joinInfo, null, null); if(rightIsSide){ - addSideInfoToExeQueue(queueInfo, joinInfo, joinNode, parentSelectList, parentWhere, tableRef); + addSideInfoToExeQueue(queueInfo, joinInfo, joinNode, parentSelectList, parentGroupByList, parentWhere, tableRef); } SqlNode newLeftNode = joinNode.getLeft(); @@ -275,7 +282,7 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, //替换leftNode 为新的查询 joinNode.setLeft(buildAs); - replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, parentSelectList, parentWhere); + replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, parentSelectList, parentGroupByList, parentWhere); } return joinInfo; @@ -288,6 +295,7 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, * @param joinInfo * @param joinNode * @param parentSelectList + * @param parentGroupByList * @param parentWhere * @param tableRef */ @@ -295,6 +303,7 @@ public void addSideInfoToExeQueue(Queue queueInfo, JoinInfo joinInfo, SqlJoin joinNode, SqlNodeList parentSelectList, + SqlNodeList parentGroupByList, SqlNode parentWhere, Map tableRef){ //只处理维表 @@ -308,7 +317,7 @@ public void addSideInfoToExeQueue(Queue queueInfo, //替换左表为新的表名称 joinNode.setLeft(buildAs); - replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, parentSelectList, parentWhere); + replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, parentSelectList, parentGroupByList, parentWhere); } /** @@ -317,12 +326,14 @@ public void addSideInfoToExeQueue(Queue queueInfo, * @param leftJoinNode * @param tableRef * @param parentSelectList + * @param parentGroupByList * @param parentWhere */ public void replaceSelectAndWhereField(SqlBasicCall buildAs, SqlNode leftJoinNode, Map tableRef, SqlNodeList parentSelectList, + SqlNodeList parentGroupByList, SqlNode parentWhere){ String newLeftTableName = buildAs.getOperands()[1].toString(); @@ -341,10 +352,20 @@ public void replaceSelectAndWhereField(SqlBasicCall buildAs, } } + //TODO 应该根据上面的查询字段的关联关系来替换 //替换where 中的条件相关 for(String tbTmp : fromTableNameSet){ - TableUtils.replaceWhereCondition(parentWhere, tbTmp, newLeftTableName); + TableUtils.replaceWhereCondition(parentWhere, tbTmp, newLeftTableName, fieldReplaceRef); } + + if(parentGroupByList != null){ + for(SqlNode sqlNode : parentGroupByList.getList()){ + for(String tbTmp : fromTableNameSet) { + TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldReplaceRef); + } + } + } + } /** @@ -407,7 +428,7 @@ private void extractTemporaryQuery(SqlNode node, String tableAlias, //替换where 中的条件相关 for(String tbTmp : fromTableNameSet){ - TableUtils.replaceWhereCondition(parentWhere, tbTmp, tableAlias); + TableUtils.replaceWhereCondition(parentWhere, tbTmp, tableAlias, fieldReplaceRef); } for(String tbTmp : fromTableNameSet){ @@ -426,7 +447,6 @@ private void extractTemporaryQuery(SqlNode node, String tableAlias, /** * 抽取上层需用使用到的字段 - * 由于where字段已经抽取到上一层了所以不用查询出来 * @param parentSelectList * @param fromTableNameSet * @return @@ -451,7 +471,6 @@ private Set extractSelectFieldFromJoinCondition(Set extractSelectFieldFromJoinCondition(Set extractFieldFromGroupByList(SqlNodeList parentGroupByList, + Set fromTableNameSet, + Map tableRef){ + + if(parentGroupByList == null){ + return Sets.newHashSet(); + } + + Set extractFieldList = Sets.newHashSet(); + for(SqlNode selectNode : parentGroupByList.getList()){ + extractSelectField(selectNode, extractFieldList, fromTableNameSet, tableRef); + } + + return extractFieldList; + } + /** * 从join的条件中获取字段信息 * @param condition @@ -573,12 +608,12 @@ private void extractSelectField(SqlNode selectNode, private Tuple2 parseRightNode(SqlNode sqlNode, Set sideTableSet, Queue queueInfo, - SqlNode parentWhere, SqlNodeList selectList) { + SqlNode parentWhere, SqlNodeList selectList, SqlNodeList parentGroupByList) { Tuple2 tabName = new Tuple2<>("", ""); if(sqlNode.getKind() == IDENTIFIER){ tabName.f0 = sqlNode.toString(); }else{ - AliasInfo aliasInfo = (AliasInfo)sideSQLParser.parseSql(sqlNode, sideTableSet, queueInfo, parentWhere, selectList); + AliasInfo aliasInfo = (AliasInfo)sideSQLParser.parseSql(sqlNode, sideTableSet, queueInfo, parentWhere, selectList, parentGroupByList); tabName.f0 = aliasInfo.getName(); tabName.f1 = aliasInfo.getAlias(); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index a94d218a2..7656c3e85 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -85,7 +85,7 @@ public Queue getExeQueue(String exeSql, Set sideTableSet) throws SqlParser sqlParser = SqlParser.create(exeSql, CalciteConfig.MYSQL_LEX_CONFIG); SqlNode sqlNode = sqlParser.parseStmt(); - parseSql(sqlNode, sideTableSet, queueInfo, null, null); + parseSql(sqlNode, sideTableSet, queueInfo, null, null, null); queueInfo.offer(sqlNode); return queueInfo; } @@ -100,7 +100,12 @@ public Queue getExeQueue(String exeSql, Set sideTableSet) throws * @param parentSelectList * @return */ - public Object parseSql(SqlNode sqlNode, Set sideTableSet, Queue queueInfo, SqlNode parentWhere, SqlNodeList parentSelectList){ + public Object parseSql(SqlNode sqlNode, + Set sideTableSet, + Queue queueInfo, + SqlNode parentWhere, + SqlNodeList parentSelectList, + SqlNodeList parentGroupByList){ SqlKind sqlKind = sqlNode.getKind(); switch (sqlKind){ case WITH: { @@ -108,22 +113,23 @@ public Object parseSql(SqlNode sqlNode, Set sideTableSet, Queue SqlNodeList sqlNodeList = sqlWith.withList; for (SqlNode withAsTable : sqlNodeList) { SqlWithItem sqlWithItem = (SqlWithItem) withAsTable; - parseSql(sqlWithItem.query, sideTableSet, queueInfo, parentWhere, parentSelectList); + parseSql(sqlWithItem.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); queueInfo.add(sqlWithItem); } - parseSql(sqlWith.body, sideTableSet, queueInfo, parentWhere, parentSelectList); + parseSql(sqlWith.body, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); break; } case INSERT: SqlNode sqlSource = ((SqlInsert)sqlNode).getSource(); - return parseSql(sqlSource, sideTableSet, queueInfo, parentWhere, parentSelectList); + return parseSql(sqlSource, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); case SELECT: SqlNode sqlFrom = ((SqlSelect)sqlNode).getFrom(); SqlNode sqlWhere = ((SqlSelect)sqlNode).getWhere(); SqlNodeList selectList = ((SqlSelect)sqlNode).getSelectList(); + SqlNodeList groupByList = ((SqlSelect) sqlNode).getGroup(); if(sqlFrom.getKind() != IDENTIFIER){ - Object result = parseSql(sqlFrom, sideTableSet, queueInfo, sqlWhere, selectList); + Object result = parseSql(sqlFrom, sideTableSet, queueInfo, sqlWhere, selectList, groupByList); if(result instanceof JoinInfo){ return TableUtils.dealSelectResultWithJoinInfo((JoinInfo) result, (SqlSelect) sqlNode, queueInfo); }else if(result instanceof AliasInfo){ @@ -145,7 +151,7 @@ public Object parseSql(SqlNode sqlNode, Set sideTableSet, Queue Map tableRef = Maps.newHashMap(); Map fieldRef = Maps.newHashMap(); return joinNodeDealer.dealJoinNode((SqlJoin) sqlNode, sideTableSet, queueInfo, - parentWhere, parentSelectList, joinFieldSet, tableRef, fieldRef); + parentWhere, parentSelectList, parentGroupByList, joinFieldSet, tableRef, fieldRef); case AS: SqlNode info = ((SqlBasicCall)sqlNode).getOperands()[0]; SqlNode alias = ((SqlBasicCall) sqlNode).getOperands()[1]; @@ -154,7 +160,7 @@ public Object parseSql(SqlNode sqlNode, Set sideTableSet, Queue if(info.getKind() == IDENTIFIER){ infoStr = info.toString(); } else { - infoStr = parseSql(info, sideTableSet, queueInfo, parentWhere, parentSelectList).toString(); + infoStr = parseSql(info, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList).toString(); } AliasInfo aliasInfo = new AliasInfo(); @@ -167,12 +173,12 @@ public Object parseSql(SqlNode sqlNode, Set sideTableSet, Queue SqlNode unionLeft = ((SqlBasicCall)sqlNode).getOperands()[0]; SqlNode unionRight = ((SqlBasicCall)sqlNode).getOperands()[1]; - parseSql(unionLeft, sideTableSet, queueInfo, parentWhere, parentSelectList); - parseSql(unionRight, sideTableSet, queueInfo, parentWhere, parentSelectList); + parseSql(unionLeft, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + parseSql(unionRight, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); break; case ORDER_BY: SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode; - parseSql(sqlOrderBy.query, sideTableSet, queueInfo, parentWhere, parentSelectList); + parseSql(sqlOrderBy.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); } return ""; } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index e62f3fcfa..5a6190210 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -478,7 +478,7 @@ public static String getTargetRefField(Map refFieldMap, String c return preFieldName; } - public static void replaceWhereCondition(SqlNode parentWhere, String oldTbName, String newTbName){ + public static void replaceWhereCondition(SqlNode parentWhere, String oldTbName, String newTbName, HashBiMap fieldReplaceRef){ if(parentWhere == null){ return; @@ -486,15 +486,15 @@ public static void replaceWhereCondition(SqlNode parentWhere, String oldTbName, SqlKind kind = parentWhere.getKind(); if(kind == AND){ - replaceWhereCondition(((SqlBasicCall) parentWhere).getOperands()[0], oldTbName, newTbName); - replaceWhereCondition(((SqlBasicCall) parentWhere).getOperands()[1], oldTbName, newTbName); + replaceWhereCondition(((SqlBasicCall) parentWhere).getOperands()[0], oldTbName, newTbName, fieldReplaceRef); + replaceWhereCondition(((SqlBasicCall) parentWhere).getOperands()[1], oldTbName, newTbName, fieldReplaceRef); } else { - replaceConditionNode(parentWhere, oldTbName, newTbName); + replaceConditionNode(parentWhere, oldTbName, newTbName, fieldReplaceRef); } } - private static void replaceConditionNode(SqlNode selectNode, String oldTbName, String newTbName) { + private static void replaceConditionNode(SqlNode selectNode, String oldTbName, String newTbName, HashBiMap fieldReplaceRef) { if(selectNode.getKind() == IDENTIFIER){ SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; @@ -503,8 +503,14 @@ private static void replaceConditionNode(SqlNode selectNode, String oldTbName, S } String tableName = sqlIdentifier.names.asList().get(0); + String tableField = sqlIdentifier.names.asList().get(1); + String fieldKey = tableName + "_" + tableField; + if(tableName.equalsIgnoreCase(oldTbName)){ + + String newFieldName = fieldReplaceRef.get(fieldKey) == null ? tableField : fieldReplaceRef.get(fieldKey); SqlIdentifier newField = ((SqlIdentifier)selectNode).setName(0, newTbName); + newField = newField.setName(1, newFieldName); ((SqlIdentifier)selectNode).assignNamesFrom(newField); } return; @@ -552,7 +558,7 @@ private static void replaceConditionNode(SqlNode selectNode, String oldTbName, S continue; } - replaceConditionNode(sqlNode, oldTbName, newTbName); + replaceConditionNode(sqlNode, oldTbName, newTbName, fieldReplaceRef); } return; From 0b241599be00ecbb051ca384f5ff9af7c2dea8d9 Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 30 Mar 2020 17:35:48 +0800 Subject: [PATCH 201/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=AF=B9join?= =?UTF-8?q?=E8=BF=87=E7=A8=8B=E4=B8=AD=E6=9C=AA=E5=8F=96=E5=88=AB=E5=90=8D?= =?UTF-8?q?=E8=A7=A3=E6=9E=90bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index 806d00afb..df23fa4a8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -114,6 +114,9 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, AliasInfo aliasInfo = (AliasInfo) sideSQLParser.parseSql(leftNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); leftTbName = aliasInfo.getName(); leftTbAlias = aliasInfo.getAlias(); + } else if(leftNode.getKind() == IDENTIFIER){ + leftTbName = leftNode.toString(); + leftTbAlias = leftTbName; } boolean leftIsSide = checkIsSideTable(leftTbName, sideTableSet); From 6754700c3708b46af6cdf5dfbd8c15277665c497 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 31 Mar 2020 09:43:46 +0800 Subject: [PATCH 202/523] =?UTF-8?q?fix=20=E7=B1=BB=E5=9E=8B=E5=88=A4?= =?UTF-8?q?=E6=96=ADbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kafka/KafkaDeserializationMetricWrapper.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java index d0d76747d..bf0cee7dd 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java @@ -76,13 +76,7 @@ protected void beforeDeserialize() throws IOException { } protected void registerPtMetric(AbstractFetcher fetcher) throws Exception { - Field consumerThreadField = null; - if(fetcher.getClass().getDeclaredField("consumerThread") != null){ - consumerThreadField = fetcher.getClass().getDeclaredField("consumerThread"); - } else { - consumerThreadField = fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); - } - + Field consumerThreadField = getConsumerThreadField(fetcher); consumerThreadField.setAccessible(true); KafkaConsumerThread consumerThread = (KafkaConsumerThread) consumerThreadField.get(fetcher); @@ -123,4 +117,12 @@ public Long getValue() { public void setFetcher(AbstractFetcher fetcher) { this.fetcher = fetcher; } + + private Field getConsumerThreadField(AbstractFetcher fetcher) throws NoSuchFieldException { + try { + return fetcher.getClass().getDeclaredField("consumerThread"); + } catch (Exception e) { + return fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); + } + } } From e5dbaf44e3b307208a82f8964a72497f27c4b6f4 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 31 Mar 2020 19:36:58 +0800 Subject: [PATCH 203/523] use flink planner --- .../main/java/com/dtstack/flink/sql/Main.java | 9 ++-- .../flink/sql/config/CalciteConfig.java | 35 ------------- .../sql/parser/CreateTmpTableParser.java | 16 ++---- .../flink/sql/parser/FlinkPlanner.java | 52 +++++++++++++++++++ .../flink/sql/side/JoinNodeDealer.java | 9 ++-- .../flink/sql/side/SidePredicatesParser.java | 9 ++-- .../dtstack/flink/sql/side/SideSQLParser.java | 22 ++------ 7 files changed, 76 insertions(+), 76 deletions(-) delete mode 100644 core/src/main/java/com/dtstack/flink/sql/config/CalciteConfig.java create mode 100644 core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index 443033b3b..399296fc4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -19,7 +19,6 @@ package com.dtstack.flink.sql; -import com.dtstack.flink.sql.config.CalciteConfig; import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.enums.ECacheType; @@ -29,6 +28,7 @@ import com.dtstack.flink.sql.option.OptionParser; import com.dtstack.flink.sql.parser.CreateFuncParser; import com.dtstack.flink.sql.parser.CreateTmpTableParser; +import com.dtstack.flink.sql.parser.FlinkPlanner; import com.dtstack.flink.sql.parser.InsertSqlParser; import com.dtstack.flink.sql.parser.SqlParser; import com.dtstack.flink.sql.parser.SqlTree; @@ -60,6 +60,7 @@ import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.calcite.FlinkPlannerImpl; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -72,7 +73,6 @@ import java.net.URLDecoder; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.Set; @@ -120,6 +120,8 @@ public static void main(String[] args) throws Exception { StreamExecutionEnvironment env = getStreamExeEnv(confProperties, deployMode); StreamTableEnvironment tableEnv = StreamTableEnvironment.getTableEnvironment(env); StreamQueryConfig streamQueryConfig = StreamEnvConfigManager.getStreamQueryConfig(tableEnv, confProperties); + // init global flinkPlanner + FlinkPlanner.createFlinkPlanner(tableEnv.getFrameworkConfig(), tableEnv.getPlanner(), tableEnv.getTypeFactory()); List jarURList = Lists.newArrayList(); SqlTree sqlTree = SqlParser.parseSql(sql); @@ -171,8 +173,9 @@ private static void sqlTranslation(String localSqlPluginPath, if (sqlTree.getTmpTableMap().containsKey(tableName)) { CreateTmpTableParser.SqlParserResult tmp = sqlTree.getTmpTableMap().get(tableName); String realSql = DtStringUtil.replaceIgnoreQuota(result.getExecSql(), "`", ""); + FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); - SqlNode sqlNode = org.apache.calcite.sql.parser.SqlParser.create(realSql, CalciteConfig.MYSQL_LEX_CONFIG).parseStmt(); + SqlNode sqlNode = flinkPlanner.parse(realSql); String tmpSql = ((SqlInsert) sqlNode).getSource().toString(); tmp.setExecSql(tmpSql); sideSqlExec.exec(tmp.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, tmp); diff --git a/core/src/main/java/com/dtstack/flink/sql/config/CalciteConfig.java b/core/src/main/java/com/dtstack/flink/sql/config/CalciteConfig.java deleted file mode 100644 index 54ae66bbc..000000000 --- a/core/src/main/java/com/dtstack/flink/sql/config/CalciteConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.config; - -import org.apache.calcite.config.Lex; -import org.apache.calcite.sql.parser.SqlParser; -import org.apache.calcite.sql.parser.SqlParser.Config; - -public class CalciteConfig { - - public static Config MYSQL_LEX_CONFIG = SqlParser - .configBuilder() - .setLex(Lex.MYSQL) - .build(); - - - -} diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java index de7141eb5..114dbd50b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java @@ -21,11 +21,10 @@ package com.dtstack.flink.sql.parser; import com.dtstack.flink.sql.util.DtStringUtil; -import org.apache.calcite.config.Lex; import org.apache.calcite.sql.*; -import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.calcite.sql.parser.SqlParser; import com.google.common.collect.Lists; +import org.apache.flink.table.calcite.FlinkPlannerImpl; + import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -71,17 +70,12 @@ public void parseSql(String sql, SqlTree sqlTree) { tableName = matcher.group(1); selectSql = "select " + matcher.group(2); } - - SqlParser.Config config = SqlParser - .configBuilder() - .setLex(Lex.MYSQL) - .build(); - SqlParser sqlParser = SqlParser.create(selectSql,config); + FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); SqlNode sqlNode = null; try { - sqlNode = sqlParser.parseStmt(); - } catch (SqlParseException e) { + sqlNode = flinkPlanner.parse(selectSql); + } catch (Exception e) { throw new RuntimeException("", e); } diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java b/core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java new file mode 100644 index 000000000..7c76ec2cd --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.parser; + +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.tools.FrameworkConfig; +import org.apache.flink.table.calcite.FlinkPlannerImpl; +import org.apache.flink.table.calcite.FlinkTypeFactory; + +/** + * Date: 2020/3/31 + * Company: www.dtstack.com + * @author maqi + */ +public class FlinkPlanner { + + public static volatile FlinkPlannerImpl flinkPlanner; + + private FlinkPlanner() { + } + + public static FlinkPlannerImpl createFlinkPlanner(FrameworkConfig frameworkConfig, RelOptPlanner relOptPlanner, FlinkTypeFactory typeFactory) { + if (flinkPlanner == null) { + synchronized (FlinkPlanner.class) { + if (flinkPlanner == null) { + flinkPlanner = new FlinkPlannerImpl(frameworkConfig, relOptPlanner, typeFactory); + } + } + } + return flinkPlanner; + } + + public static FlinkPlannerImpl getFlinkPlanner() { + return flinkPlanner; + } +} diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index df23fa4a8..fc027cb70 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.config.CalciteConfig; +import com.dtstack.flink.sql.parser.FlinkPlanner; import com.dtstack.flink.sql.util.ParseUtils; import com.dtstack.flink.sql.util.TableUtils; import com.google.common.base.Preconditions; @@ -38,11 +38,11 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.table.calcite.FlinkPlannerImpl; import java.util.List; import java.util.Map; @@ -416,8 +416,9 @@ private void extractTemporaryQuery(SqlNode node, String tableAlias, node.toString(), extractConditionStr); - SqlParser sqlParser = SqlParser.create(tmpSelectSql, CalciteConfig.MYSQL_LEX_CONFIG); - SqlNode sqlNode = sqlParser.parseStmt(); + FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); + SqlNode sqlNode = flinkPlanner.parse(tmpSelectSql); + SqlBasicCall sqlBasicCall = buildAsSqlNode(tableAlias, sqlNode); queueInfo.offer(sqlBasicCall); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java index 50103a9f5..0902bf39f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.config.CalciteConfig; +import com.dtstack.flink.sql.parser.FlinkPlanner; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.calcite.sql.SqlBasicCall; @@ -30,11 +30,10 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.calcite.sql.parser.SqlParser; +import org.apache.flink.table.calcite.FlinkPlannerImpl; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static org.apache.calcite.sql.SqlKind.*; @@ -47,8 +46,8 @@ */ public class SidePredicatesParser { public void fillPredicatesForSideTable(String exeSql, Map sideTableMap) throws SqlParseException { - SqlParser sqlParser = SqlParser.create(exeSql, CalciteConfig.MYSQL_LEX_CONFIG); - SqlNode sqlNode = sqlParser.parseStmt(); + FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); + SqlNode sqlNode = flinkPlanner.parse(exeSql); parseSql(sqlNode, sideTableMap, Maps.newHashMap()); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index 7656c3e85..bc9f7a43c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -20,43 +20,29 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.config.CalciteConfig; +import com.dtstack.flink.sql.parser.FlinkPlanner; import com.dtstack.flink.sql.util.TableUtils; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Queues; import com.google.common.collect.Sets; -import org.apache.calcite.sql.JoinType; -import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlBinaryOperator; -import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlInsert; import org.apache.calcite.sql.SqlJoin; import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlWith; import org.apache.calcite.sql.SqlWithItem; -import org.apache.calcite.sql.fun.SqlCase; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.calcite.sql.parser.SqlParser; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.table.api.Table; +import org.apache.flink.table.calcite.FlinkPlannerImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; import java.util.Map; import java.util.Queue; import java.util.Set; @@ -82,8 +68,8 @@ public Queue getExeQueue(String exeSql, Set sideTableSet) throws LOG.info(exeSql); Queue queueInfo = Queues.newLinkedBlockingQueue(); - SqlParser sqlParser = SqlParser.create(exeSql, CalciteConfig.MYSQL_LEX_CONFIG); - SqlNode sqlNode = sqlParser.parseStmt(); + FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); + SqlNode sqlNode = flinkPlanner.parse(exeSql); parseSql(sqlNode, sideTableSet, queueInfo, null, null, null); queueInfo.offer(sqlNode); From 53f4f40bee5262dbcdb7b36758988806b0d013c6 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 31 Mar 2020 21:59:48 +0800 Subject: [PATCH 204/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/option/OptionParser.java | 5 +- .../sql/launcher/ClusterClientFactory.java | 2 +- .../flink/sql/launcher/LauncherMain.java | 53 +++++++++++-------- .../perjob/PerJobClusterClientBuilder.java | 4 +- .../sql/launcher/perjob/PerJobSubmitter.java | 36 +++++++------ 5 files changed, 56 insertions(+), 44 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java index 0fd057c5a..6e2413437 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java @@ -26,6 +26,7 @@ import org.apache.commons.lang.StringUtils; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.io.File; @@ -104,8 +105,8 @@ public List getProgramExeArgList() throws Exception { continue; } else if (OPTION_SQL.equalsIgnoreCase(key)) { File file = new File(value.toString()); - String content = FileUtils.readFile(file, "UTF-8"); - value = URLEncoder.encode(content, Charsets.UTF_8.name()); + String content = FileUtils.readFile(file, StandardCharsets.UTF_8.name()); + value = URLEncoder.encode(content, StandardCharsets.UTF_8.name()); } args.add("-" + key); args.add(value.toString()); diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 8f0090db5..4c421fa54 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -96,7 +96,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(yarnConf); yarnClient.start(); - ApplicationId applicationId = null; + ApplicationId applicationId; String yarnSessionConf = launcherOptions.getYarnSessionConf(); yarnSessionConf = URLDecoder.decode(yarnSessionConf, Charsets.UTF_8.toString()); diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 0b036c87c..92259c57d 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -16,11 +16,11 @@ * limitations under the License. */ - package com.dtstack.flink.sql.launcher; -import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; +//import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; + import com.dtstack.flink.sql.constrant.ConfigConstrant; import com.google.common.collect.Lists; import com.alibaba.fastjson.JSON; @@ -41,6 +41,8 @@ import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.File; @@ -48,6 +50,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -56,6 +59,7 @@ /** * Date: 2017/2/20 * Company: www.dtstack.com + * * @author xuchao */ @@ -64,15 +68,16 @@ public class LauncherMain { private static String SP = File.separator; + private static final Logger LOG = LoggerFactory.getLogger(LauncherMain.class); + private static String getLocalCoreJarPath(String localSqlRootJar) throws Exception { String jarPath = PluginUtil.getCoreJarFileName(localSqlRootJar, CORE_JAR); - String corePath = localSqlRootJar + SP + jarPath; - return corePath; + return localSqlRootJar + SP + jarPath; } public static void main(String[] args) throws Exception { - if (args.length == 1 && args[0].endsWith(".json")){ + if (args.length == 1 && args[0].endsWith(".json")) { args = parseJson(args); } OptionParser optionParser = new OptionParser(args); @@ -84,27 +89,31 @@ public static void main(String[] args) throws Exception { confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - if(mode.equals(ClusterMode.local.name())) { - String[] localArgs = argList.toArray(new String[argList.size()]); +// LOG.info("current mode is {}", mode); + System.out.println("current mode is " + mode); + + if (mode.equals(ClusterMode.local.name())) { + String[] localArgs = argList.toArray(new String[0]); Main.main(localArgs); return; } String pluginRoot = launcherOptions.getLocalSqlPluginPath(); File jarFile = new File(getLocalCoreJarPath(pluginRoot)); - String[] remoteArgs = argList.toArray(new String[argList.size()]); + String[] remoteArgs = argList.toArray(new String[0]); PackagedProgram program = new PackagedProgram(jarFile, Lists.newArrayList(), remoteArgs); String savePointPath = confProperties.getProperty(ConfigConstrant.SAVE_POINT_PATH_KEY); - if(StringUtils.isNotBlank(savePointPath)){ + if (StringUtils.isNotBlank(savePointPath)) { String allowNonRestoredState = confProperties.getOrDefault(ConfigConstrant.ALLOW_NON_RESTORED_STATE_KEY, "false").toString(); program.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savePointPath, BooleanUtils.toBoolean(allowNonRestoredState))); } - if(mode.equals(ClusterMode.yarnPer.name())){ + if (mode.equals(ClusterMode.yarnPer.name())) { String flinkConfDir = launcherOptions.getFlinkconf(); Configuration config = StringUtils.isEmpty(flinkConfDir) ? new Configuration() : GlobalConfiguration.loadConfiguration(flinkConfDir); JobGraph jobGraph = PackagedProgramUtils.createJobGraph(program, config, 1); + LOG.info("current jobID is {}", jobGraph.getJobID()); PerJobSubmitter.submit(launcherOptions, jobGraph, config); } else { ClusterClient clusterClient = ClusterClientFactory.createClusterClient(launcherOptions); @@ -116,20 +125,20 @@ public static void main(String[] args) throws Exception { private static String[] parseJson(String[] args) { BufferedReader reader = null; - String lastStr = ""; - try{ + StringBuilder lastStr = new StringBuilder(); + try { FileInputStream fileInputStream = new FileInputStream(args[0]); - InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, "UTF-8"); + InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, StandardCharsets.UTF_8); reader = new BufferedReader(inputStreamReader); - String tempString = null; - while((tempString = reader.readLine()) != null){ - lastStr += tempString; + String tempString; + while ((tempString = reader.readLine()) != null) { + lastStr.append(tempString); } reader.close(); - }catch(IOException e){ + } catch (IOException e) { e.printStackTrace(); - }finally{ - if(reader != null){ + } finally { + if (reader != null) { try { reader.close(); } catch (IOException e) { @@ -137,14 +146,14 @@ private static String[] parseJson(String[] args) { } } } - Map map = JSON.parseObject(lastStr, new TypeReference>(){} ); + Map map = JSON.parseObject(lastStr.toString(), new TypeReference>() { + }); List list = new LinkedList<>(); for (Map.Entry entry : map.entrySet()) { list.add("-" + entry.getKey()); list.add(entry.getValue().toString()); } - String[] array = list.toArray(new String[list.size()]); - return array; + return list.toArray(new String[0]); } } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 0e5089bc9..c40f9432d 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -21,7 +21,6 @@ import com.dtstack.flink.sql.enums.EPluginLoadMode; import com.dtstack.flink.sql.launcher.YarnConfLoader; import com.dtstack.flink.sql.option.Options; -import com.esotericsoftware.minlog.Log; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.cache.DistributedCache; import org.apache.flink.configuration.Configuration; @@ -80,7 +79,8 @@ public void init(String yarnConfDir, Configuration flinkConfig, Properties userC yarnClient.init(yarnConf); yarnClient.start(); - Log.info("----init yarn success ----"); + System.out.println("----init yarn success ----"); +// LOG.info("----init yarn success ----"); } public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJarPath, Options launcherOptions, JobGraph jobGraph) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index ca2d5bdd7..287844c87 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.net.URLDecoder; import java.util.Arrays; import java.util.List; @@ -41,6 +42,7 @@ * per job mode submitter * Date: 2018/11/17 * Company: www.dtstack.com + * * @author xuchao */ @@ -49,15 +51,15 @@ public class PerJobSubmitter { private static final Logger LOG = LoggerFactory.getLogger(PerJobSubmitter.class); public static String submit(Options launcherOptions, JobGraph jobGraph, Configuration flinkConfig) throws Exception { - if (!StringUtils.isBlank(launcherOptions.getAddjar())) { - String addjarPath = URLDecoder.decode(launcherOptions.getAddjar(), Charsets.UTF_8.toString()); - List paths = getJarPaths(addjarPath); - paths.forEach( path -> { - jobGraph.addJar(new Path("file://" + path)); - }); - } - - String confProp = launcherOptions.getConfProp(); + if (!StringUtils.isBlank(launcherOptions.getAddjar())) { + String addjarPath = URLDecoder.decode(launcherOptions.getAddjar(), Charsets.UTF_8.toString()); + List paths = getJarPaths(addjarPath); + paths.forEach(path -> { + jobGraph.addJar(new Path("file://" + path)); + }); + } + + String confProp = launcherOptions.getConfProp(); confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); ClusterSpecification clusterSpecification = FLinkPerJobResourceUtil.createClusterSpecification(confProperties); @@ -67,23 +69,23 @@ public static String submit(Options launcherOptions, JobGraph jobGraph, Configur String flinkJarPath = launcherOptions.getFlinkJarPath(); AbstractYarnClusterDescriptor yarnClusterDescriptor = perJobClusterClientBuilder.createPerJobClusterDescriptor(flinkJarPath, launcherOptions, jobGraph); - ClusterClient clusterClient = yarnClusterDescriptor.deployJobCluster(clusterSpecification, jobGraph,true); + ClusterClient clusterClient = yarnClusterDescriptor.deployJobCluster(clusterSpecification, jobGraph, true); String applicationId = clusterClient.getClusterId().toString(); String flinkJobId = jobGraph.getJobID().toString(); String tips = String.format("deploy per_job with appId: %s, jobId: %s", applicationId, flinkJobId); + System.out.println(tips); LOG.info(tips); return applicationId; } - private static List getJarPaths(String addjarPath) { - if (addjarPath.length() > 2) { - addjarPath = addjarPath.substring(1,addjarPath.length()-1).replace("\"",""); - } - List paths = Arrays.asList(StringUtils.split(addjarPath, ",")); - return paths; - } + private static List getJarPaths(String addjarPath) { + if (addjarPath.length() > 2) { + addjarPath = addjarPath.substring(1, addjarPath.length() - 1).replace("\"", ""); + } + return Arrays.asList(StringUtils.split(addjarPath, ",")); + } } From b251fb96e6e739a4a14023dc1022b1ed3cc56da4 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 31 Mar 2020 22:08:42 +0800 Subject: [PATCH 205/523] deal lateral --- .../flink/sql/side/JoinNodeDealer.java | 4 + .../dtstack/flink/sql/side/SideSQLParser.java | 3 + .../dtstack/flink/sql/util/TableUtils.java | 3 + docs/function.md | 109 ++++++++++++++++++ 4 files changed, 119 insertions(+) create mode 100644 docs/function.md diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index fc027cb70..f072e2591 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -507,6 +507,10 @@ private Set extractFieldFromGroupByList(SqlNodeList parentGroupByList, * @param joinFieldSet */ private void extractJoinField(SqlNode condition, Set> joinFieldSet){ + if (null == condition || condition.getKind() == LITERAL) { + return; + } + SqlKind joinKind = condition.getKind(); if( joinKind == AND || joinKind == EQUALS ){ extractJoinField(((SqlBasicCall)condition).operands[0], joinFieldSet); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index bc9f7a43c..f37c3f78c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -165,6 +165,9 @@ public Object parseSql(SqlNode sqlNode, case ORDER_BY: SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode; parseSql(sqlOrderBy.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + + case LITERAL: + return LITERAL.toString(); } return ""; } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 5a6190210..1527c2db7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -435,6 +435,9 @@ private static void replaceOneSelectField(SqlIdentifier sqlIdentifier, * @param oldTabFieldRefNew */ public static void replaceJoinFieldRefTableName(SqlNode condition, Map oldTabFieldRefNew){ + if (null == condition || condition.getKind() == LITERAL) { + return; + } SqlKind joinKind = condition.getKind(); if( joinKind == AND || joinKind == EQUALS ){ replaceJoinFieldRefTableName(((SqlBasicCall)condition).operands[0], oldTabFieldRefNew); diff --git a/docs/function.md b/docs/function.md new file mode 100644 index 000000000..e272011d2 --- /dev/null +++ b/docs/function.md @@ -0,0 +1,109 @@ +## 支持UDF,UDTF,UDAT： + +### UDTF使用案例 + +1. cross join：左表的每一行数据都会关联上UDTF 产出的每一行数据，如果UDTF不产出任何数据，那么这1行不会输出。 +2. left join：左表的每一行数据都会关联上UDTF 产出的每一行数据，如果UDTF不产出任何数据，则这1行的UDTF的字段会用null值填充。 left join UDTF 语句后面必须接 on true参数。 + + +场景：将某个字段拆分为两个字段。 + +```$xslt + +create table function UDTFOneColumnToMultiColumn with cn.todd.flink180.udflib.UDTFOneColumnToMultiColumn; + +CREATE TABLE MyTable ( + userID VARCHAR , + eventType VARCHAR, + productID VARCHAR) +WITH ( + type = 'kafka11', + bootstrapServers = '172.16.8.107:9092', + zookeeperQuorum = '172.16.8.107:2181/kafka', + offsetReset = 'latest', + topic ='mqTest03', + topicIsPattern = 'false' +); + +CREATE TABLE MyTable1 ( + channel VARCHAR , + pv VARCHAR, + name VARCHAR) +WITH ( + type = 'kafka11', + bootstrapServers = '172.16.8.107:9092', + zookeeperQuorum = '172.16.8.107:2181/kafka', + offsetReset = 'latest', + topic ='mqTest01', + topicIsPattern = 'false' +); + +CREATE TABLE MyTable2 ( + userID VARCHAR, + eventType VARCHAR, + productID VARCHAR, + date1 VARCHAR, + time1 VARCHAR +) +WITH ( + type = 'console', + bootstrapServers = '172.16.8.107:9092', + zookeeperQuorum = '172.16.8.107:2181/kafka', + offsetReset = 'latest', + topic ='mqTest02', + topicIsPattern = 'false' +); + +## 视图使用UDTF +--create view udtf_table as +-- select MyTable.userID,MyTable.eventType,MyTable.productID,date1,time1 + -- from MyTable LEFT JOIN lateral table(UDTFOneColumnToMultiColumn(productID)) + -- as T(date1,time1) on true; + + + + +insert + into + MyTable2 +select + userID,eventType,productID,date1,time1 +from + ( + select MyTable.userID,MyTable.eventType,MyTable.productID,date1,time1 + from MyTable ,lateral table(UDTFOneColumnToMultiColumn(productID)) as T(date1,time1)) as udtf_table; + +``` +一行转多列UDTFOneColumnToMultiColumn + +```$xslt +public class UDTFOneColumnToMultiColumn extends TableFunction { + public void eval(String value) { + String[] valueSplits = value.split("_"); + + //一行，两列 + Row row = new Row(2); + row.setField(0, valueSplits[0]); + row.setField(1, valueSplits[1]); + collect(row); + } + + @Override + public TypeInformation getResultType() { + return new RowTypeInfo(Types.STRING, Types.STRING); + } +} +``` + +输入输出： + + +输入： {"userID": "user_5", "eventType": "browse", "productID":"product_5"} + +输出： + + +--------+-----------+-----------+---------+-------+ + | userID | eventType | productID | date1 | time1 | + +--------+-----------+-----------+---------+-------+ + | user_5 | browse | product_5 | product | 5 | + +--------+-----------+-----------+---------+-------+ \ No newline at end of file From c2b4d8adaf4a667f02a15e6874898f308f934d69 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 1 Apr 2020 14:59:25 +0800 Subject: [PATCH 206/523] kafkaSink opt --- .../sql/sink/kafka/AbstractKafkaSink.java | 148 ++++++++++++++++++ .../flink/sql/sink/kafka/KafkaSink.java | 120 ++------------ .../flink/sql/sink/kafka/KafkaSink.java | 125 ++------------- .../flink/sql/sink/kafka/KafkaSink.java | 125 ++------------- .../flink/sql/sink/kafka/KafkaSink.java | 123 ++------------- 5 files changed, 186 insertions(+), 455 deletions(-) create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java new file mode 100644 index 000000000..7234216a7 --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka; + +import com.dtstack.flink.sql.sink.IStreamSinkGener; +import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.functions.sink.SinkFunction; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; +import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; +import org.apache.kafka.clients.consumer.ConsumerConfig; + +import java.util.Optional; +import java.util.Properties; +import java.util.stream.IntStream; + +/** + * Date: 2020/4/1 + * Company: www.dtstack.com + * @author maqi + */ +public abstract class AbstractKafkaSink implements RetractStreamTableSink, IStreamSinkGener { + + public static final String SINK_OPERATOR_NAME_TPL = "${topic}_${table}"; + + protected String[] fieldNames; + protected TypeInformation[] fieldTypes; + + protected String[] partitionKeys; + protected String sinkOperatorName; + protected Properties properties; + protected int parallelism; + protected String topic; + protected String tableName; + + protected TableSchema schema; + protected SinkFunction kafkaProducer; + + + protected Optional> partitioner; + + protected Properties getKafkaProperties(KafkaSinkTableInfo KafkaSinkTableInfo) { + Properties props = new Properties(); + props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KafkaSinkTableInfo.getBootstrapServers()); + + for (String key : KafkaSinkTableInfo.getKafkaParamKeys()) { + props.setProperty(key, KafkaSinkTableInfo.getKafkaParam(key)); + } + return props; + } + + protected TypeInformation[] getTypeInformations(KafkaSinkTableInfo kafka11SinkTableInfo) { + Class[] fieldClasses = kafka11SinkTableInfo.getFieldClasses(); + TypeInformation[] types = IntStream.range(0, fieldClasses.length) + .mapToObj(i -> TypeInformation.of(fieldClasses[i])) + .toArray(TypeInformation[]::new); + return types; + } + + + protected TableSchema buildTableSchema(String[] fieldNames, TypeInformation[] fieldTypes) { + Preconditions.checkArgument(fieldNames.length == fieldTypes.length, "fieldNames length must equals fieldTypes length !"); + + TableSchema.Builder builder = TableSchema.builder(); + IntStream.range(0, fieldTypes.length) + .forEach(i -> builder.field(fieldNames[i], fieldTypes[i])); + + return builder.build(); + } + + + @Override + public void emitDataStream(DataStream> dataStream) { + DataStream mapDataStream = dataStream + .map((Tuple2 record) -> new CRow(record.f1, record.f0)) + .returns(getRowTypeInfo()) + .setParallelism(parallelism); + + mapDataStream.addSink(kafkaProducer).name(sinkOperatorName); + } + + public CRowTypeInfo getRowTypeInfo() { + return new CRowTypeInfo(new RowTypeInfo(fieldTypes, fieldNames)); + } + + protected String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo) { + if (StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())) { + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } + + @Override + public TupleTypeInfo> getOutputType() { + return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); + } + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } + + @Override + public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + return this; + } + + @Override + public TypeInformation getRecordType() { + return new RowTypeInfo(fieldTypes, fieldNames); + } + + +} diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 71e938ba5..632bb720e 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,25 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.runtime.types.CRowTypeInfo; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -46,108 +29,23 @@ * @create: 2019-11-05 11:45 * @description: **/ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected int parallelism; - - protected Properties properties; - - protected FlinkKafkaProducer flinkKafkaProducer; - protected CRowTypeInfo typeInformation; - - - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - - private String[] partitionKeys; - +public class KafkaSink extends AbstractKafkaSink { @Override public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafkaSinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafkaSinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafkaSinkTableInfo.getBootstrapServers()); - for (String key : kafkaSinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafkaSinkTableInfo.getKafkaParam(key)); - } + Properties kafkaProperties = getKafkaProperties(kafkaSinkTableInfo); + this.tableName = kafkaSinkTableInfo.getName(); + this.topic = kafkaSinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafkaSinkTableInfo); this.fieldNames = kafkaSinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafkaSinkTableInfo.getFields().length]; - for (int i = 0; i < kafkaSinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i) new KafkaProducerFactory() - .createKafkaProducer(kafkaSinkTableInfo, typeInformation, properties, partitioner, partitionKeys); - return this; - } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream - .map((Tuple2 record) -> new CRow(record.f1, record.f0)) - .returns(typeInformation) - .setParallelism(parallelism); - - mapDataStream.addSink(flinkKafkaProducer).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; + this.fieldTypes = getTypeInformations(kafkaSinkTableInfo); + this.schema = buildTableSchema(fieldNames, fieldTypes); + this.parallelism = kafkaSinkTableInfo.getParallelism(); + this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); + this.kafkaProducer = new KafkaProducerFactory().createKafkaProducer(kafkaSinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); return this; } - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index e6dbdf3d3..d22be3d59 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,26 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.runtime.types.CRowTypeInfo; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -49,111 +31,22 @@ * @author DocLi * @modifyer maqi */ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected Properties properties; - - protected FlinkKafkaProducer09 kafkaProducer09; - protected CRowTypeInfo typeInformation; - - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - - private String[] partitionKeys; - - protected int parallelism; - - - +public class KafkaSink extends AbstractKafkaSink{ @Override public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka09SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafka09SinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafka09SinkTableInfo.getBootstrapServers()); - for (String key : kafka09SinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafka09SinkTableInfo.getKafkaParam(key)); - } + Properties kafkaProperties = getKafkaProperties(kafka09SinkTableInfo); + this.tableName = kafka09SinkTableInfo.getName(); + this.topic = kafka09SinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka09SinkTableInfo); this.fieldNames = kafka09SinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafka09SinkTableInfo.getFields().length]; - for (int i = 0; i < kafka09SinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafka09SinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i) new KafkaProducer09Factory() - .createKafkaProducer(kafka09SinkTableInfo, typeInformation, properties, partitioner, partitionKeys); + this.fieldTypes = getTypeInformations(kafka09SinkTableInfo); + this.schema = buildTableSchema(fieldNames, fieldTypes); + this.parallelism = kafka09SinkTableInfo.getParallelism(); + this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); + this.kafkaProducer = new KafkaProducer09Factory().createKafkaProducer(kafka09SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); return this; } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream - .map((Tuple2 record) -> new CRow(record.f1, record.f0)) - .returns(typeInformation) - .setParallelism(parallelism); - - mapDataStream.addSink(kafkaProducer09) - .name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } - } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index ac5a11810..eea78e121 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,23 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.runtime.types.CRowTypeInfo; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -48,110 +33,22 @@ * @modifyer maqi * */ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected Properties properties; - - protected int parallelism; - - protected KafkaSinkTableInfo kafka10SinkTableInfo; - - protected RichSinkFunction kafkaProducer010; - protected CRowTypeInfo typeInformation; - - /** The schema of the table. */ - private TableSchema schema; - - private String[] partitionKeys; - +public class KafkaSink extends AbstractKafkaSink { @Override public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { - this.kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafka10SinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafka10SinkTableInfo.getBootstrapServers()); - - for (String key : kafka10SinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafka10SinkTableInfo.getKafkaParam(key)); - } + KafkaSinkTableInfo kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; + Properties kafkaProperties = getKafkaProperties(kafka10SinkTableInfo); + this.tableName = kafka10SinkTableInfo.getName(); + this.topic = kafka10SinkTableInfo.getTopic(); + this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka10SinkTableInfo); this.fieldNames = kafka10SinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafka10SinkTableInfo.getFields().length]; - for (int i = 0; i < kafka10SinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafka10SinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i()), partitionKeys); - - return this; - } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream - .map((Tuple2 record) -> new CRow(record.f1, record.f0)) - .returns(typeInformation) - .setParallelism(parallelism); - - mapDataStream.addSink(kafkaProducer010).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; + this.fieldTypes = getTypeInformations(kafka10SinkTableInfo); + this.schema = buildTableSchema(fieldNames, fieldTypes); + this.parallelism = kafka10SinkTableInfo.getParallelism(); + this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); + this.kafkaProducer = new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); return this; } - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } - } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 835941ca3..ea45280c7 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,25 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.runtime.types.CRowTypeInfo; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -51,110 +34,22 @@ * @modifyer maqi * */ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected int parallelism; - - protected Properties properties; - - protected FlinkKafkaProducer011 kafkaProducer011; - protected CRowTypeInfo typeInformation; - - - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - private String[] partitionKeys; - - +public class KafkaSink extends AbstractKafkaSink { @Override public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka11SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafka11SinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafka11SinkTableInfo.getBootstrapServers()); - for (String key : kafka11SinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafka11SinkTableInfo.getKafkaParam(key)); - } + Properties kafkaProperties = getKafkaProperties(kafka11SinkTableInfo); + this.tableName = kafka11SinkTableInfo.getName(); + this.topic = kafka11SinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka11SinkTableInfo); this.fieldNames = kafka11SinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafka11SinkTableInfo.getFields().length]; - for (int i = 0; i < kafka11SinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafka11SinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i) new KafkaProducer011Factory() - .createKafkaProducer(kafka11SinkTableInfo, typeInformation, properties, partitioner, partitionKeys); + this.fieldTypes = getTypeInformations(kafka11SinkTableInfo); + this.schema = buildTableSchema(fieldNames, fieldTypes); + this.parallelism = kafka11SinkTableInfo.getParallelism(); + this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); + this.kafkaProducer = new KafkaProducer011Factory().createKafkaProducer(kafka11SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); return this; } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - - DataStream mapDataStream = dataStream - .map((Tuple2 record) -> new CRow(record.f1, record.f0)) - .returns(typeInformation) - .setParallelism(parallelism); - - mapDataStream.addSink(kafkaProducer011).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink>configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } - } From 7676cc7f89355bb7918a2e5fecc08361bc334c08 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 1 Apr 2020 15:49:52 +0800 Subject: [PATCH 207/523] remove unuse --- .../flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java deleted file mode 100644 index e69de29bb..000000000 From a7cff4d5010430db436cbc1a42a4558fd41db300 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 1 Apr 2020 17:58:34 +0800 Subject: [PATCH 208/523] rdb sink check connection --- .../rdb/format/RetractJDBCOutputFormat.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java index 28479f4f6..0f3f1f827 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java @@ -51,7 +51,8 @@ public class RetractJDBCOutputFormat extends DtRichOutputFormat { private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger(RetractJDBCOutputFormat.class); - + private static final int CONNECTION_CHECK_FREQUENCY = 100; + private int checkTimes; private String username; private String password; private String drivername; @@ -181,7 +182,6 @@ public void writeRecord(Tuple2 tuple2) { private void insertWrite(Row row) { - checkConnectionOpen(dbConn); try { if (batchNum == 1) { writeSingleRecord(row); @@ -315,6 +315,7 @@ private void updatePreparedStmt(Row row, PreparedStatement pstmt) throws SQLExce private synchronized void submitExecuteBatch() { try { + checkConnectionOpen(); this.upload.executeBatch(); dbConn.commit(); } catch (SQLException e) { @@ -330,11 +331,17 @@ private synchronized void submitExecuteBatch() { } } - private void checkConnectionOpen(Connection dbConn) { + private void checkConnectionOpen() { + LOG.info("test db connection Valid check !"); + checkTimes++; + if (checkTimes % CONNECTION_CHECK_FREQUENCY != 0) { + return; + } + LOG.warn("db connection Valid check !"); try { - if (dbConn.isClosed()) { + if (dbConn.isClosed() || !dbConn.isValid(100)) { LOG.info("db connection reconnect.."); - dbConn= establishConnection(); + dbConn = establishConnection(); upload = dbConn.prepareStatement(insertQuery); this.dbConn = dbConn; } @@ -345,6 +352,7 @@ private void checkConnectionOpen(Connection dbConn) { } catch (IOException e) { LOG.error("kerberos authentication failed..", e); } + checkTimes = 0; } /** From 1a375e8f801e8ebd61d4d1ae5abc3f59aeef1ac1 Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Thu, 2 Apr 2020 15:34:39 +0800 Subject: [PATCH 209/523] use RestClusterClient --- .../flink/sql/launcher/ClusterClientFactory.java | 13 ++++++------- .../dtstack/flink/sql/launcher/LauncherMain.java | 3 +-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 4c421fa54..9748c5c84 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,6 +26,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.MiniClusterClient; +import org.apache.flink.client.program.rest.RestClusterClient; +import org.apache.flink.client.program.rest.RestClusterClientConfiguration; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; @@ -70,10 +72,7 @@ public static ClusterClient createClusterClient(Options launcherOptions) throws public static ClusterClient createStandaloneClient(Options launcherOptions) throws Exception { String flinkConfDir = launcherOptions.getFlinkconf(); Configuration config = GlobalConfiguration.loadConfiguration(flinkConfDir); - MiniClusterConfiguration.Builder configBuilder = new MiniClusterConfiguration.Builder(); - configBuilder.setConfiguration(config); - MiniCluster miniCluster = new MiniCluster(configBuilder.build()); - MiniClusterClient clusterClient = new MiniClusterClient(config, miniCluster); + RestClusterClient clusterClient = new RestClusterClient<>(config, "clusterClient"); LeaderConnectionInfo connectionInfo = clusterClient.getClusterConnectionInfo(); InetSocketAddress address = AkkaUtils.getInetSocketAddressFromAkkaURL(connectionInfo.getAddress()); config.setString(JobManagerOptions.ADDRESS, address.getAddress().getHostName()); @@ -122,7 +121,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { } catch (Exception e) { throw new RuntimeException(e); } - }else{ + } else { throw new RuntimeException("yarn mode must set param of 'yarnconf'!!!"); } } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 92259c57d..7caf6cd63 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -89,8 +89,7 @@ public static void main(String[] args) throws Exception { confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); -// LOG.info("current mode is {}", mode); - System.out.println("current mode is " + mode); + LOG.info("current mode is {}", mode); if (mode.equals(ClusterMode.local.name())) { String[] localArgs = argList.toArray(new String[0]); From 64c2050759a21e436c7e68acd19b4f0147ffe1f4 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 2 Apr 2020 16:03:33 +0800 Subject: [PATCH 210/523] deal rdb sink error --- .../flink/sql/enums/EConnectionErrorCode.java | 62 +++++++++++++++++++ .../sql/side/EConnectionErrorCodeTest.java | 39 ++++++++++++ .../rdb/format/RetractJDBCOutputFormat.java | 56 ++++++++++------- 3 files changed, 135 insertions(+), 22 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java create mode 100644 core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java b/core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java new file mode 100644 index 000000000..feed69785 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.enums; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.exception.ExceptionUtils; + +import java.util.Arrays; + +/** + * Date: 2020/4/2 + * Company: www.dtstack.com + * @author maqi + */ +public enum EConnectionErrorCode { + ERROR_NOT_MATCH(0, "错误信息未匹配", new String[]{}), + CONN_DB_INVALID(1, "数据库连接失效，请重新打开", new String[]{"the last packet successfully received from the server was"}), + CONN_DB_FAILED(2, "数据库连接失败，请检查用户名或密码是否正确", new String[]{"Access denied for user"}), + DB_TABLE_NOT_EXIST(3, "操作的表名不存在", new String[]{"doesn't exist"}); + + private int code; + private String description; + private String[] baseErrorInfo; + + EConnectionErrorCode(int code, String description, String[] baseErrorInfo) { + this.code = code; + this.description = description; + this.baseErrorInfo = baseErrorInfo; + } + + + public static EConnectionErrorCode resolveErrorCodeFromException(Throwable e) { + final String stackErrorMsg = ExceptionUtils.getFullStackTrace(e); + return Arrays.stream(values()) + .filter(errorCode -> matchKnowError(errorCode, stackErrorMsg)) + .findAny() + .orElse(ERROR_NOT_MATCH); + } + + public static boolean matchKnowError(EConnectionErrorCode errorCode, String errorMsg) { + return Arrays.stream(errorCode.baseErrorInfo) + .filter(baseInfo -> StringUtils.containsIgnoreCase(errorMsg, baseInfo)) + .findAny() + .isPresent(); + } +} diff --git a/core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java b/core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java new file mode 100644 index 000000000..a43b253df --- /dev/null +++ b/core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side; + +import com.dtstack.flink.sql.enums.EConnectionErrorCode; +import org.junit.Assert; +import org.junit.Test; + +/** + * Date: 2020/4/2 + * Company: www.dtstack.com + * @author maqi + */ +public class EConnectionErrorCodeTest { + + @Test + public void testResolveErrorCodeFromException(){ + EConnectionErrorCode errorCode = + EConnectionErrorCode.resolveErrorCodeFromException(new Exception("The last packet successfully received from the server was 179 milliseconds")); + + Assert.assertEquals(errorCode, EConnectionErrorCode.CONN_DB_INVALID); + } +} diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java index 0f3f1f827..22d7814fc 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.format; +import com.dtstack.flink.sql.enums.EConnectionErrorCode; import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.commons.lang3.StringUtils; @@ -181,7 +182,7 @@ public void writeRecord(Tuple2 tuple2) { } - private void insertWrite(Row row) { + private void insertWrite(Row row) { try { if (batchNum == 1) { writeSingleRecord(row); @@ -201,8 +202,6 @@ private void insertWrite(Row row) { } else { outDirtyRecords.inc(batchNum == 1 ? batchNum : rows.size()); } - - } } @@ -213,7 +212,7 @@ private void writeSingleRecord(Row row) { upload.executeUpdate(); dbConn.commit(); } catch (SQLException e) { - + dealSQLException(e); if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed,dirty record num:{}, current row:{}", outDirtyRecords.getCount(), row.toString()); LOG.error("", e); @@ -315,46 +314,59 @@ private void updatePreparedStmt(Row row, PreparedStatement pstmt) throws SQLExce private synchronized void submitExecuteBatch() { try { - checkConnectionOpen(); + regularlyCheckConnection(); this.upload.executeBatch(); dbConn.commit(); } catch (SQLException e) { + LOG.warn("submitExecuteBatch error {}", e); try { dbConn.rollback(); } catch (SQLException e1) { - LOG.error("rollback data error !", e); + dealSQLException(e1); } - rows.forEach(this::writeSingleRecord); } finally { rows.clear(); } } - private void checkConnectionOpen() { - LOG.info("test db connection Valid check !"); + private void dealSQLException(Exception e) { + EConnectionErrorCode errorCode = EConnectionErrorCode.resolveErrorCodeFromException(e); + switch (errorCode) { + case CONN_DB_INVALID: + reconnection(); + break; + case CONN_DB_FAILED: + case DB_TABLE_NOT_EXIST: + throw new RuntimeException(e); + default: + } + } + + + private void regularlyCheckConnection() throws SQLException { checkTimes++; if (checkTimes % CONNECTION_CHECK_FREQUENCY != 0) { return; } LOG.warn("db connection Valid check !"); - try { - if (dbConn.isClosed() || !dbConn.isValid(100)) { - LOG.info("db connection reconnect.."); - dbConn = establishConnection(); - upload = dbConn.prepareStatement(insertQuery); - this.dbConn = dbConn; - } - } catch (SQLException e) { - LOG.error("check connection open failed..", e); - } catch (ClassNotFoundException e) { - LOG.error("load jdbc class error when reconnect db..", e); - } catch (IOException e) { - LOG.error("kerberos authentication failed..", e); + if (dbConn.isClosed() || !dbConn.isValid(100)) { + reconnection(); } checkTimes = 0; } + public void reconnection() throws RuntimeException { + try { + LOG.info("db connection reconnect.."); + dbConn = establishConnection(); + upload = dbConn.prepareStatement(insertQuery); + this.dbConn = dbConn; + } catch (Exception e) { + throw new RuntimeException("connection open failed..", e); + } + } + /** * Executes prepared statement and closes all resources of this instance. * From 75516ad824776ef06ddbe77bd0a75cdb0e990332 Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Thu, 2 Apr 2020 16:14:59 +0800 Subject: [PATCH 211/523] fix applicationId NullPointException --- .../com/dtstack/flink/sql/launcher/ClusterClientFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 9748c5c84..d2b0eb928 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -157,7 +157,7 @@ private static ApplicationId getYarnClusterApplicationId(YarnClient yarnClient) } - if (StringUtils.isEmpty(applicationId.toString())) { + if (applicationId == null || StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); } return applicationId; From 5b49c0a72138ad7ab14e6521c83b1c6bfe4fee76 Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Thu, 2 Apr 2020 20:57:12 +0800 Subject: [PATCH 212/523] fix --- .../java/com/dtstack/flink/sql/launcher/LauncherMain.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 7caf6cd63..da2e881a6 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -112,8 +112,10 @@ public static void main(String[] args) throws Exception { String flinkConfDir = launcherOptions.getFlinkconf(); Configuration config = StringUtils.isEmpty(flinkConfDir) ? new Configuration() : GlobalConfiguration.loadConfiguration(flinkConfDir); JobGraph jobGraph = PackagedProgramUtils.createJobGraph(program, config, 1); + LOG.info("current jobID is {}", jobGraph.getJobID()); - PerJobSubmitter.submit(launcherOptions, jobGraph, config); + + LOG.info("submit applicationId is {}", PerJobSubmitter.submit(launcherOptions, jobGraph, config)); } else { ClusterClient clusterClient = ClusterClientFactory.createClusterClient(launcherOptions); clusterClient.run(program, 1); From 00a4564a484097816de282a05b040689c4b9269e Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Fri, 3 Apr 2020 10:00:18 +0800 Subject: [PATCH 213/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=B9high-availab?= =?UTF-8?q?ility.cluster.id=E7=9A=84=E5=88=A4=E6=96=AD=EF=BC=8C=E4=B8=BA?= =?UTF-8?q?=E7=A9=BA=E5=88=99=E4=B8=BAapplicationId?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/launcher/ClusterClientFactory.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index d2b0eb928..a6f6f3c32 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -44,6 +44,8 @@ import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.StringHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; import java.net.URLDecoder; @@ -59,6 +61,8 @@ */ public class ClusterClientFactory { + private static final Logger LOG = LoggerFactory.getLogger(ClusterClientFactory.class); + public static ClusterClient createClusterClient(Options launcherOptions) throws Exception { String mode = launcherOptions.getMode(); if (mode.equals(ClusterMode.standalone.name())) { @@ -100,6 +104,9 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { String yarnSessionConf = launcherOptions.getYarnSessionConf(); yarnSessionConf = URLDecoder.decode(yarnSessionConf, Charsets.UTF_8.toString()); Properties yarnSessionConfProperties = PluginUtil.jsonStrToObject(yarnSessionConf, Properties.class); + + LOG.info("current yarn config:\n{}", yarnSessionConfProperties); + Object yid = yarnSessionConfProperties.get("yid"); if (null != yid) { @@ -108,12 +115,18 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { applicationId = getYarnClusterApplicationId(yarnClient); } - Log.info("applicationId={}", applicationId.toString()); + LOG.info("applicationId={}", applicationId.toString()); + + if (config.getString("high-availability.cluster-id", null) == null) { + config.setString("high-availability.cluster-id", applicationId.toString()); + } if (StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); } + LOG.info("current config detail:\n{}", config); + AbstractYarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(config, yarnConf, flinkConfDir, yarnClient, false); ClusterClient clusterClient = clusterDescriptor.retrieve(applicationId); clusterClient.setDetached(true); From 3c6c7977a8ce0b85c4d9db7a0a91367e9136bdd4 Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Fri, 3 Apr 2020 10:14:56 +0800 Subject: [PATCH 214/523] fix --- .../dtstack/flink/sql/launcher/ClusterClientFactory.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index a6f6f3c32..77681a603 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -117,14 +117,13 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { LOG.info("applicationId={}", applicationId.toString()); - if (config.getString("high-availability.cluster-id", null) == null) { - config.setString("high-availability.cluster-id", applicationId.toString()); - } - if (StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); } + if (config.getString("high-availability.cluster-id", null) == null) { + config.setString("high-availability.cluster-id", applicationId.toString()); + } LOG.info("current config detail:\n{}", config); AbstractYarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(config, yarnConf, flinkConfDir, yarnClient, false); From f544d1a011155af15fcab52c8b9f24f96e87a773 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 3 Apr 2020 11:17:40 +0800 Subject: [PATCH 215/523] jedis npe check --- .../flink/sql/side/redis/RedisAllReqRow.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 5d09c8f3d..6c82d9902 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -103,6 +103,7 @@ protected void reloadCache() { loadData(newCache); } catch (SQLException e) { LOG.error("", e); + throw new RuntimeException(e); } cacheRef.set(newCache); @@ -160,21 +161,22 @@ private void loadData(Map> tmpCache) throws SQLExcep JedisCommands jedis = null; try { StringBuilder keyPattern = new StringBuilder(tableInfo.getTableName()); - for(String key : tableInfo.getPrimaryKeys()){ + for (String key : tableInfo.getPrimaryKeys()) { keyPattern.append("_").append("*"); - }; + } jedis = getJedisWithRetry(CONN_RETRY_NUM); + if (null == jedis) { + throw new RuntimeException("redis all load data error,get jedis commands error!"); + } Set keys = getRedisKeys(RedisType.parse(tableInfo.getRedisType()), jedis, keyPattern.toString()); - if(CollectionUtils.isEmpty(keys)){ + if (CollectionUtils.isEmpty(keys)) { return; } - for(String key : keys){ + for (String key : keys) { tmpCache.put(key, jedis.hgetAll(key)); } - } catch (Exception e){ - LOG.error("", e); } finally { - if (jedis != null){ + if (jedis != null) { try { ((Closeable) jedis).close(); } catch (IOException e) { From 1e5986949fdfc515cfb1ba2c656ee505007a1fcd Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Fri, 3 Apr 2020 11:44:06 +0800 Subject: [PATCH 216/523] fix --- .../flink/sql/launcher/ClusterClientFactory.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 77681a603..5a67b4e40 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -22,6 +22,7 @@ import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.util.PluginUtil; import com.esotericsoftware.minlog.Log; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.io.Charsets; import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; @@ -63,6 +64,10 @@ public class ClusterClientFactory { private static final Logger LOG = LoggerFactory.getLogger(ClusterClientFactory.class); + private static final String HA_CLUSTER_ID = "high-availability.cluster-id"; + + private static final String HADOOP_CONF = "fs.hdfs.hadoopconf"; + public static ClusterClient createClusterClient(Options launcherOptions) throws Exception { String mode = launcherOptions.getMode(); if (mode.equals(ClusterMode.standalone.name())) { @@ -92,7 +97,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { if (StringUtils.isNotBlank(yarnConfDir)) { try { - config.setString("fs.hdfs.hadoopconf", yarnConfDir); + config.setString(HADOOP_CONF, yarnConfDir); FileSystem.initialize(config); YarnConfiguration yarnConf = YarnConfLoader.getYarnConf(yarnConfDir); @@ -105,8 +110,6 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { yarnSessionConf = URLDecoder.decode(yarnSessionConf, Charsets.UTF_8.toString()); Properties yarnSessionConfProperties = PluginUtil.jsonStrToObject(yarnSessionConf, Properties.class); - LOG.info("current yarn config:\n{}", yarnSessionConfProperties); - Object yid = yarnSessionConfProperties.get("yid"); if (null != yid) { @@ -115,16 +118,15 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { applicationId = getYarnClusterApplicationId(yarnClient); } - LOG.info("applicationId={}", applicationId.toString()); + LOG.info("current applicationId = {}", applicationId.toString()); if (StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); } - if (config.getString("high-availability.cluster-id", null) == null) { - config.setString("high-availability.cluster-id", applicationId.toString()); + if (config.getString(HA_CLUSTER_ID, null) == null) { + config.setString(HA_CLUSTER_ID, applicationId.toString()); } - LOG.info("current config detail:\n{}", config); AbstractYarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(config, yarnConf, flinkConfDir, yarnClient, false); ClusterClient clusterClient = clusterDescriptor.retrieve(applicationId); From fa77a64801cd475027907b941c26cf30579d358e Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Fri, 3 Apr 2020 11:53:58 +0800 Subject: [PATCH 217/523] add log --- .../sql/launcher/ClusterClientFactory.java | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 5a67b4e40..e7ae98d73 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -21,21 +21,15 @@ import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.util.PluginUtil; -import com.esotericsoftware.minlog.Log; -import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.io.Charsets; import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.client.program.MiniClusterClient; import org.apache.flink.client.program.rest.RestClusterClient; -import org.apache.flink.client.program.rest.RestClusterClientConfiguration; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.runtime.akka.AkkaUtils; -import org.apache.flink.runtime.minicluster.MiniCluster; -import org.apache.flink.runtime.minicluster.MiniClusterConfiguration; import org.apache.flink.runtime.util.LeaderConnectionInfo; import org.apache.flink.yarn.AbstractYarnClusterDescriptor; import org.apache.flink.yarn.YarnClusterDescriptor; @@ -50,12 +44,7 @@ import java.net.InetSocketAddress; import java.net.URLDecoder; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Properties; -import java.util.Set; +import java.util.*; /** * @author sishu.yss @@ -128,6 +117,11 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { config.setString(HA_CLUSTER_ID, applicationId.toString()); } + LOG.info("------------config params-------------------------"); + config.toMap().forEach((key, value) -> LOG.info("{}: {}", key, value)); + LOG.info("-------------------------------------------"); + + AbstractYarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(config, yarnConf, flinkConfDir, yarnClient, false); ClusterClient clusterClient = clusterDescriptor.retrieve(applicationId); clusterClient.setDetached(true); From 417b1a7e1ae03e94d00b2067e282e2e229bf0034 Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Fri, 3 Apr 2020 11:54:55 +0800 Subject: [PATCH 218/523] add log --- .../com/dtstack/flink/sql/launcher/ClusterClientFactory.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index e7ae98d73..79991f9a4 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -70,6 +70,11 @@ public static ClusterClient createClusterClient(Options launcherOptions) throws public static ClusterClient createStandaloneClient(Options launcherOptions) throws Exception { String flinkConfDir = launcherOptions.getFlinkconf(); Configuration config = GlobalConfiguration.loadConfiguration(flinkConfDir); + + LOG.info("------------config params-------------------------"); + config.toMap().forEach((key, value) -> LOG.info("{}: {}", key, value)); + LOG.info("-------------------------------------------"); + RestClusterClient clusterClient = new RestClusterClient<>(config, "clusterClient"); LeaderConnectionInfo connectionInfo = clusterClient.getClusterConnectionInfo(); InetSocketAddress address = AkkaUtils.getInetSocketAddressFromAkkaURL(connectionInfo.getAddress()); From 40fb964f7cc3856cded3b20224756fdd6c7a535a Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Fri, 3 Apr 2020 14:23:04 +0800 Subject: [PATCH 219/523] fix --- .../com/dtstack/flink/sql/launcher/ClusterClientFactory.java | 1 - 1 file changed, 1 deletion(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 79991f9a4..1da0db7a3 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -126,7 +126,6 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { config.toMap().forEach((key, value) -> LOG.info("{}: {}", key, value)); LOG.info("-------------------------------------------"); - AbstractYarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(config, yarnConf, flinkConfDir, yarnClient, false); ClusterClient clusterClient = clusterDescriptor.retrieve(applicationId); clusterClient.setDetached(true); From 304f18061944f240b30e7b1c456ce85a9e28cc72 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 3 Apr 2020 14:49:18 +0800 Subject: [PATCH 220/523] code opt --- .../side/cassandra/CassandraAllReqRow.java | 2 +- .../environment/MyLocalStreamEnvironment.java | 10 ++-- .../flink/sql/side/AbstractSideTableInfo.java | 4 +- .../dtstack/flink/sql/side/BaseAllReqRow.java | 2 + .../com/dtstack/flink/sql/util/DateUtil.java | 53 +++++++++---------- .../com/dtstack/flink/sql/util/JDBCUtils.java | 10 +--- .../Elasticsearch6AllReqRow.java | 2 +- .../sql/side/elasticsearch6/util/Es6Util.java | 7 --- .../side/elasticsearch6/util/MathUtil.java | 2 +- .../flink/sql/side/hbase/RowKeyBuilder.java | 5 +- .../sink/kafka/CustomerFlinkPartition.java | 1 + .../sql/launcher/ClusterClientFactory.java | 2 +- .../flink/sql/launcher/LauncherMain.java | 34 +++--------- .../flink/sql/side/mongo/MongoAllReqRow.java | 10 ++-- .../flink/sql/side/redis/RedisAllReqRow.java | 6 +-- 15 files changed, 60 insertions(+), 90 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index 3ad8b934e..5f642d176 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -267,7 +267,7 @@ private void loadData(Map>> tmpCache) throws SQ String connInfo = "address:" + tableInfo.getAddress() + ";userName:" + tableInfo.getUserName() + ",pwd:" + tableInfo.getPassword(); LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); - Thread.sleep(5 * 1000); + Thread.sleep(LOAD_DATA_ERROR_SLEEP_TIME); } catch (InterruptedException e1) { LOG.error("", e1); } diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java index 8ac1edd41..a8728c595 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java @@ -113,14 +113,16 @@ public JobExecutionResult execute(String jobName) throws Exception { LOG.info("Running job on local embedded Flink mini cluster"); } - MiniCluster exec = new MiniCluster(configBuilder.build()); + MiniCluster exec = null; try { + exec = new MiniCluster(configBuilder.build()); exec.start(); return exec.executeJobBlocking(jobGraph); - } - finally { + } finally { transformations.clear(); - exec.closeAsync(); + if (null != exec) { + exec.closeAsync(); + } } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index 7a832d0a1..9e114fe87 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -57,11 +57,11 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements public static final String ASYNC_REQ_POOL_KEY = "asyncPoolSize"; - private String cacheType = "none";//None or LRU or ALL + private String cacheType = "none"; private int cacheSize = 10000; - private long cacheTimeout = 60 * 1000;// + private long cacheTimeout = 60_000L; private int asyncCapacity=100; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java index 3e3d0a8fb..ad9dcf1bd 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java @@ -47,6 +47,8 @@ public abstract class BaseAllReqRow extends RichFlatMapFunction impl private static final Logger LOG = LoggerFactory.getLogger(BaseAllReqRow.class); + public static final long LOAD_DATA_ERROR_SLEEP_TIME = 5_000L; + protected BaseSideInfo sideInfo; private ScheduledExecutorService es; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index 409602baa..a3bb949a8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -75,7 +75,6 @@ public static Date stringToDate(String strDate) { return null; } try { - ; return localDateTimetoDate(LocalDateTime.parse(strDate, DATE_TIME_FORMATTER)); } catch (DateTimeParseException ignored) { } @@ -113,13 +112,13 @@ public static long getTodayStart(long day) { if (("" + day).length() > 10) { cal.setTime(new Date(day)); } else { - cal.setTime(new Date(day * 1000)); + cal.setTime(new Date(day * 1000L)); } cal.set(Calendar.HOUR_OF_DAY, 0); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - firstDay = cal.getTimeInMillis() / 1000; + firstDay = cal.getTimeInMillis() / 1000L; return firstDay; } @@ -131,7 +130,7 @@ public static long getTodayStart(long day) { */ public static long getTodayStart(long day,String scope) { if("MS".equals(scope)){ - return getTodayStart(day)*1000; + return getTodayStart(day)*1000L; }else if("S".equals(scope)){ return getTodayStart(day); }else{ @@ -151,13 +150,13 @@ public static long getNextDayStart(long day) { if (("" + day).length() > 10) { cal.setTime(new Date(day)); } else { - cal.setTime(new Date(day * 1000)); + cal.setTime(new Date(day * 1000L)); } cal.set(Calendar.HOUR_OF_DAY, 0); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - nextDay = (cal.getTimeInMillis() + daySpanMill) / 1000; + nextDay = (cal.getTimeInMillis() + daySpanMill) / 1000L; return nextDay; } @@ -169,7 +168,7 @@ public static long getNextDayStart(long day) { */ public static long getNextDayStart(long day,String scope) { if("MS".equals(scope)){ - return getNextDayStart(day)*1000; + return getNextDayStart(day)*1000L; }else if("S".equals(scope)){ return getNextDayStart(day); }else{ @@ -186,13 +185,13 @@ public static long getNextDayStart(long day,String scope) { public static long getMonthFirst(long day) { long firstDay = 0L; Calendar cal = Calendar.getInstance(); - cal.setTime(new Date(day * 1000)); + cal.setTime(new Date(day * 1000L)); cal.set(Calendar.DAY_OF_MONTH, 1); cal.set(Calendar.HOUR_OF_DAY, 0); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - firstDay = cal.getTimeInMillis() / 1000; + firstDay = cal.getTimeInMillis() / 1000L; return firstDay; } @@ -224,7 +223,7 @@ public static int getYear(long day) { public static long getWeekFirst(long day) { long firstDay = 0L; Calendar cal = Calendar.getInstance(); - cal.setTime(new Date(day * 1000)); + cal.setTime(new Date(day * 1000L)); cal.setFirstDayOfWeek(Calendar.MONDAY); cal.set(Calendar.DAY_OF_WEEK, Calendar.MONDAY); cal.set(Calendar.HOUR_OF_DAY, 0); @@ -243,7 +242,7 @@ public static long getWeekFirst(long day) { */ public static int getWeekOfYear(long day) { Calendar cal = Calendar.getInstance(); - cal.setTime(new Date(day * 1000)); + cal.setTime(new Date(day * 1000L)); return cal.get(Calendar.WEEK_OF_YEAR); } @@ -363,7 +362,7 @@ public static long getDateMillToFormat(String day, String inFormat) throws Parse Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); calendar.setTime(date); - return calendar.getTimeInMillis()/1000; + return calendar.getTimeInMillis()/1000L; } /** @@ -383,7 +382,7 @@ public static long getFirstDay4Month(int year, int month) { cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - firstDay = cal.getTimeInMillis() / 1000; + firstDay = cal.getTimeInMillis() / 1000L; return firstDay; } @@ -405,7 +404,7 @@ public static long getLastDay4Month(int year, int month) { cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - lastDay = cal.getTimeInMillis() / 1000; + lastDay = cal.getTimeInMillis() / 1000L; return lastDay; } @@ -444,7 +443,7 @@ public static long getMillByOneDay() { cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - return cal.getTimeInMillis() / 1000; + return cal.getTimeInMillis() / 1000L; } /** @@ -458,7 +457,7 @@ public static long getMillByYesDay() { cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - return cal.getTimeInMillis() / 1000; + return cal.getTimeInMillis() / 1000L; } /** @@ -472,7 +471,7 @@ public static long getMillByLastWeekDay() { cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - return cal.getTimeInMillis() / 1000; + return cal.getTimeInMillis() / 1000L; } /** @@ -534,7 +533,7 @@ public static long getMillToDay(Calendar cal,int dateT){ cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - return cal.getTimeInMillis()/1000; + return cal.getTimeInMillis()/1000L; } /** @@ -563,7 +562,7 @@ public static String getDate(long day, String format) { if (("" + day).length() > 10) { cal.setTime(new Date(day)); } else { - cal.setTime(new Date(day * 1000)); + cal.setTime(new Date(day * 1000L)); } SimpleDateFormat sf = new SimpleDateFormat(format); return sf.format(cal.getTime()); @@ -619,7 +618,7 @@ public static Date stringToDate(String day, String format) { */ public static String longToString(long day, String format) throws ParseException { if (("" + day).length() <= 10){ - day=day*1000; + day=day*1000L; } SimpleDateFormat dateFormat = new SimpleDateFormat(format); String date = dateFormat.format(day); @@ -634,7 +633,7 @@ public static String longToString(long day, String format) throws ParseException */ public static int getMinusDate(int day, int minusDay) { Calendar cal = Calendar.getInstance(); - cal.setTime(new Date(day * 1000)); + cal.setTime(new Date(day * 1000L)); cal.set(Calendar.DATE, cal.get(Calendar.DATE) - minusDay); cal.set(Calendar.HOUR_OF_DAY, 0); cal.set(Calendar.MINUTE, 0); @@ -663,7 +662,7 @@ public static int getWeeksBetweenTwoDates(long startDay, long endDay) { public static int getMaxWeekOfYear(long startDay) { Calendar cal = Calendar.getInstance(); - cal.setTime(new Date(startDay * 1000)); + cal.setTime(new Date(startDay * 1000L)); return cal.getMaximum(Calendar.WEEK_OF_YEAR); } @@ -696,7 +695,7 @@ public static long getMinuteStart(long time) { if (("" + time).length() > 10) { cal.setTime(new Date(time)); } else { - cal.setTime(new Date(time * 1000)); + cal.setTime(new Date(time * 1000L)); } cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); @@ -719,7 +718,7 @@ public static long getHourStart(long time) { cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); cal.set(Calendar.MINUTE, 0); - firstDay = cal.getTimeInMillis() / 1000; + firstDay = cal.getTimeInMillis() / 1000L; return firstDay; } @@ -757,7 +756,7 @@ public static java.sql.Timestamp columnToTimestamp(Object column) { if (column == null) { return null; } else if(column instanceof String) { - return new java.sql.Timestamp(stringToDate((String)column).getTime()); + return null == stringToDate((String) column) ? null : new java.sql.Timestamp(stringToDate((String) column).getTime()); } else if (column instanceof Integer) { Integer rawData = (Integer) column; return new java.sql.Timestamp(rawData.longValue()); @@ -791,7 +790,7 @@ public static Timestamp getTimestampFromStr(String timeStr) { Instant instant = Instant.from(ISO_INSTANT.parse(timeStr)); return new Timestamp(instant.getEpochSecond() * MILLIS_PER_SECOND); } - return new Timestamp(stringToDate(timeStr).getTime()); + return null == stringToDate(timeStr) ? null : new Timestamp(stringToDate(timeStr).getTime()); } public static java.sql.Date getDateFromStr(String dateStr) { @@ -803,7 +802,7 @@ public static java.sql.Date getDateFromStr(String dateStr) { Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); return new java.sql.Date(instant.toEpochMilli()); } - return new java.sql.Date(stringToDate(dateStr).getTime()); + return null == stringToDate(dateStr) ? null : new java.sql.Date(stringToDate(dateStr).getTime()); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java index 3d66d8a6d..faab22172 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java @@ -19,19 +19,13 @@ package com.dtstack.flink.sql.util; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.sql.DriverManager; public class JDBCUtils { - - private static final Logger LOG = LoggerFactory.getLogger(ClassUtil.class); - - public final static String LOCK_STR = "jdbc_lock_str"; + private static final Object LOCK = new Object(); public static void forName(String clazz, ClassLoader classLoader) { - synchronized (LOCK_STR){ + synchronized (LOCK){ try { Class.forName(clazz, true, classLoader); DriverManager.setLoginTimeout(10); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index b8cf2c46a..e50ed5edc 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -187,7 +187,7 @@ private void loadData(Map>> tmpCache) throws IO try { String connInfo = "url: " + tableInfo.getAddress() + "; userName: " + tableInfo.getUserName() + ", pwd:" + tableInfo.getPassword(); LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); - Thread.sleep(5 * 1000); + Thread.sleep(LOAD_DATA_ERROR_SLEEP_TIME); } catch (InterruptedException e1) { LOG.error("", e1); } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java index 3d9e4d62a..bc5637515 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java @@ -176,13 +176,6 @@ public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBu case "NOT_EQUALS": return boolQueryBuilder.mustNot(QueryBuilders.termQuery(textConvertToKeyword(info.getFieldName(), sideInfo), removeSpaceAndApostrophe(info.getCondition())[0])); default: - try { - throw new Exception("elasticsearch6 does not support this operation: " + info.getOperatorKind()); - } catch (Exception e) { - - e.printStackTrace(); - LOG.error(e.getMessage()); - } return boolQueryBuilder; } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java index d6af042f4..d72530a56 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java @@ -216,7 +216,7 @@ public static BigDecimal getBigDecimal(Object obj) { } else if (obj instanceof BigInteger) { return new BigDecimal((BigInteger) obj); } else if (obj instanceof Number) { - return new BigDecimal(((Number) obj).doubleValue()); + return BigDecimal.valueOf(((Number) obj).doubleValue()); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to BigDecimal."); } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java index b8530f224..4b3f35b68 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java @@ -120,9 +120,10 @@ public ReplaceInfo getReplaceInfo(String field){ } private List makeFormula(String formula){ - if(formula == null || formula.length() <= 0){ - Lists.newArrayList(); + if (formula == null || formula.length() <= 0) { + return Lists.newArrayList(); } + List result = Lists.newArrayList(); for(String meta: splitIgnoreQuotaBrackets(formula, "\\+")){ Matcher matcher = Md5Operator.matcher(meta.trim()); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index 90dfe996b..d8028fcac 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -20,6 +20,7 @@ public void open(int parallelInstanceId, int parallelInstances) { @Override public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); + if(key == null){ return partitions[this.parallelInstanceId % partitions.length]; } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 8f0090db5..11b7dec34 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -158,7 +158,7 @@ private static ApplicationId getYarnClusterApplicationId(YarnClient yarnClient) } - if (StringUtils.isEmpty(applicationId.toString())) { + if (null == applicationId) { throw new RuntimeException("No flink session found on yarn cluster."); } return applicationId; diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 0b036c87c..c39d8b8a9 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -22,6 +22,7 @@ import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; import com.dtstack.flink.sql.constrant.ConfigConstrant; +import com.dtstack.flink.sql.launcher.perjob.PerJobClusterClientBuilder; import com.google.common.collect.Lists; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.TypeReference; @@ -41,6 +42,9 @@ import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; +import org.apache.flink.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.File; @@ -64,7 +68,6 @@ public class LauncherMain { private static String SP = File.separator; - private static String getLocalCoreJarPath(String localSqlRootJar) throws Exception { String jarPath = PluginUtil.getCoreJarFileName(localSqlRootJar, CORE_JAR); String corePath = localSqlRootJar + SP + jarPath; @@ -114,32 +117,11 @@ public static void main(String[] args) throws Exception { } - private static String[] parseJson(String[] args) { - BufferedReader reader = null; - String lastStr = ""; - try{ - FileInputStream fileInputStream = new FileInputStream(args[0]); - InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, "UTF-8"); - reader = new BufferedReader(inputStreamReader); - String tempString = null; - while((tempString = reader.readLine()) != null){ - lastStr += tempString; - } - reader.close(); - }catch(IOException e){ - e.printStackTrace(); - }finally{ - if(reader != null){ - try { - reader.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - Map map = JSON.parseObject(lastStr, new TypeReference>(){} ); + private static String[] parseJson(String[] args) throws IOException { + String lastStr = FileUtils.readFileUtf8(new File(args[0])); + Map map = JSON.parseObject(lastStr, new TypeReference>() { + }); List list = new LinkedList<>(); - for (Map.Entry entry : map.entrySet()) { list.add("-" + entry.getKey()); list.add(entry.getValue().toString()); diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index 3373f3667..c7fa2398e 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -205,7 +205,7 @@ private void loadData(Map>> tmpCache) throws SQ try { String connInfo = "url:" + tableInfo.getAddress() + ";userName:" + tableInfo.getUserName() + ",pwd:" + tableInfo.getPassword(); LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); - Thread.sleep(5 * 1000); + Thread.sleep(LOAD_DATA_ERROR_SLEEP_TIME); } catch (InterruptedException e1) { LOG.error("", e1); } @@ -248,12 +248,8 @@ private void loadData(Map>> tmpCache) throws SQ } catch (Exception e) { LOG.error("", e); } finally { - try { - if (mongoClient != null) { - mongoClient.close(); - } - } catch (Exception e) { - throw new RuntimeException("[closeMongoDB]:" + e.getMessage()); + if (mongoClient != null) { + mongoClient.close(); } } } diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 6c82d9902..bdee79f07 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -239,17 +239,17 @@ private JedisCommands getJedis(RedisSideTableInfo tableInfo) { } private JedisCommands getJedisWithRetry(int retryNum) { - while (retryNum-- > 0){ + while (retryNum-- > 0) { try { return getJedis(tableInfo); } catch (Exception e) { - if(retryNum <= 0){ + if (retryNum <= 0) { throw new RuntimeException("getJedisWithRetry error", e); } try { String jedisInfo = "url:" + tableInfo.getUrl() + ",pwd:" + tableInfo.getPassword() + ",database:" + tableInfo.getDatabase(); LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + jedisInfo); - Thread.sleep(5 * 1000); + Thread.sleep(LOAD_DATA_ERROR_SLEEP_TIME); } catch (InterruptedException e1) { LOG.error("", e1); } From c27194e6829d5a77c360db353319deefc171ebc0 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 3 Apr 2020 15:37:57 +0800 Subject: [PATCH 221/523] code opt --- .../format/SerializationMetricWrapper.java | 1 - .../flink/sql/option/OptionParser.java | 2 -- .../sql/parser/CreateTmpTableParser.java | 6 ++++- .../flink/sql/parser/InsertSqlParser.java | 9 ++++++- .../com/dtstack/flink/sql/side/JoinInfo.java | 1 - .../com/dtstack/flink/sql/util/DateUtil.java | 24 +++++++++++++++---- .../com/dtstack/flink/sql/util/MathUtil.java | 12 ---------- .../dtstack/flink/sql/util/ParseUtils.java | 13 +++++++--- .../sql/watermarker/WaterMarkerAssigner.java | 2 -- .../sink/elasticsearch/CustomerSinkFunc.java | 1 - .../flink/sql/side/hbase/HbaseAllReqRow.java | 17 ++++++++++--- .../CustomerKeyedSerializationSchema.java | 2 -- .../AvroCRowSerializationSchema.java | 2 +- .../CsvCRowSerializationSchema.java | 7 ++---- .../kafka/table/KafkaSourceTableInfo.java | 1 - .../flink/sql/sink/kafka/KafkaProducer09.java | 1 - .../sink/kafka/KafkaProducer09Factory.java | 1 - .../flink/sql/source/kafka/KafkaSource.java | 3 --- .../sink/kafka/KafkaProducer010Factory.java | 1 - .../sink/kafka/KafkaProducer011Factory.java | 1 - .../flink/sql/launcher/LauncherMain.java | 7 ------ .../perjob/PerJobClusterClientBuilder.java | 2 -- .../flink/sql/side/mongo/MongoAllReqRow.java | 2 -- .../sql/side/mongo/MongoAsyncReqRow.java | 7 ------ .../sql/sink/mongo/MongoOutputFormat.java | 2 -- .../sink/postgresql/PostgresqlDialect.java | 1 - .../sql/sink/rdb/JDBCTypeConvertUtils.java | 1 - .../flink/sql/side/redis/RedisAllReqRow.java | 1 - .../side/sqlserver/SqlserverAsyncReqRow.java | 4 +++- 29 files changed, 62 insertions(+), 72 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java index 8802198a0..fda394af7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java @@ -25,7 +25,6 @@ import org.apache.flink.metrics.Meter; import org.apache.flink.metrics.MeterView; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; /** diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java index 0fd057c5a..47416cd1b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java @@ -29,9 +29,7 @@ import java.util.List; import java.util.Map; import java.io.File; -import java.io.FileInputStream; import java.net.URLEncoder; -import java.util.stream.Stream; import org.apache.commons.codec.Charsets; import org.apache.flink.util.FileUtils; diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java index de7141eb5..443e83816 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java @@ -22,7 +22,11 @@ import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.calcite.config.Lex; -import org.apache.calcite.sql.*; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import com.google.common.collect.Lists; diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java index a7c6db9eb..479aecc5c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java @@ -21,7 +21,14 @@ package com.dtstack.flink.sql.parser; import org.apache.calcite.config.Lex; -import org.apache.calcite.sql.*; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlInsert; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlMatchRecognize; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOrderBy; +import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import org.apache.commons.lang3.StringUtils; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java index 8a8fe21f6..40b21ef03 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java @@ -25,7 +25,6 @@ import com.google.common.base.Strings; import java.io.Serializable; -import java.util.Map; /** * Join信息 diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index a3bb949a8..a420c9d82 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -23,10 +23,21 @@ import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.*; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; -import java.util.*; + +import java.util.Calendar; +import java.util.Date; +import java.util.Locale; +import java.util.SimpleTimeZone; +import java.util.TimeZone; import java.util.regex.Pattern; import static java.time.format.DateTimeFormatter.ISO_INSTANT; @@ -756,7 +767,8 @@ public static java.sql.Timestamp columnToTimestamp(Object column) { if (column == null) { return null; } else if(column instanceof String) { - return null == stringToDate((String) column) ? null : new java.sql.Timestamp(stringToDate((String) column).getTime()); + Date date = stringToDate((String) column); + return null == date ? null : new java.sql.Timestamp(date.getTime()); } else if (column instanceof Integer) { Integer rawData = (Integer) column; return new java.sql.Timestamp(rawData.longValue()); @@ -790,7 +802,8 @@ public static Timestamp getTimestampFromStr(String timeStr) { Instant instant = Instant.from(ISO_INSTANT.parse(timeStr)); return new Timestamp(instant.getEpochSecond() * MILLIS_PER_SECOND); } - return null == stringToDate(timeStr) ? null : new Timestamp(stringToDate(timeStr).getTime()); + Date date = stringToDate(timeStr); + return null == date ? null : new Timestamp(date.getTime()); } public static java.sql.Date getDateFromStr(String dateStr) { @@ -802,7 +815,8 @@ public static java.sql.Date getDateFromStr(String dateStr) { Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); return new java.sql.Date(instant.toEpochMilli()); } - return null == stringToDate(dateStr) ? null : new java.sql.Date(stringToDate(dateStr).getTime()); + Date date = stringToDate(dateStr); + return null == date ? null : new java.sql.Date(date.getTime()); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index 5aaa40a15..c5584b5d8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -23,18 +23,6 @@ import java.math.BigInteger; import java.sql.Date; import java.sql.Timestamp; -import java.text.ParseException; -import java.text.SimpleDateFormat; - - -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.ZoneOffset; -import java.util.TimeZone; -import java.util.regex.Pattern; - -import static java.time.format.DateTimeFormatter.ISO_INSTANT; /** * Convert val to specified numeric type diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java index d399b533c..dfbab87a7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java @@ -37,16 +37,23 @@ package com.dtstack.flink.sql.util; import com.google.common.collect.HashBasedTable; -import org.apache.calcite.sql.*; + +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import java.util.Arrays; import java.util.List; -import java.util.Map; import java.util.Set; -import static org.apache.calcite.sql.SqlKind.*; +import static org.apache.calcite.sql.SqlKind.AS; +import static org.apache.calcite.sql.SqlKind.IDENTIFIER; +import static org.apache.calcite.sql.SqlKind.JOIN; + /** * @Auther: jiangjunjie diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java index b0ca0335f..ffce0c646 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java @@ -29,8 +29,6 @@ import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; import java.sql.Timestamp; -import java.lang.Long; - /** * define watermarker * Date: 2018/6/29 diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index 0d97f5995..23213e721 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -32,7 +32,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index ed2931fff..63d26d5dd 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -20,7 +20,10 @@ package com.dtstack.flink.sql.side.hbase; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAllReqRow; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.map.HashedMap; @@ -34,7 +37,12 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +50,10 @@ import java.io.IOException; import java.sql.SQLException; import java.sql.Timestamp; -import java.util.*; +import java.util.Calendar; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; public class HbaseAllReqRow extends BaseAllReqRow { diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index cde8d1b1d..78d316c99 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -4,12 +4,10 @@ import com.dtstack.flink.sql.format.SerializationMetricWrapper; import com.dtstack.flink.sql.sink.kafka.serialization.JsonCRowSerializationSchema; import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.formats.json.JsonRowSerializationSchema; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java index 692e208b5..5f852832b 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java @@ -53,7 +53,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.TimeZone; import java.util.stream.Collectors; @@ -287,6 +286,7 @@ private Object convertFlinkType(Schema schema, Object object) { case DOUBLE: case BOOLEAN: return object; + default: } throw new RuntimeException("Unsupported Avro type:" + schema); } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java index 4e57b6f2a..4e7c727d7 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java @@ -47,10 +47,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.util.Arrays; -import java.util.Iterator; import java.util.Objects; -import java.util.stream.IntStream; -import java.util.stream.Stream; /** * Serialization schema that serializes an object of Flink types into a CSV bytes. @@ -132,9 +129,9 @@ public Builder setFieldDelimiter(char c) { public Builder setLineDelimiter(String delimiter) { Preconditions.checkNotNull(delimiter, "Delimiter must not be null."); - if (!delimiter.equals("\n") && !delimiter.equals("\r") && !delimiter.equals("\r\n")) { + if (!("\n".equals(delimiter)) && !("\r".equals(delimiter)) && !("\r\n".equals(delimiter))) { throw new IllegalArgumentException( - "Unsupported new line delimiter. Only \\n, \\r, or \\r\\n are supported."); + "Unsupported new line delimiter. Only \\n, \\r, or \\r\\n are supported."); } this.csvSchema = this.csvSchema.rebuild().setLineSeparator(delimiter).build(); return this; diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index c27eee376..1210aba40 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -19,7 +19,6 @@ package com.dtstack.flink.sql.source.kafka.table; -import com.dtstack.flink.sql.format.FormatType; import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.google.common.base.Preconditions; diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java index bee1865dd..19e0feddb 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java @@ -24,7 +24,6 @@ import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java index ee3423b07..b64727ee3 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java @@ -23,7 +23,6 @@ import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 9f8917761..01c67cd56 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -21,18 +21,15 @@ import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; import com.dtstack.flink.sql.table.AbstractSourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer09; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; -import java.util.Map; import java.util.Properties; /** diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java index e0e023586..eec398de0 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java @@ -23,7 +23,6 @@ import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java index 0cb11da82..b1cc53fb7 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java @@ -23,7 +23,6 @@ import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index c39d8b8a9..551a00533 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -20,9 +20,7 @@ package com.dtstack.flink.sql.launcher; -import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; import com.dtstack.flink.sql.constrant.ConfigConstrant; -import com.dtstack.flink.sql.launcher.perjob.PerJobClusterClientBuilder; import com.google.common.collect.Lists; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.TypeReference; @@ -43,14 +41,9 @@ import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.util.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.BufferedReader; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStreamReader; import java.net.URLDecoder; import java.util.LinkedList; import java.util.List; diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 0e5089bc9..5dc74900e 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -32,14 +32,12 @@ import org.apache.flink.yarn.AbstractYarnClusterDescriptor; import org.apache.flink.yarn.YarnClusterDescriptor; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index c7fa2398e..c1a32cb64 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -37,8 +37,6 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index d85a9990d..a1cf94000 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -44,13 +44,6 @@ import com.mongodb.async.client.MongoClients; import com.mongodb.async.client.MongoCollection; import com.mongodb.async.client.MongoDatabase; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index 3f1f9f7e2..37f7f5952 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -35,8 +35,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; /** * Reason: diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java index 58e92c143..1d775adac 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java @@ -20,7 +20,6 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; -import com.dtstack.flink.sql.util.DtStringUtil; import java.util.Arrays; import java.util.Optional; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java index 28811d2fd..a41cad5ef 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java @@ -25,7 +25,6 @@ import java.math.BigDecimal; import java.sql.Date; import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; import java.sql.Types; diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index bdee79f07..8e6a08791 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -22,7 +22,6 @@ import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.redis.enums.RedisType; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index a63d28607..1f4fbf13a 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -19,7 +19,9 @@ package com.dtstack.flink.sql.side.sqlserver; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; From 22ac02a84f0c9dfe61b2635ab6cdd15ec47030cc Mon Sep 17 00:00:00 2001 From: xuchao Date: Fri, 3 Apr 2020 20:30:43 +0800 Subject: [PATCH 222/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/exec/ExecuteProcessHelper.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 058ef6097..823cd5143 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.exec; +import com.dtstack.flink.sql.parser.*; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -27,11 +28,11 @@ import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.calcite.FlinkPlannerImpl; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.config.CalciteConfig; import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.enums.EPluginLoadMode; @@ -40,11 +41,6 @@ import com.dtstack.flink.sql.function.FunctionManager; import com.dtstack.flink.sql.option.OptionParser; import com.dtstack.flink.sql.option.Options; -import com.dtstack.flink.sql.parser.CreateFuncParser; -import com.dtstack.flink.sql.parser.CreateTmpTableParser; -import com.dtstack.flink.sql.parser.InsertSqlParser; -import com.dtstack.flink.sql.parser.SqlParser; -import com.dtstack.flink.sql.parser.SqlTree; import com.dtstack.flink.sql.side.SideSqlExec; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.sink.StreamSinkFactory; @@ -210,7 +206,8 @@ private static void sqlTranslation(String localSqlPluginPath, CreateTmpTableParser.SqlParserResult tmp = sqlTree.getTmpTableMap().get(tableName); String realSql = DtStringUtil.replaceIgnoreQuota(result.getExecSql(), "`", ""); - SqlNode sqlNode = org.apache.calcite.sql.parser.SqlParser.create(realSql, CalciteConfig.MYSQL_LEX_CONFIG).parseStmt(); + FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); + SqlNode sqlNode = flinkPlanner.parse(realSql); String tmpSql = ((SqlInsert) sqlNode).getSource().toString(); tmp.setExecSql(tmpSql); sideSqlExec.exec(tmp.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, tmp); From 91e8880ff70bafc2864abdf6f84598ed85f911fb Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 3 Apr 2020 21:03:31 +0800 Subject: [PATCH 223/523] rm --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2952551ff..40ff81133 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ build: stage: test script: - mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q - - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.branch.name="v1.8.0_dev" -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar + - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar - sh ci/sonar_notify.sh only: - v1.8.0_dev From 388e7f8b8397594c96822a69d5f4c565ff1a13da Mon Sep 17 00:00:00 2001 From: maqi <313328862@qq.com> Date: Sat, 4 Apr 2020 14:01:05 +0800 Subject: [PATCH 224/523] deal minicluster --- .../sql/environment/MyLocalStreamEnvironment.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java index a8728c595..029a07f99 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java @@ -113,16 +113,13 @@ public JobExecutionResult execute(String jobName) throws Exception { LOG.info("Running job on local embedded Flink mini cluster"); } - MiniCluster exec = null; - try { - exec = new MiniCluster(configBuilder.build()); + try (MiniCluster exec = new MiniCluster(configBuilder.build());) { exec.start(); - return exec.executeJobBlocking(jobGraph); - } finally { + JobExecutionResult jobExecutionResult = exec.executeJobBlocking(jobGraph); transformations.clear(); - if (null != exec) { - exec.closeAsync(); - } + return jobExecutionResult; + } catch (Exception e) { + throw new RuntimeException(e); } } } From 16514f8e19f9a1e0a33fcf34f9177263ab4dc8eb Mon Sep 17 00:00:00 2001 From: xuchao Date: Tue, 7 Apr 2020 15:23:45 +0800 Subject: [PATCH 225/523] =?UTF-8?q?=E9=87=8D=E6=9E=84=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 358 +++--------------------- README.md.back | 328 ++++++++++++++++++++++ docs/config.md | 105 +++++++ docs/demo.md | 85 ++++++ docs/images/streamsql_dd.jpg | Bin 0 -> 199826 bytes docs/newMetric.md | 32 +++ docs/{ => plugin}/cassandraSide.md | 4 +- docs/{ => plugin}/cassandraSink.md | 2 +- docs/{ => plugin}/clickhouseSide.md | 2 +- docs/{ => plugin}/clickhouseSink.md | 2 +- docs/{ => plugin}/consoleSink.md | 2 +- docs/{ => plugin}/db2Side.md | 2 +- docs/{ => plugin}/db2Sink.md | 2 +- docs/{ => plugin}/elasticsearch6Side.md | 2 +- docs/{ => plugin}/elasticsearch6Sink.md | 2 +- docs/{ => plugin}/elasticsearchSink.md | 2 +- docs/{ => plugin}/hbaseSide.md | 0 docs/{ => plugin}/hbaseSink.md | 2 +- docs/{ => plugin}/impalaSide.md | 2 +- docs/{ => plugin}/impalaSink.md | 2 +- docs/{ => plugin}/kafkaSource.md | 4 +- docs/{ => plugin}/kuduSide.md | 2 +- docs/{ => plugin}/kuduSink.md | 2 +- docs/{ => plugin}/mongoSide.md | 2 +- docs/{ => plugin}/mongoSink.md | 2 +- docs/{ => plugin}/mysqlSide.md | 2 +- docs/{ => plugin}/mysqlSink.md | 2 +- docs/{ => plugin}/oracleSide.md | 2 +- docs/{ => plugin}/oracleSink.md | 2 +- docs/{ => plugin}/polardbSide.md | 2 +- docs/{ => plugin}/polardbSink.md | 2 +- docs/{ => plugin}/postgresqlSide.md | 2 +- docs/{ => plugin}/postgresqlSink.md | 2 +- docs/{ => plugin}/redisSide.md | 0 docs/{ => plugin}/redisSink.md | 0 docs/{ => plugin}/serverSocketSource.md | 0 docs/{ => plugin}/sqlserverSide.md | 2 +- docs/{ => plugin}/sqlserverSink.md | 2 +- docs/pluginsInfo.md | 32 +++ docs/prometheus.md | 7 - docs/quickStart.md | 63 +++++ 41 files changed, 708 insertions(+), 362 deletions(-) create mode 100644 README.md.back create mode 100644 docs/config.md create mode 100644 docs/demo.md create mode 100644 docs/images/streamsql_dd.jpg create mode 100644 docs/newMetric.md rename docs/{ => plugin}/cassandraSide.md (96%) rename docs/{ => plugin}/cassandraSink.md (96%) rename docs/{ => plugin}/clickhouseSide.md (97%) rename docs/{ => plugin}/clickhouseSink.md (94%) rename docs/{ => plugin}/consoleSink.md (92%) rename docs/{ => plugin}/db2Side.md (96%) rename docs/{ => plugin}/db2Sink.md (93%) rename docs/{ => plugin}/elasticsearch6Side.md (97%) rename docs/{ => plugin}/elasticsearch6Sink.md (96%) rename docs/{ => plugin}/elasticsearchSink.md (95%) rename docs/{ => plugin}/hbaseSide.md (100%) rename docs/{ => plugin}/hbaseSink.md (94%) rename docs/{ => plugin}/impalaSide.md (98%) rename docs/{ => plugin}/impalaSink.md (97%) rename docs/{ => plugin}/kafkaSource.md (98%) rename docs/{ => plugin}/kuduSide.md (97%) rename docs/{ => plugin}/kuduSink.md (95%) rename docs/{ => plugin}/mongoSide.md (96%) rename docs/{ => plugin}/mongoSink.md (94%) rename docs/{ => plugin}/mysqlSide.md (96%) rename docs/{ => plugin}/mysqlSink.md (93%) rename docs/{ => plugin}/oracleSide.md (96%) rename docs/{ => plugin}/oracleSink.md (94%) rename docs/{ => plugin}/polardbSide.md (96%) rename docs/{ => plugin}/polardbSink.md (93%) rename docs/{ => plugin}/postgresqlSide.md (96%) rename docs/{ => plugin}/postgresqlSink.md (94%) rename docs/{ => plugin}/redisSide.md (100%) rename docs/{ => plugin}/redisSink.md (100%) rename docs/{ => plugin}/serverSocketSource.md (100%) rename docs/{ => plugin}/sqlserverSide.md (96%) rename docs/{ => plugin}/sqlserverSink.md (94%) create mode 100644 docs/pluginsInfo.md delete mode 100644 docs/prometheus.md create mode 100644 docs/quickStart.md diff --git a/README.md b/README.md index db25164b6..0a42e30ce 100644 --- a/README.md +++ b/README.md @@ -1,328 +1,36 @@ -# flinkStreamSQL -> * 基于开源的flink，对其实时sql进行扩展 -> > * 自定义create table 语法（包括源表,输出表,维表） -> > * 自定义create view 语法 -> > * 自定义create function 语法 -> > * 实现了流与维表的join -> > * 支持原生FLinkSQL所有的语法 -> > * 扩展了输入和输出的性能指标到promethus - - ## 新特性: - * 1.kafka源表支持not null语法,支持字符串类型的时间转换。 - * 2.rdb维表与DB建立连接时，周期进行连接，防止连接断开。rdbsink写入时，对连接进行检查。 - * 3.异步维表支持非等值连接，比如：<>,<,>。 - * 4.增加kafka数组解析 - * 5.增加kafka1.0以上版本的支持 - * 6.增加postgresql、kudu、clickhouse维表、结果表的支持 - * 7.支持插件的依赖方式,参考pluginLoadMode参数 - * 8.支持cep处理 - * 9.支持udaf - * 10.支持谓词下移 - * 11.支持状态的ttl - - ## BUG修复： - * 1.修复不能解析sql中orderby,union语法。 - * 2.修复yarnPer模式提交失败的异常。 - * 3.一些bug的修复 - -# 已支持 - * 源表：kafka 0.9、0.10、0.11、1.x版本 - * 维表：mysql, SQlServer,oracle, hbase, mongo, redis, cassandra, serversocket, kudu, postgresql, clickhouse, impala, db2, sqlserver - * 结果表：mysql, SQlServer, oracle, hbase, elasticsearch5.x, mongo, redis, cassandra, console, kudu, postgresql, clickhouse, impala, db2, sqlserver - -# 后续开发计划 - * 维表快照 - * kafka avro格式 - * topN - -## 1 快速起步 -### 1.1 运行模式 - - -* 单机模式：对应Flink集群的单机模式 -* standalone模式：对应Flink集群的分布式模式 -* yarn模式：对应Flink集群的yarn模式 - -### 1.2 执行环境 - -* Java: JDK8及以上 -* Flink集群: 1.4,1.5,1.8（单机模式不需要安装Flink集群） -* 操作系统：理论上不限 -* kerberos环境需要在flink-conf.yaml配置security.kerberos.login.keytab以及security.kerberos.login.principal参数，配置案例: -``` -## hadoop配置文件路径 -fs.hdfs.hadoopconf: /Users/maqi/tmp/hadoopconf/hadoop_250 -security.kerberos.login.use-ticket-cache: true -security.kerberos.login.keytab: /Users/maqi/tmp/hadoopconf/hadoop_250/maqi.keytab -security.kerberos.login.principal: maqi@DTSTACK.COM -security.kerberos.login.contexts: Client,KafkaClient -zookeeper.sasl.service-name: zookeeper -zookeeper.sasl.login-context-name: Client - -``` - -### 1.3 打包 - -进入项目根目录，使用maven打包： - -``` -mvn clean package -Dmaven.test.skip - -``` - -打包完成后的包结构： - -> * dt-center-flinkStreamSQL -> > * bin: 任务启动脚本 -> > * lib: launcher包存储路径，是任务提交的入口 -> > * plugins: 插件包存储路径 -> > * ........ : core及插件代码 - -### 1.4 启动 - -#### 1.4.1 启动命令 - -``` -sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack/150_flinkplugin/sqlplugin -localSqlPluginPath D:\gitspace\flinkStreamSQL\plugins -addjar \["udf.jar\"\] -mode yarn -flinkconf D:\flink_home\kudu150etc -yarnconf D:\hadoop\etc\hadoopkudu -confProp \{\"time.characteristic\":\"EventTime\",\"sql.checkpoint.interval\":10000\} -yarnSessionConf \{\"yid\":\"application_1564971615273_38182\"} -``` - -#### 1.4.2 命令行参数选项 - -* **mode** - * 描述：执行模式，也就是flink集群的工作模式 - * local: 本地模式 - * standalone: 提交到独立部署模式的flink集群 - * yarn: 提交到yarn模式的flink集群(即提交到已有flink集群) - * yarnPer: yarn per_job模式提交(即创建新flink application) - * 必选：否 - * 默认值：local - -* **name** - * 描述：flink 任务对应名称。 - * 必选：是 - * 默认值：无 - -* **sql** - * 描述：执行flink sql 的主体语句。 - * 必选：是 - * 默认值：无 - -* **localSqlPluginPath** - * 描述：本地插件根目录地址，也就是打包后产生的plugins目录。 - * 必选：是 - * 默认值：无 - -* **remoteSqlPluginPath** - * 描述：flink执行集群上的插件根目录地址（将打包好的插件存放到各个flink节点上,如果是yarn集群需要存放到所有的nodemanager上）。 - * 必选：否 - * 默认值：无 - -* **addjar** - * 描述：扩展jar路径,当前主要是UDF定义的jar； - * 格式：json - * 必选：否 - * 默认值：无 +FlinkStreamSQL +============ +[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) + +##技术交流 +- 招聘**大数据平台开发工程师**，想了解岗位详细信息可以添加本人微信号ysqwhiletrue，注明招聘，如有意者发送简历至[sishu@dtstack.com](mailto:sishu@dtstack.com) +- 我们使用[钉钉](https://www.dingtalk.com/)沟通交流，可以搜索群号[**30537511**]或者扫描下面的二维码进入钉钉群 +

+ +
+ +##介绍 + * 基于开源的flink，对其实时sql进行扩展 + * 自定义create table 语法（包括源表,输出表,维表） + * 自定义create view 语法 + * 自定义create function 语法 + * 实现了流与维表的join + * 支持原生FLinkSQL所有的语法 + * 扩展了输入和输出的性能指标到promethus -* **confProp** - * 描述：一些参数设置 - * 格式: json - * 必选：是 （如无参数填写空json即可） - * 默认值：无 - * 可选参数: - * sql.ttl.min: 最小过期时间,大于0的整数,如1d、1h(d\D:天,h\H:小时,m\M:分钟,s\s:秒) - * sql.ttl.max: 最大过期时间,大于0的整数,如2d、2h(d\D:天,h\H:小时,m\M:分钟,s\s:秒),需同时设置最小时间,且比最小时间大5分钟 - * state.backend: 任务状态后端，可选为MEMORY,FILESYSTEM,ROCKSDB，默认为flinkconf中的配置。 - * state.checkpoints.dir: FILESYSTEM,ROCKSDB状态后端文件系统存储路径，例如：hdfs://ns1/dtInsight/flink180/checkpoints。 - * state.backend.incremental: ROCKSDB状态后端是否开启增量checkpoint,默认为true。 - * sql.env.parallelism: 默认并行度设置 - * sql.max.env.parallelism: 最大并行度设置 - * time.characteristic: 可选值[ProcessingTime|IngestionTime|EventTime] - * sql.checkpoint.interval: 设置了该参数表明开启checkpoint(ms) - * sql.checkpoint.mode: 可选值[EXACTLY_ONCE|AT_LEAST_ONCE] - * sql.checkpoint.timeout: 生成checkpoint的超时时间(ms) - * sql.max.concurrent.checkpoints: 最大并发生成checkpoint数 - * sql.checkpoint.cleanup.mode: 默认是不会将checkpoint存储到外部存储,[true(任务cancel之后会删除外部存储)|false(外部存储需要手动删除)] - * flinkCheckpointDataURI: 设置checkpoint的外部存储路径,根据实际的需求设定文件路径,hdfs://, file:// - * jobmanager.memory.mb: per_job模式下指定jobmanager的内存大小(单位MB, 默认值:768) - * taskmanager.memory.mb: per_job模式下指定taskmanager的内存大小(单位MB, 默认值:768) - * taskmanager.num: per_job模式下指定taskmanager的实例数(默认1) - * taskmanager.slots：per_job模式下指定每个taskmanager对应的slot数量(默认1) - * savePointPath：任务恢复点的路径（默认无） - * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） - * logLevel: 日志级别动态配置（默认info） - * [prometheus 相关参数](docs/prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 +##目录 + +[ 1.1 demo]((docs/demo.md)) +[ 1.2 快速开始](docs/quickStart.md) +[ 1.3 参数配置](docs/config.md) +[ 1.4 支持的插件介绍和demo](docs/pluginsInfo.md) +[ 1.5 指标参数](docs/newMetric.md) +[ 1.6 自定义函数](docs/function.md) +[ 1.7 视图定义](docs/createView.md) + - -* **flinkconf** - * 描述：flink配置文件所在的目录（单机模式下不需要），如/hadoop/flink-1.4.0/conf - * 必选：否 - * 默认值：无 - -* **yarnconf** - * 描述：Hadoop配置文件（包括hdfs和yarn）所在的目录（单机模式下不需要），如/hadoop/etc/hadoop - * 必选：否 - * 默认值：无 - -* **flinkJarPath** - * 描述：per_job 模式提交需要指定本地的flink jar存放路径 - * 必选：否 - * 默认值：false - -* **queue** - * 描述：per_job 模式下指定的yarn queue - * 必选：否 - * 默认值：false - -* **pluginLoadMode** - * 描述：per_job 模式下的插件包加载方式。classpath:从每台机器加载插件包，shipfile:将需要插件从提交的节点上传到hdfs,不需要每台安装插件 - * 必选：否 - * 默认值：classpath - -* **yarnSessionConf** - * 描述：yarn session 模式下指定的运行的一些参数，[可参考](https://ci.apache.org/projects/flink/flink-docs-release-1.8/ops/cli.html),目前只支持指定yid - * 必选：否 - * 默认值：false - - -## 2 结构 -### 2.1 源表插件 -* [kafka 源表插件](docs/kafkaSource.md) - -### 2.2 结果表插件 -* [elasticsearch 结果表插件](docs/elasticsearchSink.md) -* [hbase 结果表插件](docs/hbaseSink.md) -* [mysql 结果表插件](docs/mysqlSink.md) -* [oracle 结果表插件](docs/oracleSink.md) -* [mongo 结果表插件](docs/mongoSink.md) -* [redis 结果表插件](docs/redisSink.md) -* [cassandra 结果表插件](docs/cassandraSink.md) -* [kudu 结果表插件](docs/kuduSink.md) -* [postgresql 结果表插件](docs/postgresqlSink.md) -* [clickhouse 结果表插件](docs/clickhouseSink.md) -* [impala 结果表插件](docs/impalaSink.md) -* [db2 结果表插件](docs/db2Sink.md) -* [sqlserver 结果表插件](docs/sqlserverSink.md) - -### 2.3 维表插件 -* [hbase 维表插件](docs/hbaseSide.md) -* [mysql 维表插件](docs/mysqlSide.md) -* [oracle 维表插件](docs/oracleSide.md) -* [mongo 维表插件](docs/mongoSide.md) -* [redis 维表插件](docs/redisSide.md) -* [cassandra 维表插件](docs/cassandraSide.md) -* [kudu 维表插件](docs/kuduSide.md) -* [postgresql 维表插件](docs/postgresqlSide.md) -* [clickhouse 维表插件](docs/clickhouseSide.md) -* [impala 维表插件](docs/impalaSide.md) -* [db2 维表插件](docs/db2Side.md) -* [sqlserver 维表插件](docs/sqlserverSide.md) - -## 3 性能指标(新增) - -### kafka插件 -* 业务延迟： flink_taskmanager_job_task_operator_dtEventDelay(单位s) - 数据本身的时间和进入flink的当前时间的差值. - -* 各个输入源的脏数据：flink_taskmanager_job_task_operator_dtDirtyData - 从kafka获取的数据解析失败的视为脏数据 - -* 各Source的数据输入TPS: flink_taskmanager_job_task_operator_dtNumRecordsInRate - kafka接受的记录数(未解析前)/s - -* 各Source的数据输入RPS: flink_taskmanager_job_task_operator_dtNumRecordsInResolveRate - kafka接受的记录数(解析后)/s - -* 各Source的数据输入BPS: flink_taskmanager_job_task_operator_dtNumBytesInRate - kafka接受的字节数/s - -* Kafka作为输入源的各个分区的延迟数: flink_taskmanager_job_task_operator_topic_partition_dtTopicPartitionLag - 当前kafka10,kafka11有采集该指标 - -* 各个输出源RPS: flink_taskmanager_job_task_operator_dtNumRecordsOutRate - 写入的外部记录数/s - - -## 4 样例 - -``` - -CREATE (scala|table|aggregate) FUNCTION CHARACTER_LENGTH WITH com.dtstack.Kun; - - -CREATE TABLE MyTable( - name varchar, - channel varchar, - pv int, - xctime bigint, - CHARACTER_LENGTH(channel) AS timeLeng //自定义的函数 - )WITH( - type ='kafka09', - bootstrapServers ='172.16.8.198:9092', - zookeeperQuorum ='172.16.8.198:2181/kafka', - offsetReset ='latest', - topic ='nbTest1', - parallelism ='1' - ); - -CREATE TABLE MyResult( - channel varchar, - pv varchar - )WITH( - type ='mysql', - url ='jdbc:mysql://172.16.8.104:3306/test?charset=utf8', - userName ='dtstack', - password ='abc123', - tableName ='pv2', - parallelism ='1' - ); - -CREATE TABLE workerinfo( - cast(logtime as TIMESTAMP) AS rtime, - cast(logtime) AS rtime - )WITH( - type ='hbase', - zookeeperQuorum ='rdos1:2181', - tableName ='workerinfo', - rowKey ='ce,de', - parallelism ='1', - zookeeperParent ='/hbase' - ); - -CREATE TABLE sideTable( - cf:name varchar as name, - cf:info varchar as info, - PRIMARY KEY(name), - PERIOD FOR SYSTEM_TIME //维表标识 - )WITH( - type ='hbase', - zookeeperQuorum ='rdos1:2181', - zookeeperParent ='/hbase', - tableName ='workerinfo', - cache ='LRU', - cacheSize ='10000', - cacheTTLMs ='60000', - parallelism ='1' - ); - -insert -into - MyResult - select - d.channel, - d.info - from - ( select - a.*,b.info - from - MyTable a - join - sideTable b - on a.channel=b.name - where - a.channel = 'xc2' - and a.pv=10 ) as d -``` +## License -# 招聘 -1.大数据平台开发工程师，想了解岗位详细信息可以添加本人微信号ysqwhiletrue,注明招聘，如有意者发送简历至sishu@dtstack.com。 - +FlinkStreamSQL is under the Apache 2.0 license. See the [LICENSE](http://www.apache.org/licenses/LICENSE-2.0) file for details. + + diff --git a/README.md.back b/README.md.back new file mode 100644 index 000000000..db25164b6 --- /dev/null +++ b/README.md.back @@ -0,0 +1,328 @@ +# flinkStreamSQL +> * 基于开源的flink，对其实时sql进行扩展 +> > * 自定义create table 语法（包括源表,输出表,维表） +> > * 自定义create view 语法 +> > * 自定义create function 语法 +> > * 实现了流与维表的join +> > * 支持原生FLinkSQL所有的语法 +> > * 扩展了输入和输出的性能指标到promethus + + ## 新特性: + * 1.kafka源表支持not null语法,支持字符串类型的时间转换。 + * 2.rdb维表与DB建立连接时，周期进行连接，防止连接断开。rdbsink写入时，对连接进行检查。 + * 3.异步维表支持非等值连接，比如：<>,<,>。 + * 4.增加kafka数组解析 + * 5.增加kafka1.0以上版本的支持 + * 6.增加postgresql、kudu、clickhouse维表、结果表的支持 + * 7.支持插件的依赖方式,参考pluginLoadMode参数 + * 8.支持cep处理 + * 9.支持udaf + * 10.支持谓词下移 + * 11.支持状态的ttl + + ## BUG修复： + * 1.修复不能解析sql中orderby,union语法。 + * 2.修复yarnPer模式提交失败的异常。 + * 3.一些bug的修复 + +# 已支持 + * 源表：kafka 0.9、0.10、0.11、1.x版本 + * 维表：mysql, SQlServer,oracle, hbase, mongo, redis, cassandra, serversocket, kudu, postgresql, clickhouse, impala, db2, sqlserver + * 结果表：mysql, SQlServer, oracle, hbase, elasticsearch5.x, mongo, redis, cassandra, console, kudu, postgresql, clickhouse, impala, db2, sqlserver + +# 后续开发计划 + * 维表快照 + * kafka avro格式 + * topN + +## 1 快速起步 +### 1.1 运行模式 + + +* 单机模式：对应Flink集群的单机模式 +* standalone模式：对应Flink集群的分布式模式 +* yarn模式：对应Flink集群的yarn模式 + +### 1.2 执行环境 + +* Java: JDK8及以上 +* Flink集群: 1.4,1.5,1.8（单机模式不需要安装Flink集群） +* 操作系统：理论上不限 +* kerberos环境需要在flink-conf.yaml配置security.kerberos.login.keytab以及security.kerberos.login.principal参数，配置案例: +``` +## hadoop配置文件路径 +fs.hdfs.hadoopconf: /Users/maqi/tmp/hadoopconf/hadoop_250 +security.kerberos.login.use-ticket-cache: true +security.kerberos.login.keytab: /Users/maqi/tmp/hadoopconf/hadoop_250/maqi.keytab +security.kerberos.login.principal: maqi@DTSTACK.COM +security.kerberos.login.contexts: Client,KafkaClient +zookeeper.sasl.service-name: zookeeper +zookeeper.sasl.login-context-name: Client + +``` + +### 1.3 打包 + +进入项目根目录，使用maven打包： + +``` +mvn clean package -Dmaven.test.skip + +``` + +打包完成后的包结构： + +> * dt-center-flinkStreamSQL +> > * bin: 任务启动脚本 +> > * lib: launcher包存储路径，是任务提交的入口 +> > * plugins: 插件包存储路径 +> > * ........ : core及插件代码 + +### 1.4 启动 + +#### 1.4.1 启动命令 + +``` +sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack/150_flinkplugin/sqlplugin -localSqlPluginPath D:\gitspace\flinkStreamSQL\plugins -addjar \["udf.jar\"\] -mode yarn -flinkconf D:\flink_home\kudu150etc -yarnconf D:\hadoop\etc\hadoopkudu -confProp \{\"time.characteristic\":\"EventTime\",\"sql.checkpoint.interval\":10000\} -yarnSessionConf \{\"yid\":\"application_1564971615273_38182\"} +``` + +#### 1.4.2 命令行参数选项 + +* **mode** + * 描述：执行模式，也就是flink集群的工作模式 + * local: 本地模式 + * standalone: 提交到独立部署模式的flink集群 + * yarn: 提交到yarn模式的flink集群(即提交到已有flink集群) + * yarnPer: yarn per_job模式提交(即创建新flink application) + * 必选：否 + * 默认值：local + +* **name** + * 描述：flink 任务对应名称。 + * 必选：是 + * 默认值：无 + +* **sql** + * 描述：执行flink sql 的主体语句。 + * 必选：是 + * 默认值：无 + +* **localSqlPluginPath** + * 描述：本地插件根目录地址，也就是打包后产生的plugins目录。 + * 必选：是 + * 默认值：无 + +* **remoteSqlPluginPath** + * 描述：flink执行集群上的插件根目录地址（将打包好的插件存放到各个flink节点上,如果是yarn集群需要存放到所有的nodemanager上）。 + * 必选：否 + * 默认值：无 + +* **addjar** + * 描述：扩展jar路径,当前主要是UDF定义的jar； + * 格式：json + * 必选：否 + * 默认值：无 + +* **confProp** + * 描述：一些参数设置 + * 格式: json + * 必选：是 （如无参数填写空json即可） + * 默认值：无 + * 可选参数: + * sql.ttl.min: 最小过期时间,大于0的整数,如1d、1h(d\D:天,h\H:小时,m\M:分钟,s\s:秒) + * sql.ttl.max: 最大过期时间,大于0的整数,如2d、2h(d\D:天,h\H:小时,m\M:分钟,s\s:秒),需同时设置最小时间,且比最小时间大5分钟 + * state.backend: 任务状态后端，可选为MEMORY,FILESYSTEM,ROCKSDB，默认为flinkconf中的配置。 + * state.checkpoints.dir: FILESYSTEM,ROCKSDB状态后端文件系统存储路径，例如：hdfs://ns1/dtInsight/flink180/checkpoints。 + * state.backend.incremental: ROCKSDB状态后端是否开启增量checkpoint,默认为true。 + * sql.env.parallelism: 默认并行度设置 + * sql.max.env.parallelism: 最大并行度设置 + * time.characteristic: 可选值[ProcessingTime|IngestionTime|EventTime] + * sql.checkpoint.interval: 设置了该参数表明开启checkpoint(ms) + * sql.checkpoint.mode: 可选值[EXACTLY_ONCE|AT_LEAST_ONCE] + * sql.checkpoint.timeout: 生成checkpoint的超时时间(ms) + * sql.max.concurrent.checkpoints: 最大并发生成checkpoint数 + * sql.checkpoint.cleanup.mode: 默认是不会将checkpoint存储到外部存储,[true(任务cancel之后会删除外部存储)|false(外部存储需要手动删除)] + * flinkCheckpointDataURI: 设置checkpoint的外部存储路径,根据实际的需求设定文件路径,hdfs://, file:// + * jobmanager.memory.mb: per_job模式下指定jobmanager的内存大小(单位MB, 默认值:768) + * taskmanager.memory.mb: per_job模式下指定taskmanager的内存大小(单位MB, 默认值:768) + * taskmanager.num: per_job模式下指定taskmanager的实例数(默认1) + * taskmanager.slots：per_job模式下指定每个taskmanager对应的slot数量(默认1) + * savePointPath：任务恢复点的路径（默认无） + * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） + * logLevel: 日志级别动态配置（默认info） + * [prometheus 相关参数](docs/prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 + + +* **flinkconf** + * 描述：flink配置文件所在的目录（单机模式下不需要），如/hadoop/flink-1.4.0/conf + * 必选：否 + * 默认值：无 + +* **yarnconf** + * 描述：Hadoop配置文件（包括hdfs和yarn）所在的目录（单机模式下不需要），如/hadoop/etc/hadoop + * 必选：否 + * 默认值：无 + +* **flinkJarPath** + * 描述：per_job 模式提交需要指定本地的flink jar存放路径 + * 必选：否 + * 默认值：false + +* **queue** + * 描述：per_job 模式下指定的yarn queue + * 必选：否 + * 默认值：false + +* **pluginLoadMode** + * 描述：per_job 模式下的插件包加载方式。classpath:从每台机器加载插件包，shipfile:将需要插件从提交的节点上传到hdfs,不需要每台安装插件 + * 必选：否 + * 默认值：classpath + +* **yarnSessionConf** + * 描述：yarn session 模式下指定的运行的一些参数，[可参考](https://ci.apache.org/projects/flink/flink-docs-release-1.8/ops/cli.html),目前只支持指定yid + * 必选：否 + * 默认值：false + + +## 2 结构 +### 2.1 源表插件 +* [kafka 源表插件](docs/kafkaSource.md) + +### 2.2 结果表插件 +* [elasticsearch 结果表插件](docs/elasticsearchSink.md) +* [hbase 结果表插件](docs/hbaseSink.md) +* [mysql 结果表插件](docs/mysqlSink.md) +* [oracle 结果表插件](docs/oracleSink.md) +* [mongo 结果表插件](docs/mongoSink.md) +* [redis 结果表插件](docs/redisSink.md) +* [cassandra 结果表插件](docs/cassandraSink.md) +* [kudu 结果表插件](docs/kuduSink.md) +* [postgresql 结果表插件](docs/postgresqlSink.md) +* [clickhouse 结果表插件](docs/clickhouseSink.md) +* [impala 结果表插件](docs/impalaSink.md) +* [db2 结果表插件](docs/db2Sink.md) +* [sqlserver 结果表插件](docs/sqlserverSink.md) + +### 2.3 维表插件 +* [hbase 维表插件](docs/hbaseSide.md) +* [mysql 维表插件](docs/mysqlSide.md) +* [oracle 维表插件](docs/oracleSide.md) +* [mongo 维表插件](docs/mongoSide.md) +* [redis 维表插件](docs/redisSide.md) +* [cassandra 维表插件](docs/cassandraSide.md) +* [kudu 维表插件](docs/kuduSide.md) +* [postgresql 维表插件](docs/postgresqlSide.md) +* [clickhouse 维表插件](docs/clickhouseSide.md) +* [impala 维表插件](docs/impalaSide.md) +* [db2 维表插件](docs/db2Side.md) +* [sqlserver 维表插件](docs/sqlserverSide.md) + +## 3 性能指标(新增) + +### kafka插件 +* 业务延迟： flink_taskmanager_job_task_operator_dtEventDelay(单位s) + 数据本身的时间和进入flink的当前时间的差值. + +* 各个输入源的脏数据：flink_taskmanager_job_task_operator_dtDirtyData + 从kafka获取的数据解析失败的视为脏数据 + +* 各Source的数据输入TPS: flink_taskmanager_job_task_operator_dtNumRecordsInRate + kafka接受的记录数(未解析前)/s + +* 各Source的数据输入RPS: flink_taskmanager_job_task_operator_dtNumRecordsInResolveRate + kafka接受的记录数(解析后)/s + +* 各Source的数据输入BPS: flink_taskmanager_job_task_operator_dtNumBytesInRate + kafka接受的字节数/s + +* Kafka作为输入源的各个分区的延迟数: flink_taskmanager_job_task_operator_topic_partition_dtTopicPartitionLag + 当前kafka10,kafka11有采集该指标 + +* 各个输出源RPS: flink_taskmanager_job_task_operator_dtNumRecordsOutRate + 写入的外部记录数/s + + +## 4 样例 + +``` + +CREATE (scala|table|aggregate) FUNCTION CHARACTER_LENGTH WITH com.dtstack.Kun; + + +CREATE TABLE MyTable( + name varchar, + channel varchar, + pv int, + xctime bigint, + CHARACTER_LENGTH(channel) AS timeLeng //自定义的函数 + )WITH( + type ='kafka09', + bootstrapServers ='172.16.8.198:9092', + zookeeperQuorum ='172.16.8.198:2181/kafka', + offsetReset ='latest', + topic ='nbTest1', + parallelism ='1' + ); + +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + type ='mysql', + url ='jdbc:mysql://172.16.8.104:3306/test?charset=utf8', + userName ='dtstack', + password ='abc123', + tableName ='pv2', + parallelism ='1' + ); + +CREATE TABLE workerinfo( + cast(logtime as TIMESTAMP) AS rtime, + cast(logtime) AS rtime + )WITH( + type ='hbase', + zookeeperQuorum ='rdos1:2181', + tableName ='workerinfo', + rowKey ='ce,de', + parallelism ='1', + zookeeperParent ='/hbase' + ); + +CREATE TABLE sideTable( + cf:name varchar as name, + cf:info varchar as info, + PRIMARY KEY(name), + PERIOD FOR SYSTEM_TIME //维表标识 + )WITH( + type ='hbase', + zookeeperQuorum ='rdos1:2181', + zookeeperParent ='/hbase', + tableName ='workerinfo', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1' + ); + +insert +into + MyResult + select + d.channel, + d.info + from + ( select + a.*,b.info + from + MyTable a + join + sideTable b + on a.channel=b.name + where + a.channel = 'xc2' + and a.pv=10 ) as d +``` + +# 招聘 +1.大数据平台开发工程师，想了解岗位详细信息可以添加本人微信号ysqwhiletrue,注明招聘，如有意者发送简历至sishu@dtstack.com。 + diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 000000000..48aeefc5a --- /dev/null +++ b/docs/config.md @@ -0,0 +1,105 @@ +### 命令行参数选项 + +``` +sh submit.sh -key1 val1 -key2 val2 +``` +其中key的可选参数描述如下 + +* **mode** + * 描述：执行模式，也就是flink集群的工作模式 + * local: 本地模式 + * standalone: 提交到独立部署模式的flink集群 + * yarn: 提交到yarn模式的flink集群(即提交到已有flink集群) + * yarnPer: yarn per_job模式提交(即创建新flink application) + * 必选：否 + * 默认值：local + +* **pluginLoadMode** + * 描述：yarnPer 模式下的插件包加载方式。 + * classpath: 从节点机器加载指定remoteSqlPluginPath路径插件包,需要预先在每个运行节点下存放一份插件包。 + * shipfile: 将localSqlPluginPath路径下的插件从本地上传到hdfs,不需要集群的每台机器存放一份插件包。 + * 必选：否 + * 默认值：classpath + + +* **name** + * 描述：flink 任务对应名称。 + * 必选：是 + * 默认值：无 + +* **sql** + * 描述：执行flink sql 的主体语句。 + * 必选：是 + * 默认值：无 + +* **localSqlPluginPath** + * 描述：本地插件根目录地址，也就是打包后产生的plugins目录。 + * 必选：是 + * 默认值：无 + +* **remoteSqlPluginPath** + * 描述：flink执行集群上的插件根目录地址（将打包好的插件存放到各个flink节点上,如果是yarn集群需要存放到所有的nodemanager上）。 + * 必选：否 + * 默认值：无 + +* **addjar** + * 描述：扩展jar路径,当前主要是UDF定义的jar； + * 格式：json + * 必选：否 + * 默认值：无 + +* **confProp** + * 描述：一些参数设置 + * 格式: json + * 必选：是 （如无参数填写空json即可） + * 默认值：无 + * 可选参数: + * sql.ttl.min: 最小过期时间,大于0的整数,如1d、1h(d\D:天,h\H:小时,m\M:分钟,s\s:秒) + * sql.ttl.max: 最大过期时间,大于0的整数,如2d、2h(d\D:天,h\H:小时,m\M:分钟,s\s:秒),需同时设置最小时间,且比最小时间大5分钟 + * state.backend: 任务状态后端，可选为MEMORY,FILESYSTEM,ROCKSDB，默认为flinkconf中的配置。 + * state.checkpoints.dir: FILESYSTEM,ROCKSDB状态后端文件系统存储路径，例如：hdfs://ns1/dtInsight/flink180/checkpoints。 + * state.backend.incremental: ROCKSDB状态后端是否开启增量checkpoint,默认为true。 + * sql.env.parallelism: 默认并行度设置 + * sql.max.env.parallelism: 最大并行度设置 + * time.characteristic: 可选值[ProcessingTime|IngestionTime|EventTime] + * sql.checkpoint.interval: 设置了该参数表明开启checkpoint(ms) + * sql.checkpoint.mode: 可选值[EXACTLY_ONCE|AT_LEAST_ONCE] + * sql.checkpoint.timeout: 生成checkpoint的超时时间(ms) + * sql.max.concurrent.checkpoints: 最大并发生成checkpoint数 + * sql.checkpoint.cleanup.mode: 默认是不会将checkpoint存储到外部存储,[true(任务cancel之后会删除外部存储)|false(外部存储需要手动删除)] + * flinkCheckpointDataURI: 设置checkpoint的外部存储路径,根据实际的需求设定文件路径,hdfs://, file:// + * jobmanager.memory.mb: per_job模式下指定jobmanager的内存大小(单位MB, 默认值:768) + * taskmanager.memory.mb: per_job模式下指定taskmanager的内存大小(单位MB, 默认值:768) + * taskmanager.num: per_job模式下指定taskmanager的实例数(默认1) + * taskmanager.slots：per_job模式下指定每个taskmanager对应的slot数量(默认1) + * savePointPath：任务恢复点的路径（默认无） + * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） + * logLevel: 日志级别动态配置（默认info） + * [prometheus 相关参数](docs/prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 + + +* **flinkconf** + * 描述：flink配置文件所在的目录（单机模式下不需要），如/hadoop/flink-1.4.0/conf + * 必选：否 + * 默认值：无 + +* **yarnconf** + * 描述：Hadoop配置文件（包括hdfs和yarn）所在的目录（单机模式下不需要），如/hadoop/etc/hadoop + * 必选：否 + * 默认值：无 + +* **flinkJarPath** + * 描述：yarnPer 模式提交需要指定本地的flink jar存放路径 + * 必选：否 + * 默认值：false + +* **queue** + * 描述：yarnPer 模式下指定的yarn queue + * 必选：否 + * 默认值：false + +* **yarnSessionConf** + * 描述：yarn session 模式下指定的运行的一些参数，[可参考](https://ci.apache.org/projects/flink/flink-docs-release-1.8/ops/cli.html),目前只支持指定yid + * 必选：否 + * 默认值：false + \ No newline at end of file diff --git a/docs/demo.md b/docs/demo.md new file mode 100644 index 000000000..fe4b5c720 --- /dev/null +++ b/docs/demo.md @@ -0,0 +1,85 @@ +### 样例1: +注册自定义函数，并指定某一列作为eventTime; +kafkaSource join hbaseDim ==> hbaseOut + +``` +CREATE scala FUNCTION CHARACTER_LENGTH WITH com.dtstack.Kun; + +CREATE TABLE MyTable( + name varchar, + channel varchar, + pv int, + xctime bigint, + CHARACTER_LENGTH(channel) AS timeLeng //自定义的函数 + )WITH( + type ='kafka09', + bootstrapServers ='172.16.8.198:9092', + zookeeperQuorum ='172.16.8.198:2181/kafka', + offsetReset ='latest', + topic ='nbTest1', + parallelism ='1' + ); + +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + type ='mysql', + url ='jdbc:mysql://172.16.8.104:3306/test?charset=utf8', + userName ='dtstack', + password ='abc123', + tableName ='pv2', + parallelism ='1' + ); + +CREATE TABLE workerinfo( + cast(logtime as TIMESTAMP) AS rtime, + cast(logtime) AS rtime + )WITH( + type ='hbase', + zookeeperQuorum ='rdos1:2181', + tableName ='workerinfo', + rowKey ='ce,de', + parallelism ='1', + zookeeperParent ='/hbase' + ); + +CREATE TABLE sideTable( + cf:name varchar as name, + cf:info varchar as info, + PRIMARY KEY(name), + PERIOD FOR SYSTEM_TIME //维表标识 + )WITH( + type ='hbase', + zookeeperQuorum ='rdos1:2181', + zookeeperParent ='/hbase', + tableName ='workerinfo', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1' + ); + +insert +into + MyResult + select + d.channel, + d.info + from + ( select + a.*,b.info + from + MyTable a + join + sideTable b + on a.channel=b.name + where + a.channel = 'xc2' + and a.pv=10 ) as d +``` + +### 创建视图demo: + +``` +``` diff --git a/docs/images/streamsql_dd.jpg b/docs/images/streamsql_dd.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4a32461c10c6a5256af92e7655010d9d8a03077e GIT binary patch literal 199826 zcmeFZc|27A|2TSN-=eaMQIws^T4Yk$5>laTQ%UwFBxITqA$tfRrcy~+vt`MUt(CH7 zA46ozOfn8LbLPH!|L%9Wzx%uQ{_gK_??1oa=Zte4&Uu~JbKhRiGwkndG9-G!+{zr{ z;D8`Y@PXJ<&^fa(pDPe#V*{x{5X1*<;n)swf-?^Af#Q51u7A!UXfOB;L7a~`{^RPS zE&q0vbLtW2znydcb0d2hGCbvty^am`#$Hp<(b@+Y9<{RJ`ujAn{d2zOpHtC(xuoG# z=rz}TB>#gY_$Z#;0&U~thB&7+G9S&qm`}Vr@v8^ zbS?!&@e4>uN=eJ?*{iIgs-~-_Z(w-v(6QrY<`$M#C(fL8aCAC%-r3`_=M^t+AK&2X zAvZ$9!f!_3iHVK78=sJ#aX&LF`$10blftLZo)^6+e)+npx~8`7O?|_=w)T$BuJ<24 z_V)D;3=VxC9+{#{&-|F3`#HZr`@Q;SjlK?VZ2rXy(DT2b1&;p(v;PXOZ2+$=TwI)7 zynpfH*b@2|@oik(yY};JH?`-z7`S8if!lmyN7Eixw(=|LoTiFj3i>7>p`<&thxQk> ze=z&+LoDil2(y1j>_6}tg^mFs`@3xcKb%{(09J8=jSHXvHf|p7e>R@~edGOSjw?SwKK~$b3K>w@m?63ce z7o`7-_61C0UBlkRc^66XlBYy7}um5=d+~TaJ z@mBgKl=ZKR|9=u8cx!cdGD2XT<<=VEH&$h{$bL3v)7%n*a;#b#Xq?#RUUMf5LCW~VJ z>tuI5Q!ElKfQ{*7sFvAjYPNTz2FIzP752Mn9*&$NOiB0M@?t}z-)x9>mhevJ`6d8| z@PiFeUOjp8YGmKRN!a}Qw|YWGJB5Wu9+@P-Y$Dx z&xUq9$=|Yk_k-o9)Jw#DEHmrOmhTHX7RPeSx$bn9*qzdGROL{s4b$DcZ7X&54`gp9 z_HR1}yZ=9YshT8XxcDW{g7+2!e<6V)jXMtUR?;QIqdQM=;l<{xl#x6eHdH9n;5>`l zOQ6`Zp@8RX=v%Uz-odLJW@4wICkZP5)9fYMI|%Pd5p2j+32|sVhu_PF((vo()mAo? z329d6#9KTPe~eOuW-CgSSR*WMfAbb{0#uc>uXsTyz0IWNFeGMg_U}7^J@B8%%-P?+ z#sR2Y{dQ5tqg(xwZEU5N;lW4zxZS~pY-rSn^vA==#+7sjJtIQg^fN$`d>7Q7y#XQ6 zX+Y*v|I_T}-2V$vkvCS%S2rq}fNpQ1Z;-bnU-**i;Mm{eSFUDJHgwN`Rh!3#%=NCXeGWT@ z9GRq#U|CWFeFJva1XP|p-IAd4XiHNgWv;X54hCwHXG5aR24Q(8saer-?GoiL7S{KV zxM&P%DaOd0KYvOx2>KxZR{bvP1;Mvxm_gTp&5Ci?SSp*Y5vXK9Y4BA;fds#=EsX3dw+_UAeR zk~VsRKJVT}o(uH9KHD@+=3RDT7$O5VO^PVm!&RM|1}WEBGC$8vjQy;$AMX%uHM?)# zYkp#;kca%|u;0`NWE%j=hz(sLz{(Ba^e~e!-6Ilqti@-_k0ksNQ5aa(^eQm=c{ycv zs<}@sL-+NG-&5wC_e3EG)mqM!9cM$_2U+*o5cvV3%X&<>f*rMPx6^JTO1cPoYgeCb zXtHi>HTs1Fp+9Y`fi5^*ge7zps3c%UKG!w z$P%UWIs30lS#pQ0(+gEEWT$@!RLYVR3fxy;7j7S1ne(xlFdoVj?L!1#(KrzS22Gh_ z5zE9-JJE5u*cc6e?De9A*@%_u_-9$_nOQ%`QM*4do+RAeDqZ!E^9br2?9rmzmFaQx zx9Fn~ z0}pTPCB+lJAES%1o&s7Gp{la-Xe;zs7U?l{Z%+Bxgp17-i_jkGBuOhR>EZAR(9)Cj zj`~n{Q;7VS*Z3>%W`GYHlEjNIAO|z1Qqm{^dl%d9wG5s1**DR;=l7A4m%TNw>+hRO z--Yt~4P^BdhtDH~+SepwTa2jXIdVvlYx1PR|&+zggaQd&gsoSN9 z2u&{ZoompncDEkWIEsv)__$z8e)w=EU4@guqr`@YY{+f*{;M^JJE~5SxqW3?aw(jl z5mrUNM_pNF6fuu9QQP-f%in$)7CTjr#Vy`_q|Rq^!f_8*Ot&cWrQariG}@jSI-Po# z4N)!K1UpcoYk91QGa2m-6upVUV9ov?r$YpuTjk_I*QHw?iOj3N)znSjst(@(FyBVH zNz@QR0y+S77g1oP;9OpvXKFVNdSDk~-9e+CBw9X;^e>=wOWxX5sc>wU^322kv&eRp(Qsq@Z!3d$rr}XAlRAM@Gk? zoa=e7JrC|n+||Q@U5l7G(lq8ah8linLT6xxS{J|8`mAQHwTjb`x|v#)Bm1$%_;!_? z`K=e6EP(?m@5_J3vmO$#1o)sAy@K_;$&{gh6NKw1!X90SB!fWDdN?F=cFT_Ewwh(6 zTe{Xe{O`wXocpzKK>bf1B4)sp&u2r&%o_p02_l4Ki0~xVO!^DqzB1Ebj=8gv4GF&n z8pnoU7rX)+im6^NvXeoN=$4YaU4pMWEt@!{r7B5uGjA+*5G)AP6NEc}VZn2;=&R-2 zi)iYpWP)Q8Y%Ma1uAQpy*Y%6lX{W6&2mG=7s-eH-W5m>mKhrq`#S23j4TYsi)BmbSj+BqZO?es-m^n= zLGDEQK`y6_t}$j1AbBI9wVYVP>O;O7IFTfp`EW-D%e1d*Ybk{X=*4>x4n(}z8?;ii zFdDS7BR~D&jO8QCN2qzrvUAM0YA4vxAq;#7g$T%%Ft1ZBmI@ii@f~;*8Y+&dO5-)B zarXyY42^jDVV8BX$>F;F=CrJ^oef^Iu5%vZ2M-V4Wv!47Nohx+``Jm83as=j7WCoYDek5`gGfed$vA8IBC1B!^7k0w(m=RIOKUBY*DTWBE43A zN0P;Hva;|frYu}knkouMzsu!L$thmwwDOe9*00bA>b^`|FMWHzcHYFJAiVrX1|nkD zf1S|-E2p;IuP!)vE{wkY=su6srNzmk-kh6&`xdFjvNY{?hp_D;aW_k!*FRpAm=E6CKZ&ITl8V?P_>3hejiWp4XK)~t5gw(*I&cydF$ zlWg{2X{)I?f}NzE#C|uRgK|t%Bknq!N}VLnbw-r)+mPE2-TCb&Q}Dtg{ot=$aZ9B8 zvO=-+LFiH>dZ?%;nz6Xi=S&eb3}JQ<3Q)|WE`Z>dKur7i8vxX^4AgoWM3^%F{FjqB z68G0!W&u^`{K?4@RSewy>$FJUGmo&u_6d1vY|JeRQG9aLC)YiPE88hMJLsuCoqF`y zOqS1OwY1IzL7n{^N0jbyav(;VCM*gAG-Q$zb2jU!sY`dC_zy zq-bK+^2w?N@7-Bt*5$9GF6=hDUw+?8NUV5EB)Sn3O|14>Vnd8$-zOG`5H4hDa_7kW zYYnH2u!ytO{k;)RbvUQj)aTon1X|#sugvM z$%9Rl5s6=O@#j|bkbN1CAxnnTs}p8wvddXw9uA(f6T|yGlK7ymwc}w0 zAW~n#8<7N*I9{5$gK9I^UY;bstFK9OYTU?cq~@^mth$k<@zA-x&$(BQyLEqjm42X| z<3qVRD-{<`nM}aEws^;e;tLGx8CUBlDE^-fS&sv2&U|TTsEqY_sK)Iuwqy&n^z0O1 zY5`ob*^u|j@Ja`XYJpB7ZDndt_iSCjD@;B;#>(+~>1-1|Yg#g#b7pCNSH|^=E;pAy zlINs!Iaor>J@6N5>hw_bB{-gH(Z1=F_=kTcU208J=9lcKMf%>+-5OtY9$&sH#d$AG zZ$5hd~%;hFp9IdRnD7p zos+V%Y{t;T_tcHxb7(6R9rtwjEEuAD<+-)QDQQqLFw42oB18J>XL44!{Hl%@q4~8NFH4Iq*z73+m6sHlEa$YUS zr|%LT#NHuJ84+^Z1aFFaVw@Q z@8T zj>dh*X~+2w!OzD!LIlQK4yJfJTb}Q?+7VTD=a-|Df(&w#K)y}nX6|7e2Y5D4ZZ+YC zf4O|Rh3dFzaaVV={`z8$#;@s=TkadyZN?hqvgM~tTRWvhEzw_6m(-{KYOFAb`J-yf zmpG^28aUF{-=)TH-q|}ida@>c!RF}5jr}f*GS|1oT@1x0XW!GFIJ@h1yB37*y^@d^ zei74+VoFFNTlWHfY*(j8f5mfzRZSHr8n|ka|9noz>e}QK%dI=uGSTG|?1;%bYAL=C*090M3${k~6)s#aDl`Ilk1nBjVyM zUy7xc=G-aTC)u)N>cw2!MjBqSp#vybiNKUQ!-ls0W+h%n4=Ya_w7s?HTAp&ryWguP z8(^;PA(5xUf2k%UOZn2pw5NQ-ElE$HEvuV8Y-n2xPMi%r;VlKSv;Z(n`^m|8`7+xn zi|+W_-TnQ&R)MOflvW1^^Yc0JEs#+9O##FSO@078)Ai6cG_2~ioH14Y9Y_O#Kg3R8 z98M9O&p2J*Pp(hVt{AbtZkoK?+^9XmT!d?ue?E_muX%uH(FReOwm8-yfbBogD27_1bMM^iuwyXsn~7 zw9wOQrgD6Sin(=|Igkc$>9A_hvifZ5-`sW8mHFzJAnCcv z|GR=`;KRWVh;W4sRnILiGo+DDJ5P~iE1LEZs_|yJbb#iXQFRI>tj+&`1@yV(_8SdJ z@576Wg0hQ;Z9C!JnbWwKk< zUF7_xm_ys!io?7ym~KsMRLMQtyq$c*6pPb=tEydRFr}m_JLb+s;D#JoMABc#aYFs1 zD;vh=weRTWxheUs*XDi>HSqC~iamJcY$Eo7!`c0EmdbA-Lg5d+2T{Oz4>7m5AzS5H zZ_!stqeZQl*K?i3;a}REVSeqeie=vhTpSP`y_ITP-;|j5p#9kAbn(<##g)Kr^-XC2 za5nOpAc^)*{Z8zhtRl3OOQ9oi4iw9!-Y)kPm%GlNN8MY4cTM_!+PA70U$`M+R1vl} z(q!qh<;*zjUPWOQLTqS!6Br4o$;CW&A?<8T82Sx(vZQe!GucE1HOpN?)a~>>= z9bL|j{lN7~V{}wVP_BMDrOD|v?h31bKz@X#T4P{!uNJ#u!ZdoBgMyM1*}iy)r&#g) zqRi`2y{65g;m_MU0u6eID!-)EOuI@U!V6L&&TmQ^3JY54K#T_b|JbN?7=OweKN_}N z`q-X%+FD&rjgiiMb#&rbI9I4+kW%G|mp^}os;dh{0U6sFbtnPx1-kLG{QIX9tk(o` zflSRxOh`u^jr48~yTfPm)Z1;sM+YP&D-KuQ(?|~&@VuQMvE8DEZ@WGo_>HJ*K>dA; zBdBeN@OK)zD|KX#RshbET7ELSty!k2-!!>~rdHaVtSj3b-t;_EA(al-RS?n~a%&#jX=X^}tu z7}Fqs|9`08MMfVH{BKqH{_m<~4yoJ9jBYbyux(!*xwFUY9wA5TrhckynF}di*t&S; z|1c#ld+7h8TA9dfAtIOoEP^7^6`dmOM*pBAjBr6$B}fZQmG7q^4zwFJ^)ULClNj;C^VZ%Rg6B`qS?lOp z4%G#JsUP4wZ9}r3AZZZD4Fu}dlsGV36O|B6@Y5ca=y8B=Qzwt7t1vJ6m0M;XXgzSU zRA`lzWfV+Te`a$0*_Mry9AjM3*d;*03}mY_E3sJ{>A>&$3iO}=EvE;F!*qO|JF;>j zeaO5qdiiBK_nYc;m&Bm7xR~M0-%k%|8@?|GcSV^3;qOLO{>otRm{E@*0!7oB7H4QR znX`|J#_b5tU41{jns++snOr8L^3?p@NBUBFkOPk?Po&A8zk`_}AVdu|F~W)UHAPV4 zS|)|@GH_?_fahSf$o4%%dJ6Gc^nq$gna5Y&+z~ym9FzLqoRypj;&d($Y(&j7f!ao; z;zu!JXipWK$P-ct>t~SAg^xcD^RYen|q^7nlI6p6c`LoX1r!met!D>6G?0$@r+BV~9w?8N6 zK^FEtB=QXlAL5Dyz$!D{yUJ48P?5DWB8Vmv(N&>L*#Vj&-ThmW6EK@x@aQy(8+R&q zb}d9$*&_1o(^E=IwI#`{?2vS7%oW!BOabo+He_Z8Ypt}SqKP2u$wqB&xyae?)KN+D z5%}_?D9-ei_f33qk>8>2#@%J}1rB8lo3%G?o6 zfBFsWhmImv+a=YYMZNdAjkaP-&h=;(YA%UjcZI}R#vVO-P(5zv+LxQkOtH~vC#EdO zN4Xv0wIV!_F%-y3&=Ni}oq2_APr7*GE(QzNNAY{+oX?uJdK1KW?tic4!d7(``YRaR z$@2kc0OrPry`nF%Qt?tSVfrZSIBgrdc(jgY^6L3`dqd3FjmN({(tgy}3AkozV{6{O zD_A3=IM;}<-ph4526h5<&JI|MB)Eo62=d$8XZd;{z&Sgg{4PT4Nn;U!!%w7Z@^xwy$qzj&NluN! z8~rR$AIdwH?uE~6f+HZ)%UqRF-oxq)34rxU<~|k?kwIE8TeZ>0t(Nvs^S2SLCG zImmj!^m`tLvK+XIwEBd;JaH+-FGKdz4^`+#t)l*6j`Bpp)F!AP?`mZ|V(wx?w}B8>oGE&-{0NGqvg<*u;}ZAX{P{Tr2wq69e#m-(70Uay{nbFK3rZ z={$Rr_0i&Y*2$L=H!k5JEG}OtfLGROWUG9|F>7 zK>?`AIHLs>L=v!LN?QQ?L*1%~7DIOZQn+WltY|1D-9;z7bq6(zXjKxqfSzsd-bJ&Z zNbf)NhR02rtGI{7`GS+75H&0K;xuJE`e}lk<%JIy8U4~1PTPMfP|t35xYt@VGv2$r44N!bAo$-((f|{0Kr)1!c3LlLv-2Ewwqt5uNF&6{dI|f)YlxO)f`}VBhIi)qZ$wf25BN zY)+NVtygGIHk4}9cCgcM5%cvP>j9el$8{#G@2_1#;@MEgDcmUZ!{t!v76`Du|OC zU_+M)BWK<@9>OM+)!BuUZ(Z>)4!t(|UMXS4rpX}&l>7zn4C{J@^60z)c^!fG`ikrb zQ=VFuPc!|DD8`vrCOU-+)JOcNGYvYQ{louLqtdmAK+H!;rdT2(jH`l68D3$E^cJ9_ z*sv5-bZ!Zvkg(dJ>g}vlxhvOe;Lp>e`mR@Ho%;jghoglf4KIj*nk;;n_g%9TRXK&= ziQx`A@NV)%@AZ6b_v)&un&eASe7o|jEh))j+-J=o=n#kdVZg4n-X?pRA0l{=DoMv; z@2-Ok5O)~32-w=TX<%1ly1E}%V|_z{bx~?hlwxJLQ9cJ}miRo08Xh{j8wN!K^RupV`E1hA|ET{#dJf0XNHt~{YS$Myi|3Y+g9tXPwNgZPGMpRkRJNG0kTzqMI>Ay#Gz4GL z2QD^YtBhPfslBc05?&rVdFFzWJuU~j9R@W4-zWUW)T}w^IKgdceRzn z%5iNd*wBxDgWAK~f$9jw?}vRu#ntVp=mU*)<)fRboHHiX``B)PN7s5fod!6+X z5%EP;8!WZaB+2p_xLmiPWTwt5?|l;|p7l=Hn0LE)W@cJm;kzt!MB!l|W(MTSe^o0= zB`g);1Jzns9(@-hUcIu@&%`iyY1AjnF;rqjxHP>eCul{zVR=>QxC1%eN=}$i;gSE+ z?inG?iMca^;kt;a8)Ax`=^~0Vxo0srL)byHJo3*d5YW_MUCltH&q;SXhAw>edaDt&_0&PL%ldQcT-=fqc_|NZj)BtWIZ$@F7b7Xb!VYo)NyM zov6}!dY>>(68M>JoB+*h*@%J~MbmFmx4yyYl)#@_e*5S8kUr+^EQl+CL9er8ZttPnzG1yG_G4JM!LL6~I=t>pj>+sS=6bdg8guKnt&?DtQi#G( zDbhluUY%TCXW;Pf-Gx}COl^jZ%@jjKdgMch1TjRvRJVd}HeIf4-te^5$(gOOO=v`r z6EG|bFzh)phN>*>YMC6FrUzwArAE&TYG@N(0%Tn)1%t1;wu$89PokT~#!wHYf6QZ! z0U~IU$Zyb8M>aIvMYo5&82U^>)`J;Zc4YOnT1WhDH$hCjq{OP5k(D;l7vDW9Sn-ir zh&?zPiT)N%u8Uej#}asP%G7Dsw;ZB@4A#I(D2NpL^ESoY(vubXf#{jC=`pR9q zkKidXvU|my*42qgs?EJo^kVUlUb@@?^&5=^-Gklz)=EJoK5FlKjkM{Mx70Y3`6f0lgbTIPDV<)*-9 zHX^bgW>A&EVsRVQ&jz-b4&dxC(05lQrBgH&PuDc3EBR?Ye(Dmw^JPYh5=VjhJTYOC zdgZTK!B7bCCMv#YY33Q9K#GmyszW;8l19Su^^B&a1Un^F+?Mc`vcLGGpIKrP%ET(mxDY1wM6G5^X&oIV$0KrZ>$8>u;qff7i`cn6(2lev|)oAKQK2@7se#b;`J~U|3ISg%9 z`R#FizQg5QBf;x+=XYv8q6~Z; zTdF_{>&Km^SO#G1-=tcR`W>w+L?6|@j#0ScRB(F9N>*QA=NqB53&jO2PzkDHXjzqF zH`0pigEQ=449%b;`x}fOI!4nf-tMz-rYB*-<#Lwc)aQ!{8sV7xDJq;GCXFrxLqRpX zGMl+$Zko5HM-X{GA%x){4r}Oh$Z$y3x>{z|=Hfd1)WTr<`?D`UPXACWKm2O*HyjR7 z><1)~tm%Qgox6*uzLNEgeWh1&x{HgGd#wbjb_zb?P@+Rlzs;T#TB}Lab0VWfQ`u3? zhr_a`O%kWa70>quthouSxX^ftURVj2Jr)a2)fh9|b+=;l?b+0v@Ocafbz5zLV!-O_ z%mv@%Os;Mq4^FpesyeZ=|Bp->IFJ?#* zT1`loNw9KVbO-^$zc1Kf$M17!jj|y5^;t(pPu{i4yU|(o>E=>LcsMvmNFa|P*EfT3 zF#7f%?1=iiS`b58X&Rksq*{A()Ob}Vny2v;(Q{0>7Bu$sq$@HCu_pE9-_fwr^B7u0 z8-Z%09p#Jx1=`UbDK_*@`J#TY{xhSo@#?;=xhvsMy+SkmUR?TY8&>{J00ONISlEt4 zrfY9*S;WA?Q&>ceR3I|ig$TR_#>ACXg4;n689qb*O1qy+Pn{~gh;pc~);#D@+pqC7 z%<0ybBvacVFZ;-t%i2%jtWE zwa-VGT~3`%5A*s?>`G(G`2pnYuwwWxBx=B9JjoxMSbM-FtZLdR)%bAWWvy(TvKZgG zfB;pcA2u_+R{MK2rQ#c@9IH-CfAd_hwj0&Y)c@AR!QevLoA(tA3JyjP1ii0xS@Yaz z%Jqw?bj2L!4A0PQ47L8mOzf5*`CSE?PhJf;)_%iXr{7rdmPzIF4qb`R-g9AAzXB+4 zM(32y?AMn0R2lm6o~-Z8P66179?gm|QE(-+@{>GKuv*d5s*jSq5`0e2T3Zxze#Z6K zmFC@zc_-f*&LmntzpP>M^Zt(CDd?|YeNG&Jr~+fb(TtVUbeLfZpYSHrxEuE7QIa+G z9q`#~QfUqnCvTjd`*NEWYtFf%4smR(fe>LQO@t|la;Bl$L0m+xkT4j$iMui4L+x|7 zc-kp*B^())WauUCR7o_sz{w~oLtjUf@L$kX*`02eP}^{qsJyXErx&fFQR<~$C#p!f zhPO2ot+aOE;FHRz-g!BrKbO94c-n@bL81nunPNjAVPHiz?|>~@NtH!0&#LR-IHd;v z34<8!8M;CJ79&CHQ)Sy1uJ0+#HM?-_-JXeO_c`YO!qPHk-rtw=4Z+ekH zJIK~GVoJkWu}fG+TkgrP@J)UT$-0*T2|0IXC&pi&o0`FB0I3Tmv3qc`rBFe0Uc))cVP zx-#NH7hq+rt-u-}^4l?WYp6q8+(+`>g1$WOptj(xhu!V@5$)}g1!0fx7s@Ibbz8gY zdT&M7Ov1ZDnc{`-h*U?^u#AZO!jnF0IC;uS(w255&G*^v*x(HAfrkA`F0P&Rb`O?9 z!anRVRgmLliec!oHN7McqBJnYt%vd4xcwx!toA{#$AB^V*^@}yJrURFoyrYoKiuBy zdAHdhoo5hnB#;w{R42ml-f4<5b6*@&LXfXC?etOF5bIjJzf}2xHR}g z;%@EguXm;0{pQBjHKee~Aq-Pu2O@BuYzOb@Svo_}9uag&7fcm%Q|-g4KRYq0M=w1< zR1sge7j6#qVt{@bna+4a*NlIMz9e zV&d=PUL2paH(z=~f+!)hde9gHDx$f(<+#b2d$!_jD#%TsgFxW5%46`dINg3U`5~sV zC_Z2fEeBWY6Di|%w)y9EBc!jDU&tu_uJtS0KU?I%?lD=vH|q3t<_<|jxRZ)u?(jnd z_D`cndZyOfZOpMed5(3|uGr-tVKXV}Kd{xuUXiRHly;kZMBk-Yv_0abo&hypK+b0X z?0xtiBGsNj;zoKwD>$b`5++~@n+Vzlmf2@L(1112D?xM1V5OF*(Q>Q9`Q$+BZ8KXY^8i0TKI}MgKf*nY5>-Ut)FXn+C}% zO}|=X|FL?vvzfkRG$<9rN<&Q9YykWK>pAlvLj~!_?}BT};qvwySx$qALRqQ)-!QHp z{P@BH0s`KfT()_`ac^J5>N!ICo^R$y#SQJW=18$pXj}?2Tfw3ZR*FSjYzDq@)bej z2K?jg@9CS>Rk4$DB~NcXG(VZ-VI^<6s)z^0omK~&B;$J6DqV%99RtMb;h~0~1c_f- z*Xh0GG7%Bi()Ntm+%WzGiYAMJ3LDwVhFm*O@ovQ82PeU@h}a;I-1Rm#^gUyG5;Ucr zY}T7J$&g4cH7O1Zl`ptD<5OA{vYUVR?`u~AUfzgK%Qy*uyo+>TsAq_`K`$}}%m_+T zfJ5;F>V4*U+HN1OwubouEiI%|OS?eUo}OcWIq}mQA&~y(*N+_g`mLl^TG!RPvRCIWGzM99g7QHv z!FvVPGoYuj8g{gx#XvS;mwN4d*s2*7pFhP4oay^9C_FH5KU?Uk$?%?mMw7Kni!bu# zhjI}+K+Y#XZY^#dqXR6XlEP#Wepcx8d#0U1vSzj4?NZHX=TZ{=?b2X`O?|;>#6A&^ zYi%@a>9pJA1f021gN3#5`4l!32Wo{}x2o&w`{euVK@aVN(Xx9jwo1`UIY{4y$>1++ z$f>IwP|*a4x(Xgdt=zb#zoHZ-vr7L{TOT*QFzO8eNwL^H|Ku3o+XwTFW@!v<19$I@ z?%?$|U;*5jP?^eR}aj2aF#)zrK$lT#dZ{V5fz%5m@b&F**hO|gB;r}-}KI8>CQc>Va8`Rw7y z4|=l&yWrNTM3tqit6ngDamVcg;E;Pv*maFQS?YS?#1x4>BXw^ zr9&S=_mm|XJN}ZDzp(4UD~sj#U8NU%@rV1DlRIi^Qv-z(eSREoy}q*}Z0nw;Q7 zO5l*>naE+oqt;pn%N?j9T{dS9576i4BW2aUys{%4{A}M zG#{dh=XaJ;7S##ctmsDj86Pe;)KJ&&xALrLHq~kwfA;^F6m>JDriyZzc6cRr`EpHN zea^4vO{5GK>d3v9Jlmo=@_ZnzWpYMz>|DaH$HyLSXvl}&O{VP+P=BlE7t;55v%@pO zFXC!;ZIDLA$H%?LcV%YQb8Q2SQzAFN?p(#>5qO)8!`j*m?#;Vvq^zH=uHSz=*?9B# zwh@Ir)|VzOoQP3W@Pe-CU!9zILJ$&y>j>Nh9yuWvU z2=4rb>Fiowqgf!r`&p?0}FkCqe3noZ@ThL}559H@)1{)psA!$`0<* znoY9I+e$tai-Z%%btI~pHoUiox!nlif4ozkcL;nrMQ6FF-%bA;Be8W6G7CF~U%kJY zoR!aiHerk-_Ew68KySJkgzEx^<^`COi)jgtEx7Y`qr4PM%u=%;zQs$#je?vF95>t=HvceWr&2pf-+Pd)I#Z5@ zjnr!5c!ATee~lGd1dTVLFNd3-?JUVwtrU0EPaiIPQ@g$4{ewN3ba9Bu0F%84!y>{l ztVldxM+r^^_Irf0h2tmzdfH6~pUgYg%+L|Hk33PUrwY{573xMp@4U})4RC<6F_i@J zfXFb0yq=2ZJ&zx4J^*uKsf@1NlS8@qcnZU(Ywp9hhk`V^ooQb=G$G#Q3KDWh!SPy;41t-tz@= zE#7O9NI1U8S_TBxL6c`iU}elQiGt=szKm#Pyu#_xnoAV>{!EJ4dCYy6w%=Dv50y}R zzsrNsmn1CnRA##p0nFH8rlgQaj+E|>MJcJXn@rji{*I+C%zLyJx027Q)Ic1 zYRUBM#a)~h5o@br=uiCg$@*%I$~iFRreZmhZeez=*LytjMzCg?)ML^Y<`_r$lkBw}FN)dD2u+=>B+DQHBytLh>b3&!EnngW z`1eGrc`85ho=|(07)awhXI&cPcI;iH z_lM%6j&r99ytzulJZibWwW;&&3I3h}nV5o6wXg-ke+#shZENB!Bebf5FC1XX1=DW& zjbGG5?_1C}Sih;)Ici;~7RPE=-QJOv@#Ek`M?4zjh0N`|u62VXayACSJ_#{miab@3 zp4o7mm+tynuE+Gj_a-ys@e^ojz+8v6Jdmw}i1;{DpE}tl5+Ci}@b#CL$-uSIc9d_U zhTs%;4J9VoZBT{sUDh{nV`FLsRt@dIreb&qprxU*XUUW@j%k||$8CEt>7Cm?5}0Lh zdnH6FKqH#$Um#Jns+1{vlMjIxz#9W+GwXzg-eIVxv7>EM<;)!f`e7Pj*@321GJU-6 zcl8(JedqOrZ3*eU0|zW7??vyD9+nDi9Zg_(oqBuAz@}hhcSJN+mXs}TP=CaBpqs}< zN?O6Im-Idk5!?mne1DKYE+y1-$D=yb1N>Sixtiqg3RTp#iWHj@hIKJbOj}D=v*0nh4B4dp6a6(Ew2>AdCs(=Hnh0vEp5BThRjmQ6Bp8a zN48s}3E6Dh?Y-~NgUbP*OEHs9Eu>_k2-*`tp{8$ypoQQ@+qwbW(af$M?svRy zEjwz4jNER%+ESE;nLWn1j-NzRUCUuji&hxn4(p+N=Msbke}>2&{3F7X>oNer9s*5wkvU=&mwF@_G zhgwft>U2q=)Jw*;w?U{$kOhDYtJNOu!;+=z13|yVbY?x#v+A9=CD@kzs_55+=QifE zidYY+neEOL%hjZN^$LNQ3ruS`jf{?-6j>uC5XD0Lz|GC5>MS;7->)_KDQaXwTw5t+ zJd$(aCS*g%6=6!mAp(;$EF##$F#p@BJ_T$6{cDZzv6eJHLAtKWx1RC^g$IM1Q|KQP zox{@k#-A@*TWj@-pW^&OM1=0K@G zYNe$=T@G8zxA|5Z zwbD6R#^k#L<~;i`kM46I0mRh?2jG8;KVn$>K4IqN#kLWh*^q(A#<|b_z#3i?T+OecH&`$?;0c{nFlF}8v<3XzE0WC*H6C*xNYvN zn)${)^HjvpA3!p0e;A}NT!#H5@-u{ol>=I{fmcdx$0WM%@BQ=Eq?2(nrp$3%8|xYM zt~(^i^$GhQqIo zAKyg3&r4KvBsHyEwTj)fign#;Zs7)|yj?@%T^<4u+=D@>gZm+i^h*;jAErH(@=9UV z_{;&H+CiDOAM-SoPljs?d8c~#efp(P5~Uc~UCr?u*jaH5!p~qRFoCOxjzf6yT?GCn zalb;QA4DVtb!#Sqiog^MqRXRRGh_)lc@wFBJiNj(0bVz&EUly zf@Q;GwlE5?Br1sTV{ z=stASWP%ANoOhCzmFY=A#SI2c{;WGS>AXSno!7lv)zTv?W^tNvZh1!DlyjvVHkPFy zAHs3I!tDioDrO2<>Gp_+VH|3{)>w}!oF*JFFvvCgb{Uw@Zy!XlO`%Wb1XMUTOToR* z=h4;r^?+pa@Tcd<$Ly7&Fcwo2KXUSGvCk(9`HjN5(T`)+TR!X*I(YuPRFFCo&<6|F z8){bmmdSahadK~I9d()0*aV_zDbA&I#noCU^pM%tis~S|e=9cD6Mf^n=UVxnL{jZWmSoD8 zyd+U=Cu66~A=(mp^Xl72B11i}66PH`7N`?*dlx+b?kM$kbaGb%vF5u)KRE-3baB5| z)_#kY$)8WXH=j9w(yD1|qZvUFP4y;qU}FiKAn`9B!BR+_hw*J~^0K|xjr8C4uN~|i z=^ej3kYwqZb@b)!ry)nIX6_j!ARj@HkwW?%n;J=?+g|TsDT*V3lT?ezL=@HT)|M4| z9*c-37Z&J}sCH4!YCew6m)z#ltoMw5x>)iI`JSWs%Ykp-bL&o1*V|tp!YPouUnhVr z3p$EbNFry3P6{I&I2S50sp-gs+P5Nq+Yi0M%y{5P~HVFfs$-}g`PZk~lvU3fvboSY1L584j9 z54n5|`Q*^pS0gL!7~%I|k8794>0N3+TV_Np^_VCo2s)$vLE~z(I-FPz7qyXU#NR$A z7W+DdI#!@|z{w|1ebH(1skx-4U(CsQJ4Jo76G+=BoC_UX^mHZ_t>N46uafO|8=Dj-=_uY8wp?(aK){Yn`K0b+ZlQY_&b=XHK5hx zC%4pK)z3WRd;H#b;RAQ00C}Ep&j*$QFLH*2_FcMqpX2X)%iI#+{)>dx)8KUoH0LvT zlwpJaZhu^vplJQAq{*Hl9dh_InGJnXKPM+DT9~7BW!n?h><0+hj>Exb3?ITfCu&hU zy1MLtWADx5q3r+t@ex9nNo8NAEKx!wWXX`*Qi`-QBy?xp}#$D6Swrzmeo#I8<}XRyhZKV@`yr_&wkjOxJ>^N`QpR_EjNFP+%&%%|;n+mC5n zksZPv4`{O%^Ru)JqVLxPMs2;*I%qfx>;*Sv_XD9i(s{ZJy4IED(nVNg%sMdK1HZFD z=t7xcNpy$lAtuTC*W%}E4r}}UnD~sDG$7;Y0QoZ{%-c6-lrmlRr9!dh|Y%w&K?Zp5X&-Hx1aAdMxioB ze8yUmx;OTovqDg2|HV4nR~7@f8Op_++dr0Wlz7ii?iW0AQsj9ks%Pyg4TTKi*@@(I zDBn7Pc81QK*L`FIg`UYj?JRnC@J0T!CYR$)c`hD_%D2xLi5ymvL*33mPq`vI=fLCA z{i+ZUrwg>DUuMM9IjM3+-CB~DW|IA)eX&lHBA;BA8D}}(CEEAxdRd#NW%0Z$k4dVP zt#z1U-yNE63EoZ^K$N|&_@6l<>3cph+k{pX^OAClkoaq4p{^{d+^vK6*6}lw0a9{ju(Av7uH^} zA9p$8WcyrO_t!Sd(?2qA@A_b6dED^um#Jv23vBKQXHL6s)@+07!&N2lrEu@PI1#@_ zzoQ?h`};CnvmE*kI+}^Q+=<9m+tS2uxCh577*v?Dm6(Eo^eu;sSO~=b7v)eRe`CO-HJ|bN!g5RiVDg1M(N27Jv;+M6CoMdTnA@gGP|D-qwr9K_uT5Z}A<~e#aSBNJ zk%Y~4+nmP=K76411+bQH&oUehKl+1@ch{}AZ&9dQY5ahQXJaP-dM^}O$zShb9uC=?JuJd|^-+aHgT!d4U6{2S2smW)1jT@4-mM&uC zV96c8GTcC|5<;7MBD7e2b)Z96517f@7#gSrH~y69@>ozBcZn1fw@L6vsv&Y83N%;l zBqu^sA^D(&KYNC;i%|hx!T0qb+mvZqj>LTYykv$n^3*$5I%?VR<=Uz8ZNGTkx&7$W z?H8IGD9(4LHTxv|k14I1GqSaqk$yajhikvUsvDJ+7)Door&qa;e05THH>SM!V~oa& zREax{BL1K6>#)#XWu|>k{^ak(|7bA6GH#Ohh|}(9`>$uuf8*&n_c@Ow7DkxBAl#c` zOa$o}rko`7V@8X^{SMJ}RIS-d~qTV{H*X1(*&2QBH%+Etv8f!v2*8({G zM*RATlnaWziU`Ndo_t1*mgMx1NY}gh(-ZHaRlm)cZ=Xo8*0A6k-!_-?_6Hvaiwl{| z{G(+TY*Le|N*Q`VUJ&ThedA!2TV9lun8Yp>;=T8HoSuY=t|TxdX1Jjz4fI`@Ra3@m zhzyJ`S(8x634}~)u@12FnA{uC$#md{T}z6y$blrQy>DE$J-v2@Ogbudkerj{zOGF3 zgh*5P{={M|nn+H;qFpd&O1>O~W9sZP-e}5MP2c%}*kDGKPNF2AJTb4wzXusN0c2YY zIG~T<3Mbk{zz-Mb7v90=s(a=>6eN!w>iIG6?HOd!F)nfh7D+q&k`;5fno*_YOyR;8 z66j)pw^#5?`mvuT#Wx-^xv?o!5sEH&h7_@@v3T z?ySwH((Q7ipCRwuUB{!K}v#(9NXtT&sE+|pk%ZW6WU zIM9HS9gj0%=7R@m91!X5rksW-XBPMzfH_#n0(U!^2s;KK%_Ge>+AnJSJY`z&fgSdi z>J9&L=cF;y(K>fL<(BLG5HU}cz7yS$s@bY|`gKJ4WdXWGrD$)Jp`xes^*9ePs4e$F z{#FuGf)1^diB5z_ij^9^o7Pz>MEVMC$J)n)4jt%TZV5relGl+R?VlIDIaUkAFD(#OU0aq3@3@eQa(D}Sp2~Ts<;n=L)cMl!2wE7_DS4Ud=2duKU zTN#z*->SjxZxDR+C@uLX9^o#8g{Tq8HXpzwZqzk8-_+O)e;y^1F_f0`+4oIhu9=YY zymQ*8hNP?ZC#oh6p>t7gc#0?Z^Qb9bA?q|4s==<`D<`}F74IW4{qC!_Ak`+1BQX4W zds=W5PcIAw6u}1xZJ=Ik{^i|rLK>F5(%50lc)5YPk!!uQ&zU>qL(5(cl=MV|sxjGp zRvR-M8oV;dFQ#Egp%*{e`$7f3i`rihzSlc}&$9cl7axnip&(1oxH(+_AR`2ifMkH` z+#!N@_t#Y{Y~iO+jMt~!54$>9IM-OXux(1~bh2vlxTo{OXCDdUstD%*4f5q>=U|yIsBN_XRkl@GY#R02VL&6 z3o+2mW7t(IM!2TyTVGW;fO@$3-H=zO%aO-rMIqIfZ^)D+@BZ2-op-qU&hxs?<(oaz zKqn59Ho1!et&0Un@Q=ggV zB18Bh%mdIr1-3{g0&gPWx)4x~edp-zeb2~JQJMR2`=r~7TVaq+nZX=v64VQb8)KI8GV*c%ZeF%P<0L(GKb!*&Nf(pmK0y+(M!cVU+V7gH5;;h!*g zf5qcvi?)C1uemlF^8U^DRQ5_@TldJjqMKp;_3HyF6#a!jtmLvTClt(KpJa7U#??mQuobv7fO^T zPY8{11qbG_iG&&#{>=6yOXoXV$vqeDK%|bwJJRC%n)A*#NlM79yDP6)mn`w#qO%J%cUI?wL4Y3 zV&e>33hzO5Sy%u5h>BLqyI3VP4g6f2==<@h=!#kLfOq-UCgDbM-{e(ishc*G!m8#* znWo23hNwiO?V&_c4Oh>1gK8|Z9@&IlH`B)~_T-%HqrQtG7QMy0pG1_HF{9}96~YiZ z?3a;^%)C7pJ~IfKU9PLnQ7L;5TEGT|RhXq|0oc4Tbw5QHHfbPU>&Ou~*Ji`R-`${_ z7q`bjDKWA0REHcnGF0Qi_j#ueNV2Gj`8=g;L98~1F3h- zrxfDtsk7UoR1IlTYx3g{r6|v9uW%UG57kdiTvGjVkE`c>qR!_ZQ_t(#gF)zOUuN#Y zb>eBi>oOy8I44)W=m_bkirTU>Cq{A)W*E-<3zgF!`7ic<-RhPqx2#0IH){$D%Zc4S zkr~%+DL>}md^KPE{Pq1lx2k>w9%TFM9;Y6FW4Fifh}4S9yR!8=s3b1Ac-Pf61pkmI z{NmHN6!mrRHD=#JEG`q_rF?xlf<;?>S3g~@Lb zw&F*;&VRH!`VRo<|M?X4=7NX<5IF}+(QWC2kZn|gH@X%lR_KhYDNfz0>L%YE=sZ?h zo?q5m-*2fP@r32X8TIWC-c2!!G-3+QpJ5}1&Vf4Att}8qjA-Yur}F^9uQ$dC2To4a znV)k^d1|cdv*Tb98>82MmC#QN!LuWLzJjd+)i`!drA%-0lh%5vv39JFg~aF|c?GoK z=DX6nn5umZVH?Wq7&>S{OPp@r>H&(y?9~Hy}7fP(uZ>6IfvoMcn!wB8g+|d|p&ihDo^^wE}sMX5B|m+~~%5Bl1ZuROY$u zXl+oa=XEqQ4Sl%HkBcrv>G4<4;`4T`?$=KliTEg8m||G)A&^sO_w$oRJ1^Ep@ERPy z@c13OlrSA)027!zgxSok>e4JsXjkdHl*yNsl2Nbj@BNA_f16&{__ptaRlP(&SGdxC zA?xGehNlZ@DS9Yv-lAa1S={4d3`JKFmI`?>>TFbhx1kEVZALwn;B$=eZ7s3+*7ej; z{Y3o%$-|LlJN;2&s4YYb`!w_eUoo(A6`MQf&B2f5ptDiCo%Opk#r!JSSr9#rlEs!7}T zGSIt{9pzZ*4OA^==OCl9%jN0m=y)%wt1jiyl)Qde{g4@3AdlQqJi_r4JS~(F3~2T= z&#@TyVfew>Cadm@$AhzXgBXOf-cs>jE?)R}h0Cms{=z0wRzv50^Ae^bT$U;9V8Oa< z8u}9szHKn40~})>6L?>5VK;Ha{#BZ%j^b*N2u8C%MHh)G7V&+h=`!2wSI!hvryh=4 zLPrj71&zI}5a&uu;ILppnCAi~9M?8iD6cikkKOOAfC^^I($oxua`#FUB4El$F&9HN;&trj(qoo7estbhpDDUq?+r)*NptlEb$BkDs?dL0Ne2!Ujxw7)VX?bhsh+Cob#h@>n4Ny$0nIS_?z8NI z1VUCCdWvlo1pk5n63jDVp3FQ`hF^OnPdU{jO7p}>kJ2=j5^rsF+S$&%;O#dlmBhps zncBuRd&#*=D8Vh*mtsye$>@iIXj;pYhpR zY!8IoVIuhrj&gD#07Ef@0{dmET4L5GD_%HQ@#K|u?)3J%XyGF65oobRt*of@c#5r! zzyCOo8;PO((qGFRG)CVbup*y_^A&@Lwyw;Vt!Ko+Z6aLL`Y$=6MJ_ixyJ8;aDpkko zUm~21O>5DXqn_PBA>1bcA>|ysIvmZ3XwMtT;0Wk~&KI8SCbdQ^l{(|Jt_r&MQikCN zOTV@}{jvZ;M%AGgHc;c?*wLNCu*5Aj49^Ns_JugakTAUs>`uq37$cB)1Hjmslb-s4 z0_Re_`iPN-g+ePgic8B=SCFn-u^Ca5-!wk5UOOdwRIuf&V^0m#E3&Ev8&DT&hYK+^ zEurdrQ>dJEYw})ion~)F-IID39(Uxn)+wzUs)iDM6%k6;<8fEowAlycC{sXG0aoYV z>=k>3D**ch$}T}JU_aF?f<aJ4^}++itv%Za17;kgm|(|E;fO&$Y!5MLo>+f~ax) zx4W`TZ`o(9nP~=oO}PBwc`wS))Cgu?j?7T1_Y*IXu55f$70*6S+G24jit<^hHa5X+ z7N>2hEsCzehNNzVRVg$*_#CC?WO19O#>l`?4~9y0*DtpQ2DAetqjuf7z?_<~XnaB$ zR;1~HbDsyWU|RZudc{Qq4O>N9O@0xS*W}WkME9YDtq-z@vC;Sp~m+j{Je)9M; zVdNoTza6j)O@l7Xc<#sk9mE#!q!ys!E4f+^5cB7jKPGudY~i8M7Oy*SDj9w?(oj;` z0iHHNI|80o0OBUtzM9d=GHPb|*2*JYo|lcM&g2(T_ZT&+o{aYK?JUZF-&=%S!g>l{ zkFBPb@zGZFP&XXzn&DqM6s@a7IP+yb&v4@6r@U;~Wtz4vEG%s^+@6MF+fu%BHIWgA zy|FN`6Cgx!VjYHA9ki5NhusX7H9Qj1?g%c)Pe^_V(diqrby4qW>pa6YX-?nij&LZ` zB*4n#%GE|&xREAx&vD&);iT2z`R>;TOG_=%FBh#kXw|yAx+WdU{^9(!?&(88z~SL2 zxh1gCHejPCV8EfQCm6{htC!%x94BG%u>RavIW8Wm#nH20^LMaZ?m76)An#|WS7YiE z38D@lAW$4v2)#8UJ@lQ!t7j>MxrY?|*-p><%++!piN$yGYuDm9v){N3H`_~uD1|-^ zoR9K5qZAJ~48TtB5fUVP0_u0aIh{9V^)xsxg}t!4V*kFvV+u5q*KH%GH}-M5tm8%b zOjDL8q}on$9ac|g41PX8zuzL6;Ra66A;<;;OTMP>j9Fzn=!ngSsu}xhtb$lQD|C3$ z--cjni<9(ENP<{e)8X`IHhk;1OR8F|tYn!4uV1S?$#{dkstEEAt39o3p^&3%s5&M} zGK%X~OO{po<*L`%8HL9jqvKQKx0IUiJL{qRKp>g~V(?^e2=XxWo!BwrLy!&WOE>H^ zx1_QrTU>mV>!BEbvbfP9uKvW%7Lx+8m-=xA*RoK%cc2IHg~LF#%KHrhXi{k8oL|=v z*L!Mol!#_x_58_ZhXWCL%}(lki5z#Hxkd%dc@uJgnh4R_+)j1k^@VWk`)v^En5N~3 z_XKR?oUZ$_uXjqxvd%F;{&n%w;=`A!wPW{A6fe5Rp$W)m;s;_CsID#}n+4I-{JyS> zsLMRTM^7;ySW<0~ifTHvT!s0?R2!2(EE1seMj#xZ2mB7wdKUwn z;uEmhnk6+R)YUr>J1us7$!A*pwUN7Ya^?ZB|XQ%<3K*N|;) zWHHr|4b;5DNO4#EFVBA*DWhqG7F2%l(ca;sRPX*lJG(;guu7}pr>(@}o5Fu~6}YWI zAkt1w#J=~c7n2$52`E>63j^p%%*Trr3eff$ZXdqP>f77BxM=iMJ3eA2NIY2@}#r z%r~gTKBUZAD;7~xpZkpD8sXw5+i7QdZWhq)n++Y~u<{I7UgC&8YI^xNIW{_Q@2#_! zK3HKoi8TuVgx62-+qcWGLgReu{DtfC2R^uLJ0Yr}a%kQLsR4guRI8U_!wAU8k z>#@1?;pgn<` zsu)2>Q*|4}Wf(aLSGo^WO3ZXLxQ!Z?>!@TU%E=y7PKp;zyn!lNB>KC@MxL9L2xdPU zBd>fNG8KvKe3|Oh=(pQf5cZ;~tPX+HW8#g;i`VT(Z4(VCIbN8LRh}#2eRY0pbmva` z)zOxIz>bqdno5{qDICT9HCHwCMbC>8XQ6Jh5_Nok+f91NSWRG^J@XYp?#vqR##yPP5xyCyu=o+~vZiF(BE?dx7I)S*_feV8g{ubAthO zvy`hD4y_J%P*VJDIc~4!d(R6pOT~GMoyhe_5cz?3q$hTGF2K@1ieCXd!(+j96VJzu zj_f48f&&dFqV>K-N9R5ITouTEba$ZKGKTC@Oewa4-GB_=W@{T$vl=5^QEJ56Rkp`W zy2o}@s`uNVA*p3+W@ANDrA0>uix6_3v`PIeNs&LSofob8>aX*qLCsyA^S9#qy#&>>) z2}*bhLDTJEh&o(Fq3gjm_ZTlSE*TxzcSt`*k#m<@VU;AywQ|g;*IeufJ_09*8SSYX zJU)hN_b9tAY!dnkOI0X-zWry?4!w3up;9}4^{L`Wh$ICwB9FYEemetYHkp0qjDHd= z)9hq_7as|D;wABetG}=18*iD~N(t{mvArX?E;dG%>^`UoV@*gzBAk5{Xs2Paf#4}wRv5N`AFsSu~`D|_asD~y0v1A z+s;(Q%J`YN8V}X2n>SL;I<3xR+ez}_p z-P;G1Cf{jg@MM_J8&IySPs9gCblDKihp=vXOI@Z#PtVWNa=9UEMU;>*R6d*HzFC z`nlG$z{v%@!kzU=7t8%mg#}Oa>fY3@8(_Jm9$iS?MA)|1ZEh3KLnc&gboEswj_NiwA%K25HGE18keMs#k8)RHZre$9UE z?W9vkjR`8`QPPvOMnhHJB#wYz3*ukBLFD-v9^W> zH*+&vUo3jG@O~ROJb(j6mr7}8^zo-G{EF%>OpTtFTW;`Jn zFG~b5>(_L9>apmQw=?daBnygLoin2EAGef|lHlM=*FHhwGP&7P>O?F+-i$yPZ;=t? z`3Ql_sgjRsOC0cob~$v%7bV<`9X^)h{-jMx`Z|-1`JF=za<-X^h|T=)POW2?Vjh}_ zvvn0bmuNraz}Hn2`mP#xzDub8y!SGdH*Wj)_gqZnJDSSt)zZ*{zg8z{Gs3ZU3w_z< zC9nvUt;TP6_wr?_K5y4w+7;3}?LL+p`euuG6cyM=?ztcE-hY=T-`eZBJ_6{kbFIQ` z^-dpqhu@k6;Q9>mOS^OB6B1<=Z68`+zi_SY&imbOY=Y{&35^Mi3f~JOEzHHyM@)kI zwaT*k4KilVMG5!+g0zI&{~##e6EdpD&~#4~7T(yel9XuJ7R%|r9zc`zy$$%;UNJ~T zT#I@mfe}EET2oPfm$Jm`4A{``v?e*Q42@4ukB zWYf3?F9p211RMlS+y+WV`DY<;s-E2Rt1ES4Dicn9Q#Av%i-a>2KL)r++T(weCP?;D zoSL+_eRseRg|jM;eiX(zuB!jC(@0%Us@<40b$d~4F<Ah&Pbh-%vkXec$# z&+?Vq-<{|uv~Sc>K5Q{{B2SX%Tp;epLC#JIv6*3<3{4>Nd2vj zQ{!c^@&3kJjahM;H~VMwp8g!BD%N0%wmzph%;3ceh#_s>11H<&&5bFaQqG5bm9rS! z)p9F~G|(Iyu}_K%oz~ET<@ZxSy46GH36!2X;i>A62YgL!UgKV+q(@or<6l%YU%LE` z<=(Z$dP5e`p5<*@)qCQ+W5}U3L_WG%E28XV=BgOxtHxEdDzL?Y&nEiTFaGs8ybr(m z*dCn^&$=dX{!~_(rBWf~`HzY$*y(gWIlu3P9DK{UHjnQ(W9GzU}Puh(w|xyxd13+h|EuadHPWw`EM&+jKg zq$FnAkefc{DEl~P8fiQoTe1niouR)Vu20SlH28kk{0oTv&OObfi*za=(%wtwlj^_> zpOpv9rac6;D`fN$V%W~aY4Q5GJ zQ_aZWVa`|&!K-=ssI-;dV@?%}IB<@#2@$38!!^W*2gM{uWgZmuJC)x2$n`ay(+j;e z^5kj|57W`ZdM596V-nD_t9O6Ci&UEBE``zRN_yA4$`=y*4% zwDaGYY4)FPGyFZV^v9o(f0*g@$N66WfB&8D4=tO;PMo#Fi;BQq>*2sAplmJg*RG?- z<^040^A7y2ms48+G;f~qvP%7}{qtWm1NPq>WA;xsaQ=~^|J{H8tJ04D>;N+oY+_yz zaq`fI=7i)cZd@5qj?0KKv~8MLG)FAKGT;>obP zys@>AFP>3s%V-2+Y-@+xHJ);?N>{*anwsl@*FX-}Fl-8X#Q6>T}x->W`N<#@g1GJM_pQnXU% zB#{*i7dV86r6D@(T@30P#sC1lc@{((gcd{QA*9}!a|3mf8;WP3pf{m(eNaNkLI5*h z(V&ZyTnf^pLT89n3<~pQV=ABNOBvfshT<>tcaoCj)5cX5DpOa(-M~lAAt_` zu>{0&8ML}Sfog34e%%J@CuI{!pkV_gnvO_hNPhcZTKZ>fC)n)iPmncZ9Zp0`>${^-!aS2i)H_0ytU=5QfI z;=)eI4>Lh?j)ll++`PQDI4jN6ug#j35l6J!K=C1M!#mBqT7CCiSr4A+fBp1O_+cjY z^6N|O#1s=a(gy5y&)jCu?QfgC-V9i*6@=Je~dcDBcKcerc>^2f4ino^1N*>d*C z&h{ULPHkHj9+MnIPL2pf7IlI8f6!e0Czd7*y2ATa>FE(;ct+UzFcRB@8(9MT!a)Hb z{$=2qPIL%v7>NlK=lDQXcSH_!&KG2>q}Code(w#7{pI3k3X|hK%Y&S*I=eC6c1r}% z0Rp~s$sG_?3>L-C+-ZZB-+^q?ElwBtbj)1Lgtn+a9X@OwOfZn&l~h%^SqZL?l`Ra5tr4YP;_)P!{K#l!Tcigm6uzCtWR<}T<| za`^bPp#HC zmjV)n2s09@J0wb%ub`)JBc3#NsBy*@^dG$^0I|@XHOpvqhQ|7_YX*yH9N$4f)`YLy zXs&8iDtA}8TTZCTUFFAX9bxqsn~I|{e0ioEpP{V~iva{J04QW3$)_TIfINoi{3_m- z-_uSl(T@6RV^$O#iR&lip`b6rvll_$gmGR|3`DVkU;>jp+(zdWNq6;P!h?qhUfj>5 z=TN3ni|ie4!&OaOJu7O?`!7``nKJrz9c;P88;c{)(~d?Wt*NY-m4gn%^$TK9`BqT( zwL1m3K8jywPvaK;m%!U$&t|g$owgqo|}%AdB<+a{lo!e9SRPXt`E%~ z{9c?WF1M638OJv8I#5-=&KT>{Awhh-@bDX5g^Febxon`8jm%*+UxN))TqptR)t?2k zR+LfD>xF;!9OeHe<0A-*Eubu@i`fvA7Lfpek#KjeQ+mGzOlU4kKDzz6rj1ABF1xAe z_-5#Wuzsd%&a_!+#j9V;O{E%Bd3w&&U;_vgL)fP!W*HQGnR4P7!iC5Z77fZWb=$DB z)dOIhl03?zIO$NIm64CionL|n#BQbUB__1&gh<1H*IGE>cvDK{Yn#!0_g90b9Tb{1 z`Gz?k-xQ-q0$_s2RQ9g{zS!v@69!8Ue3b4sHfA2l#?W6uGxo)SNw4?*tQ!8|=~oU# z&^rMSSYG;)NbU8`-+V6i2nJpe&U;-*skK%9#|9Uz%9Cgj0Se@dKWmf zR^T9?+Pp~}k(|gw(hH3^44Ci{_%}McoihAZ_4aR^)mih~9$7#@dt*1Xcj^DDsqh~W<_G_*=?(r3h6iJ4*8vjNsXsr{^xv>c zh)r7I;04d2|CtvgB9elevPC@cUyT`tBGQ5pEf=Gm^4(4;ZdeFyQnyc4=x|;!+HDFPI|{b@+-a;>oQc+HFi9ld0pp<7fS9_`GDL|% zxFxl6{9+wnh7xrSipm~(*Pq()eYvknA3js#>k4G=INx>t>bm6A3mu>ua+tU%2f z{@1T>pe|U+GSlA@N6{e7tGZS$367c?^iw3SNbQl31`R+sy7AbD`$9fRS{lI2E1^&QweFFtd2#`A8KwSeGQ;Z0! z5Rr3O%0-0u103OyQDSf0-!ed2w;6zpj+hYQ`=mMQXbeE#*+BJ!bx@+R$}0KY9@;JT zbsu^!^k`>E=-H!$DFPz71(5laX9FKVpS+|cwF@nbCv|9Ideq!Fkeq~=?| z?9g{>a|u9;9)d_}SZ8#~_}eUBb0y zO*cbPc)Cb_3!UGctc_h31$R7ggRpu!4gD1$MaaxGP~}#ezzXitPcf4Hwlki1$J7Cw z{BGRMM|E45KZfg8RKI6-#EfY6{2VE!3xNUv)*I7h)t8`7<6n36Y}KrhHULGT0Rci{ zBDxS@X@Q0oPKcxjIjXdp)GR#kY8>VEaPT6y-CH$e8Xm80eOY$<=mDY?5)Fj#K`?QF zcy)Av&abzDYE)ha{LK0l@V_D52OM0p*`K*kg#Kw1l#T;KAUX4}b&7o* zBi_`5RCGjvS^*7HXkj3#bu-v-^X0GH*-HFs`51NRRA%E56sX2R0W)h3KoS|%;|h}M zhquw0VXtv+ue$Yw$^@I$q(e1R-#-2_L_Tz(ah%+<91f9N2(a8iMDcIj+W=r2+31-V zvH&AS7>hj4go*}HPzq?>T0poRz^an{n(;+z)x;R$_JL}kCakwp74UC|8Td1?6N~b% z%Yksf6#N^XZSfX)15Vl6x z0Vjk9TnfDEdynx*OU8E>8*E8V_)RC75 z-Sn;E{I!737kKFzL^9A~!>m6qz%k^aiQl#iq}aF6B!8O25ZGe{XhIUtOq}~yn-~YL z#xUCGSOD1rdq98x1`hqDqGR57av$co=7EP5{zosx#s}dh%@yfF zK)kTpItT@M8?XST+_Nb)h+Rg}1pur20#u8EC}yq;-oOgmdK%_ru@@;GhdMg%?usj2M$xm%WRR zs$Sr)iKWHH4-sFV2hxHFk>d~U4rqw*+T9P0S_+sp(CIx3_<7(z?!8RCh}50F2+qoN z0;Q@>RP`h$rDfF!pZspth*`6^!=Vg!wz?wJr zRReG44j?@Nu#^azYucs`@6hQYQ5bqBWjpS$9earcnS4Dm8A&52xa>L6@uM_EXc<_3 zoH$kvtdOr1+GIW(&5#As)Tdby(J9(Mtu$hnd9?+yHvJE}bLQY|<@PQQIZ>aUqH`qi zA6us-ZG~1F35&~Ah%O8~30CXwa?7_)Dp4bf`+xlKiC2GIG9VRMgZ++#5Ieta+8p}{ zf3Jbk|D<|m85iu+(2HfDiJ%<~?=KEgzK9caoFBn{a8QySrz!6<^0_l7bw7P>OvdpU z&*6JLEBi9RZVmwQos3~~NF3tr3XRa}$^0eJ{bt8Cxk>Gps{MYghgJ8HmI>29xMApF zR#S|iAH?Nz=I{|;9T{ZjpgU%HdkJX9&IO^ILvO*|fVe+V)~qnu&O17|So^)CL!#c& z5VJ_dO$<|&aCuZ*EwtVHwb9_Jv(eeI(ui9%KJKT!mWV~8m1}L^$Zn}Cz}i3Ejq&M~ zU5^h((l)1$ZjA)DHS;|;GJxJW9W{c@cQk9etXFpS@vkQM{byuyWX}@kE9jH{FjD$@ ziLjOeO9RDYSB+g|%>(=G{hgU!b(2VL1#mpl6?3XQLp!R~-vj&hSp`$$1}X{2Qd1~o zU2H8F5&y0w)L9+zT-CU+Fv;Qb&qvi+y~pYeww77RZc)9BPjvbjN_|Zi17BvzrpQDp zyRtY1)j`f}1ey0A7nggVY7Y1a1Ddsc`u2J&@%2OPL+`ahh@`dEXEKPe(I}b`nM^6a@>8D@5}lOf=QwDUJXgi+SudxOr6SHdju_#=Cm)zGF>G z=k*hQ&5-0J-3k^qr<;$iUnh@W?^*QX-Ftzp)0|fA_oWKm*H8tl#24;>5!(&3nCbOn zKrVnuTscj(KQOiUb$n^hL(`(v z!-sje@A{xtGC{Ln_0lS^E8=_Md|<1&5P+?A+nn~|-?IFFsvljWrk-G4B242)V(7y6 z0ACxxg%mI`Hhshl47IZh%h}!OtU0K!7vESCc`~I_`__O3c#rj-C1tkt&JC2eIV}wV zp2wmsjsqyTv+daBICa4M%EbRL^8*(AlU0ZPduxu^W%f^=ssHG0BY(Xw;E%09wUaDk zsDG3A%8)9tK!Ai1M4Is;n&&8-qnLO`*$DT^v6Doa##4!<`Df{K*O=sDNmb2Ic_H-B zfX>f}oQp=Pn`Cb(HC^=&}S4gY#sXaq3&8F?F^?V6#SC**7XS|=Hx zeETLCuREM4P{nPqn#6ed?%hqgeCDN#7fs|dxZPyqW^{I4(SP>HB68h)G?cEF4JhdW z&Qgjs*hkRwHg*w5xwDsz&g({?dciTW=ky60bmWptMIl>MktFB4~*iW!=^)Y|GL~Jbn?JKJP^0ZZX zMhuY*2Ak|{nQx|Up!9*E%jA??F%|AP@ibSWq{FuO;RD_RLaF{y!3$D$xqjoaRdvuK z0$rH81d;3!03z!%gJ0KGh6tBYkQSceVL*T3I*Mm_v;+IU6xdID^yileE3-g&yu?m$ zLc73AzDfcozqRM*HdWV5l;1Q&Dg?*$DOmL|kPcoOs0>v;r1l>Lu6>7TmVV%Tb3oS! z{#GEox{umEdC!E?+wtf`WpY1%>IKW|7D+dqntZ>gn~$6U&)p>ElNP#E=^DZ@wSf{| z2Z7J?SZd~Q6lP_@7Az&t9Hw3m05MTX;HNu+JrcuyztIEwtg&$0pRV-$RQZ7gx9QOz zLm&nSOg_gI5$XJ|R|UWeRQolh0%JeW*{|+yQRZ8^37mW5NbZ&%)AwK$Joqhrtq?=7-uyJA+m+6xAiM!c%ain^3(xmxQQ16lJ!D*l4GV>O1pjRFA! zdx=^924hr~0;&L+;E*&{vVqDHh1M@{Ls6+?+L$?!&1n564CNHAAMhzLlxT+uT>tUj za}?TT+`Ya1sSD zXto7Y*hXdMjc6$P1qNPgM-5-YLrwFvmaA7q7ujtbG9>LCM1n;-?`aP_6C6I^g9SK;%c5#mA68ZdDVU!1c2U$bcszg{K&zf1bovDbmD`K#YAS!EozR zlq@jK=xg9k@1f7iT1TN?{F@Eb_e2792lkdZ4VaW93ur=^C<64531Hpj6PurSY(p*M zgK*o*FH<{4K`R57_ZF#?OH|`@*UVl6xv;i(3K_ihLM_`kc^kp-W=)9P9s?gRGXxzW zuG-*^wU`lHEQo4CKMmKahQVq~7zcnuvU7_+w#i@>XpA5aVEbkg^N0-?4!=GK<8fNN zgM{HMbi(bN&F`NvH92xe_-3}-t+v{9k7!BVLIWu(uKNH$ABE_=L#&I}sS4#nsB@r4 z)2B50=p+%yE^<~VhGJ|IErVn{ciKQ1juWV;5}T<{*a0js|9jX*0D$!#K*}d@33VrE z{30*#I>pWOGsac;AMTAmEO}bvf%X^UUDp?#*5%F$>GG@~g2Q<`p?Viz*%k)3x*hUZ z8Y(Hr&Ln-8-EtB{uC+`UWyJ9&KRKxPO*S-rAOR6rgQzQtw0RJAY{j}D*lR7}4M^tS zoRRX;AEw9vq>dV-*aa)ch#Em9BO(6L+gV#JAK3SbweT?@G^}BQu+RAKp{i8G0&C=< z=I|J-+4t3`Z;v z(UciKynBgc65S?FQXi*4X zb9ZZJtOdx+4e=;xFh#WntV6@RsY3PyT&;rf=&I8)$2@_UE-Qd z{557KB(q+$E)1k0vELmr{#TP9x4`iaKG7DJMz43RhXd=^wGWS+Zm7bnDmdWQPw;_k z6L!{oMvgH;yaD}G#h{?xl1yk?>O9kYCkO%YAp-bf0zI{%4o?&Dgcx#h#6d2K5^ltY zb`NTtqg!HDRAzt>G?zyCzWCj|e&JV*S+s?IwXT7l1m-n>#_YxR>;|0<5cu6gu~!%0 zOBlg2!mY4KoPjpRT&+K|LX9wWZq%GE2K*`@DNa+_tYF(Q46l=$V=4D|x2l79ywb~N z=3TdP&|eO#_bi`NU4OMn1+JI@%Ghc@oP(XQ-;|**zB4nFfg>r812_W=uKs^f?%2P- z08F{T20B>#_is?k2I`$Zw%Z7i=mM;!4z)HDAaW{1P5=7~ghl7y{yOTnuQD^Piy%DU zv4QHIKme!U#*j2%u0=t|@vM~!-l}6$)iG#aN7zd${=2oz`%u%oxbug~bku(w|KxKn zsGUf1b=%!T^MXjSYTzDDzS}V?GSm2VX?x~n(PaX4AGlZzQoofThTOXV%*AP4bDC4C z6a6HXoaYNz^sJv2h&io53|U7MeHct8FN&e_&LXDwD3VBRE-+h8*bt}+o7Y3>yyF12 z@7E71e*GW~-G299(tv;ED5ih5;QWiXE}}iW+V9E`?CR(;xdqme@q0-Q`kxBz|8BAi zjRgF`Hh(mJ16a2RH=yO?de{c3Qg1=xd)6CyfG$sQg*8NwGbLkq8rOH+`eiXFE(*%^ zqKid=K;b273nRosk)aSxY}Ef-uo|1z0xJ(7=L{&tNQvWb7OpMy2u~ND-$3n4hc9lR zE)|+E_9PQ~KKNttCa}6Ba;BJSf|NQy>zBEqN9lLg|I#QrmV-#1!2LYFRv)vz^Z_WR zUOBpVj1F#214xMd8>r6$l}DhS=SfH+2{%>?T3gn@biLJUa2@R$-K7m{2l?N1oI&KM zoQ+~p=}n@Xk_IA%eQ{^AK33|Q%_Nm z{Saz&XmH39>mv~<5&!j``N}w!JWYfpK8ey#mx8Zrz1V~tAHtGH#j)f_yg0DMw57o7 zWy#TRnhXMeXgfF@{t)RZ@d_T6GnfkiBP(QL=F~>}%Fj{FNAJf?SBke!zAyP;l=3)1 z=DJfvSi3A$^yck3Ho;lkCs+FMIuJU@1W{l+2VfILpuwB*fj0vh|FfMIiyabl8;HRb zr13Em;)J0XxWcLcc;W?379tr|@&i;Sk#aWYOz(>|R*}U5YbW0HeQis>Xe20cbV6Ym z0J&wGHGCQQWZf9Wkp< z;2NMc{D;}-kXsgFWOsc!|Hc#oym+Up#|LNFu1?G6hykD1u`hT z9nKr5v_hyL3gruoYu5jVyEl)As{jASM?!Xrl%=UC3b_iYEQc0Lk~S&IR4Ut5+FX|8 zh!DkumMfL1R9a|SNJ);dx1wZ3HDktBwi#ItbLRAWjjru;U7zoLf4}$Rb3Y!x$NfiL zU1w&_d7tj3`l&sBMyO7r2brSI>L=*2-d+9CJp^6FzaE8+{0#amg1q% z^B_-Xc6@Ncpt0+@NAL1R#-{qB}Kz?%Ceo_YC39sljJfi8Y!rV$xSCu^X$tU~uRf zkc=d}c@*pzj=;Gug>m1o0{TNK7th%0(?uEx+6Lp|o(6c6vfa9CF-D!KVx*W9{P9|n!q6@vdMLbz?`f~o&=Euedh4^%c`*w zP>jJe<`iVepo=!@1m6_b5RoqA2)P^k2)_3daHf2m4`;eMA1vDg5Ga5Iq^fd8AEVu0 zx)b>@sVY+o-z-{#OY_{5{T4PkCNMXB{=D+`n{`umd;ZdNIX&7u+MM<^159f+p}uR4 z8DY{~EQ_?NO33CrWo9H#;wIY4#FIkF3HKofVqqT~{%FE?$`nv*7vWV*s3W%uP}~r< zxSH8N3qp=Tx(F{tXpeiARrUld91(|VE&WhGQ20$;bpuPFxOr=cOO-IvdwHh=*kQFgG z#ebbYsqLj1qNFfpCn^a1MJ)(P#c1e)t87|z-@+GNg zeGM;I`93&GyUAoB`;_Ypvhm)c<*A!fYm8%kiF`hY*Dk(q~KYj5!HoCR1 z{OR?KCdVXui^JdImu@(Dd|$Yxs7$EDFZru`ei_8LLE%AcF|LM8mWU-UAh_Va5!_TD zIEg)Ng4Ve`{X?`9i4VXi5&wdXu`BC4CF4B!cw&ygJx|twdu27snv8Vb_ynxPSOZ&Z zNiK&t*J!U3NYYhsm&xs^+v&VrAN*!G5iWx&*SP3QZSv@V>F%=f2vf#!i|lWvX518> z7E34V%(5%2)?Lm_f$>|cbq+|qt#G(YHOX%b8KgBCroELsqK>-jQ6e@lzBWTZdm0RZ zsTo;pb{rX607+S%rf8ADfAI+=?DTT^4F&d&LE`OpU^~_z0!L6Igq9%T7(l{Atu>s0 z2ve9|?dy!e^yqxzTPqz67@HLAeZm;gI}vv|4BEbw#2U zca4B)mV!zKUsdI0$ELuu!cX4S4(sYxUa95wFPOk(<68Fa7a4t!qt-htnkNNdxS4Av zq5zeRu0Ac*+mUHZm?6Qm5nILyL-NKa5TPqunZwJNf+A>Z-#TPx0zBFbtoCKgX{r8W z)UH#I-f1kZMYpRUJFsJrO)*E};c(GI;|8c*>%}d!-Z)7r+!?|UJio4R5O1A3rN2JM z-djN$k#7_S(sWQwtosKJ?-IY+{hhLbvy28V)F_Bl_e5@lpJl_=zfHKTdPmlZx_y!&c%V(2&I-%&sfjQ!ntZFo$Y@wrtQBW>MqO#&eKGX%a%t zmWJy zqJp3T;s(7XSbX3;G7um5o#OL?LLNc5o%E5VoC;+03m~vBhb+mZ$<1GAU2I_O<>-sa zw7H zYSEuA_;so<6oJs>3@HPp-61PUYxr+eRC$zHuqLq{Kfj|eI3e$B{@KQ@TXJsAUuN>+ zCgW*x%BCSE|0Zw+lOx!iUPTyB0CeLE8&JVcL0L}|5W!|F8W>a;p6TMp9CG!L{$fvU zdky^454T@Veu8d?v}VH+Ot1!N4lSM2;vUCPHuGL5l+7tv^YKDZXd-u``tpkA;WVSt z*aqxAf+xw56JU>4EOjTewxUAKZmMAa^eRF#99cvfXn%078xi3dWBu^|I8ee``89$8Ip^s zqi3^!xQi1)N5IVhxGG_&2-ebpvMgaf8ubHOh25ujzvdUPqh>_9*O}QYZs}R{p=Ls1 zeK7y0=8E^eRUOUeRW0@D+A@6(9h!I;O_2HD!0|@sPm4bDL0tpXX zHx}nF-3G1btG&q?k?^QjfXx5+0YH!r$ymR{V<dOIN-FI^@j_sdB88ufq=~E8ETb_H{ZeVa)+iEd~oG!fKh0pb0@-Ju(3U> zL-Dz0;5tT9cA~ZPTo47i`6TslGGyR$diWp=bWql;DD$WahNj)5NIN`bIHNDo#_ZOY z1~nbU)8FRJGtFG|ZvPs9d1?EMi3hzGB74iO>_C@9&%gZ0alXAWWX1m8szSPfl)^7K z3TTzzDHHIgbQt=fR_F3ZSPdc&0tdntFzUXc%x+M-sM1pigM6 zMsq5jgZYq$!=}jT!A2QG{S>)`O->3^Qsc1DM$OL>H}3Moru# zNl;01FJs~hMWg~paE_-ECtP1eQ_;_>v$4GC9hh;fl>AWi)Z1la0K-4>(#>Ps2+!fQ za4vGzC`-r0bmsj+oH6sI!rVpVo5GgU6sM1anj#O8lM(WelLFxS${z_TgR#);mwf61 z*R%n;`SeU8RY2j#ED1Z|^-w^b5n8qu7H3+Atryv~?oHb`At3xK9LbKv4a-6EdKE4>bGnE^+%IY(&-@BsiGswJRtNce|FL35Q&3)B%#g)^?Q-7JKrhh5R$_HA=<-9Ohr3y!VAQ7F-ZW_f0tq8Qg1pC^I+X$s1pL%M?smv z#0qKPAK0&}R}4wTyOQPwJR>TuZzk51hgoa@+R8mgj|a+gqtue5sqTA#6vAjoWgioHMmKQ zc+wy8AJ|>37$gGP`6~dhSREQU4`VYWt_C2jIS9AW3^yrSo0k^()5*EFcA$BthgFBW z1jbwI@>NxWLNlkmPk;IJUZ!h!`r>ATRck8`b7N_Jw{D#bV9Sx`5fZ9CrWRZ2LLaPf zSlH()0H7AEk-PMz-^5C=ikR!16BQj3Bn~W|g0fzNSHxX~uoU#h-X>Ou_2wLU33r9< z=t(nLJLSlma!?lr8NWdEu&V(G{vfCd^)shIgVjbQklTkPdNLiUqD>5#wgCUcr)q^o zxi2a6Xc$y{88AOtP;=1Wc%#k9JJL!0wH-u8Kx=#SM946Z-v*oWJK{A3Q27W~B}=qX zGW|Mw%;0~KdLH_3K<^o{Gvpr*?r6I;vEo3qNP>gmVIvEeV?a27`ga_>U!?hejDwLu zi$HXA&45(Dp2pg-S@sG`V7tJDfdkVJSu}2fIB^stHjst76aeHfa1nzJhT~BM42`S; z@osESy*xjM(5m3#m#75{|D0EONEuR!7dWVP{s6bfsBts)gf9bFwH-q|TF0Bc`GTYNgonrz|- zR#7zxDe*-M8t!c4ar0}$D#7OPI)`HGpzCSti+3*{+2A++8?D0+_!G1*YquzJRxmO3 z3mj(@)@VjK2r0k(jz2ym{`ioN$Ai4Eg0UT`n7}U^@XRQ*Xof)tL?p(W%D_!J#1p}A zVr7B=tNMbDUE+u*I{tpCOB@Szmfuh~J!m=rt`ARaZlT-?&QriuS)i*0?(`hs+apoq8^;6f=8f$Wik@L@x>Gh zo`WB7K*9n*?UFh`7?++8y`gn@oD%mKM4i1^=$>eC?a{k#k`=CXVfy35YL3C&*tT`y zGqG3IC$(2Y@?*U!u8&l4;%E9=DV@^l= zwncKe=nZ&`M0cEgQi4Cia1-#iRiN|~w^}@(^OuJ6-|H{n#UF zQ>KA%T7v2IxVr-YxX(Lp?e%b4eT_X5iI$tgQffvG0HMX&BH)u^i)4ijNuY6z&_+Hv z0W_A3kvJX=WF~MP8TMET28(P3!!0+S`thi0Uw4nbK_3J(EJ*ab@FIR=>d?&ee2w%* z&tX8E>SJewcIIZ2ik7P6eA+JbPe-rQZ;oDg_hSfeHmGQ;;1dCGbg!3ziwNAy<*UZ< zq;b&i*F5pzwUDfGNx&pkCc#BFA^=al8aWH!FSHF&K3wjZ+PKME*>1$3a(R*O|l8NPS^?dU`>O20>2ttJ6p+o zn|(g!YVXfdY5O4oUVyS5(gd5YF0h;UNL9Ri0E8Pt%b?y<*B|++8M>TzM}%;YegOVu7z7wM+c3cXN5XD#q$>U)#v z_1(pj9+n-{cGW=x8kMr5U+mjmGo8(2mvTntM1ehX4?NH*AU4CW1hn6Mz)U391bIp@ zm!==k$no7M_aQpq0*GEV0=Ae6c~-z5F>gZgm46cWE}-wFq%sJIK$QEy%k!kgrPaeN z)&>iQAN5yrnj*qW4U#qqk!{A}{Y6yYgV%9a*LT65I~#^ht(e)dXlL?Nw-Zv8nQ*D*eAsyHDCc_9N2ZoSvpWyDr8+dkWY}$)Hjk( zRsZ}nRq|=A-21mNzT0G7WU&{jLnUzBQ1*3~GYJU{h7RC1Qw8~V2Xg&r#YK4+M2;1a z)w|E1*9lrRp;|viiaCfvh^eaP2lc!j{*!tPf9&E#*u}q4)E>z$qJQ7B-(<=gj{G0= z$Q)Ao%Z=Ztj$F$@dkukvBtU8df*Uf+sM^G2aRSV;nvEy>4JjWkE7rMx4}0{H+2*d~ zu+?hD+!d=T&F}qX_dwwO|3U-C1S0# zXTZ)A(Yc@g+Ng4a)R{l~QI3p|l+H!E55vYSskO6=sv~E@sKzPzZ!N9EgL{|#+Tg-9);@KV{wnMXt+N(l`UtKc zkF*waey0rg@*!cy6K@(P=e3Hy5){c!3_pH?oB$eb1qLV=3#Af!>HoZhRld`IN05hh zMxYExGyGsi&YTJ7vk0vVuEY91Re`e-;!c0R#1!UL1M4D}rcPsYe7meV1g6yS<6ziR z2fPPR(eQrAI=F@KW}4yp3F66O9a8)e6&Aw1R?*$Rjnt)nAt8+r)>!lBf`EM`YYa^P z-#=~OF04?}D1o&L<7Qy_C^7LRPZikcgNT=Tf#FlsSkSpK_o@YD4s{w(Gy za6VP>-nw!XO_$aoW<8L%Y&Sf4rvnS8=AfLH!Q|qto@07v&J}$AcM9zsHoD9fz1KrQ zlZLAhe5NO%AsX6?ug#6zi@z6KSYKOY*3!A6@(OBR;v2JoCH<49q0X-=auLL>=}EB5 z0G<|sue;ry4i@Vxd1T-=EK+CT;w`muIg#=(t#$p-X*@S8QXMzE=E@AqDxWmPLNq}k zarUV>2X1dVI=mqhCO~>1g1u~Fvb`=^da42GYXcpkdlTV)0(n!`PL7v?I4@LEr3J$C zzQWN=R!uU z2ljzRs%X>@i(m2+ZS%rZ$<+=t{<{4Tth@%ZgAt()zpwn07Pq6+c7dB;dFG5K-fni4 z6Qyg+#r5mTl`e)(_v``*2&X5L+9z1HiP%{H*y&6=O|bMEl$)Xw$@35GFIZ2X|eHwyeK%K56Ykqhx7cPj3L#d<&W&rmd8#8S10vGSWMKec2w zR@C={VCy6I)!irCB;3Jt24Vybd;$GWq-%(EY+LlR7rZ~u)1RLK8xcS~1_Nt@@mY*2 zMq&m6f)9kE)}dU#@(6<0Q2N5)qD}uJ=M|=&ybKG3axe)!`MA{VvjL*;&)1Y z-;Z`Ffv|H%QI#}-BORaP)Ey_jH9dPKtJyU? zeBVg3hT5%gUB02h<@6*;q4DwKP(C58Pin;{S5Xg)$#4!@Kqt;SL9VDqJ}Ndy!%E4Y z!;`BHf=Tv{%fZbU#6%~a+=)Xq<8W90!qmgJyR_zi_|SZHi|pBeO*5yz3VS(UIzt@4 z(Q{2#?MYyNMXb^OmN<{p7>Be6y>%zuoM^o$7<<$1f2R~ei{Y@)9v6tOeBg6LP(D7o zdXQ90hl$JRFdYi?4#Tvl*5*i1N!yfx5?wxHrIXRYH)s>&r2j81hOVJq{Gs2Bi8H?& z46jxmiKB#d^yt9zM)*vT)}Wbq7C5E9Q#L_sj9j+(7kKhgOHx413nngfq+s2&dTikA zN|3!`Uvg;yZLrB*JfT)DT;NW+S{*~XEw{t1)!Cvq0gQfSVEDW6CYRxzWsw-Pf3u_~ zz;;QwX9_g?01b6&4<^=$N#!(A`;EP1hKsgQWQ%`wf<_Kc$oL19)N=HQFG z$qB-2pV!$wk$dXi28C-k$MTUHG703ZEdBV;NQ)ht zB26yUqK#VKlolWFhO(nzT)#5wu*#(XVS!WlU4u>hHRq&FIs)l~N1j(iz25BO9k_7c z*U5Vg;ty?BPjK8gRZb|2u@qb2&Pk$Wi@sBS{oSKZPXPHcqmwb%vggOrPn_pHDmZ+; zIneR?ucK1Wkuy=2unx&=X?0Ko@9*ZBuIKyEmEWJP3AWfZPu@j;$KCAVBM##|yQ#+? z*P7AAKfyUWSduEJ?&>HDyo52T0+hENTsh1*4xwuWzMMl(^3OIN;0YA+V zn9bvB5YZkr9c4+O+`B&3!rmu*x&krfJ=5Zu-Gmy~el530;c1tM67F z!4$F$z=-WO7Hiu`gGLzv>F9{-lr)*eKw2cjIq$v2VZ|Vw>ZyjGG%Jc+HE!;oyx{IN z-3ZU#)pr=(PcNc1hCfUytIATO`$Q80HQ+%k1M$DlZjdw7E7v(ht)2{y|+FSR6Z4EI~Nqc%T87w$oMo0}K-z`VWoCte9x!cVXGF#dpLe~s2YgIy>^{XLarc5==?(IFY zA7(pFkRl7-vT8*)^u29holm))kGz6@uU1)-xj4-rv&+L~T zb;RNuK*WVxCMSjMNi|q@y!ypG{a9M-F&*h?r$3GYn*X;L_#8PC#KI4N3yrl;&`C=g z-){_&F0e&vIzY^x1DBfQe2~t9PRapy%LIf&w>z~*dqA=fWT@P|1SB! zb?gI**zhkAVai{vEsv*vP4oF5js^HncA`}6=pa?Li!+LskPQrh>yaQX6o-9jxxmp1 z%Q%8MKcTz7<1ctq=dd_H*Qi5bu_d{WCTIbe*dbPPpNw)rO2nagY94wrrWEu<2eHVP zQ~(*Ud=fOK1MX^Z`o|SokKBs?XTN%mO{fKv@*vw~48#{P?rnxb)O;{Kmhq^{xSwOu zleFPA&+`3S;(XrjT=yzs+$*_y_Gk>m8C6Sz|DAEnn<|SwN`O88c50V6A&3Zrra0(`n-+`l=T*6t6yiEC^5;k$oFQ^RStfS# z2|e5@YfD&cd0~6XsZ9Yx{yQ_nWy)83XdGDm-cZ2o|9}V_puI}n6k2qqF@$yu`2)XYYrvlzrmZz;(nqXQ6Ct}gNy`BiBP>|+xwKK=kw|EgZl;k2FG12 zl1p!edeq!v4oH2cBnaV!W<;Q+A_0SxGsM}NAmS&YFQ$+-Xrlrmo*R!<7DEOL_|+6( z(K6G-pi+Z5jfy;{?gWuuq)QzP>T?n1ANwW4Fk#?AT*lDQ&c{g`oqcuht5?3J+XoS1 zI-wRsDh!mh;qp9;7#GonazE2XR-q;DpeaHk_&rl1Kt4J^jw|fj_(s^7Z6pV|Pwjax zcsZ0SzD;Tc70cnF+^vCn>wVby*1O~1l)kY^$OuqX$kI1pk7_~BMMGku2KxH20MKkf zE7bDV!(H0Jv@wVR!?E#B2+@F5WI%}zom2q0F0(`&gYM~TTo1$MvP5%;wY9{4;iM`r zlcwFVR{}y6E!)Jc$28)nRGG9&AEkA`d#h-}4WDHqEe`lYeexoMFh0HDL0u(%KHkfhP3b} zf^Q|LPUQSsU0$4HgWkqE&SvV@GsAN)=_hO1Je*H~6&oC848DP!9|s~Ze~!VRh16h2 zH&C_VGpGixyaXy7Qp`VomZX@2PNCkDBQl3e>7qX(9bxELq~i_kkyTveGCt!2!gG(c zF$|bBbn7-Av_3EI(eb_EyU#W)Pu2)EDVL(P>KjS~ibDUJil}jT&bRu_k_|#2q|vG+ znufB#naaK6BP+fWr5pCL0+S}(w_pDfrT=d4-QHafL!7rKwM=dG{ovsMR6Fb)^GJ9{ zNi9oXaO>t}Rk<;9#pbW(%0w_%5HMG;Pklqzz+n6NufY#67ublX3N&g?B&FdHQ<(@6 z(2fs&c-v4}w)NfJd9x43r5N4y4gJM7=B5E#G|6zxC{?<7JIWS!k4D-CjR{)l!i614 z5|5iNHH9JAQiB^*9gmX7G5b2ZCAb%_U$g=%h%tD!^sS5O)2nmo>tF-;*zH1>TU%VG zzR0`lpvN+xzkLXcUy8fl+zg6XqsTHR@PNs}hK$_P@0_a?QmdAj&$o@1o{crOmywDr zpss}Y$_WfUJ5JgpaDpgs-@Zj%G`@noq;5c*YG;bF4gp@u;FK~a`8Uz3f)!qegkJkx zrW6zrm+3M+#6!A1jlvW@uiw10ICJ^pjj7dkmou_fy6BIumD?^D8fC8%wkO`=IM0mv zOg3%Sd#rG4Zkqc&X-l%ASROdB3q+lMUZ2bt?IV3vbOk4i$s2{$HwL9PZmL&1!s20- zC&tP|hG;@2p#|+?^={%;o)8JP30voq{orx3hR+p~*E$r+;B2hQuyS}Kn>2sl*Zt8& zX}DmbKGShyYEo*qL-k#+lNpq+k*XoVzI@Y}q_BJ+>u|9C6YkucBp1ZtHF_U|T-m3H zz_|xuHMu2sjICth#?G(on+gvqFUR((YLeyyYve?Wx)_439NfU;&57_}xhkfDdC`Ko zM%Ladd8cm;`rm*#q8HhkOoEOV;XDdBVL2lwobsjUeA1pIJWv2g!} zXYrnkJW^fkqNnAP5Kn@7u|h=P;-=L?i$8Q^J-;Asdtu(&oGTd%pV>d3X`l4MF(PJy zC26%SwU!5>=0nF?*gA+E&Ybu*~BjGEq5$) zKi@fQv%qWyKP`0Ewj&d!Cw=?6&ZWzKVksrWw=oGK3_y5-Eeza%9bY^iH!dset8!QL z_G$F@3VBw(z~^#UTT+w7q3xLp0Q2#$t-w`1vS6I~nBfN2x0d6>Evr(Gk{OpIHKgX~ zNpRAEdvI<-Id;Ow2v6kFV(bj-2euyed{%!*RBbwy_c*`sQd|6n=?9oP#?l|P3FicH zpm+sZWYt{*R(u;*`a5M^J0Rw3sSt!2UID}j)_qLTwWQYqfJ!z06eo=ORyjoD>&CBBrfyS0nA2|`~TBO25KGsylb3aMw z3y=z}QJ*`!t*d+Awi{ddx5ay(+Zm!CGu3$}#x9lMe8}$F2U_GCdE5{>Qv#8e(*J$| zYf|}t^qU-fADhtZ9Oqu*4qEO9cFjp3Cdsu_C6gdZS$4&$k@NA?8?UbgHEG2idTA{( zj{9ITsbX)EuyAPC_88lsz8?|8-?KdbbC~`=+5}w_9}zjisM&LN(_DqVL&t=+^kYR< z z7UnX0#G&nn#n-DU&>3cgcL*f<^3NUzBLGCXs0)1vr}ZI5-0 zTG^jUIx>LtYjBGkAbyz?0gE~y1z8rESc5aPXPt)kh#Vp(i+?3`KSgfE#XMItg|NUE z&Q}?0>kaxZC%11)y%oIC_h=tY4lNUapvDGhHu})tm2??l74{e-LR>9KOPJ&kjkBsy6Q;q zaNH|`XuT73rMW#k%EsTr%NFsWgF@fYGPWlJ$~8KeLmnDZk_nqKN1?L6?<6Kz1%D|P zPyXVm-ElkwQ~Bt=Ani&_EZWTy((x&v(85#-B9^nF(Nq#Mr_^ z0Fz!vWl`=8RJM3d(oByU7mI$ zdQos9I>|;xaImD1QC^dzom-rE*}wL}iRP{wAC%OkV|qU&SdP5@Hw1bGp4xI7`T&OV zf8Bm6(h`vqL!G{_%nxqtdJZo~fv>>VSa#6Ul-`)mV|2HOtclGMrYs8KqsoEWc5zp~ zuxENd6^U+oEi&b#W(i(FS@1m$4p#bBn3Bh#>d-N5uU%b z(JWTYrJ$$4T!2oR@<-s~@!~Vc9u8Rk=Xx{+7s~C_1cos-uGwAMLj^aRr{tQLnC|w> z|W= zR+V0UWgH6ev!FOPx64XCV3%KTnD*xOQ!V-qDVyozFqa3)=<##VDlq_d;!LC(1%1_W z1MO^~^)$+=HbmP2t;A%ZFBTPpoM`+b>)(>(as(TKN1U{(5Rv*ICe}ZTm9>x6-P>h~ zlZw{2^huL!s&bJpulaGe#WVAAuOqMDzpDCq{|eo&Z@LdtWZlqv2B-j-0ylV2c{|E3 z%vnv2C?1n4N1XNiUyK5#TKjKaJnNrA7ftG2FNrPhI;;T48EA&Xtoeza}Z2efYG~=F)GYEe>tf@(@mD23_{wXZgBq;Cb+PqXlnW^<|c?{rFH# zChGEUJVE=DCt7MgoJtir5lEF-DZ`UIQ$g#KJ(@J9X1hI=cPR&DLA&ic)2MIf<@W?} z-IIE9y03RlOdV0?;1GU5IP1tAap_598Uya_u@00~3 zSijdEc;fb5a^&oDzr`_RcR|{+R^ID{&|$1Tci_m}*_mH^H^fh=-IeJwW$w*s*G|bq zT{R@s<`rr`kwgR^3NqaHLBR?a1k>V%Y(3EYE=uhu>S|m*|D|Hy-rV<_S5L6xvPB%iVSJdPcLyY+BcpTf z5ZZP#7&tS(Ml@k7+Szbf#p~_ccJiyWvBd&c^YJTua$fDD3>&b&nlcG>eNv(7@E*Ok z?2_QgE*jN$Pd@Tf%2_a>hVK64XhufVIG% z0Ma1_L}8sYj2MxkLI_@0!pGn>xGD!jL&hO|T8~p~mrFc|I-xg;&y*uHuUZ7`1Q$x} z?FB9mZRPm1a}k*rj${RTYCGPVpP1ZQuq4HR~y^E_fVQsl9l|qh!4)zsxO( z^~sOzvY&EUPLfRH+3uMHMNH82iY6151{ivpG0t!CR(4;N>FRwaAb4ES`sEc{r})O? zOt2jFcXWiRSS;8rYY6&HRuQiYG4jlFC{loM+CT6J{cntKNg6-#;hoIR9w9pafGEKL zYJNuw{}rhD;(8j09k~R6Lu(9hn2rGs+TQ?&`k#OUM9-Fgea`jaDXU&E8T*=11-%98ptXswick}9Jv5YEmHdCvfG`76@^Twa54FHoh0Rm9q{Vg!m<{;v-7-~as|bEN-A+roF8BUM6iDIs&{ zJW!ZN7pfRV!F|^VLyzAO%F_2wW85g+{n-D?#q+w;TjO-oztGz6K}{bb@Hh(1ofRul zd|oDrv;2|yzY09N1bTj zuuzpmCrgXDw@pwq4KdU{OyXd4}{{J554OFxZzA>2y>E@^R?UCe`=Z7M#u1nT`|_y{riIV~90e0L{^j`k>6HCxXW+?)o?K`sS! zL3kv{mY7{pz?={J%0!$bOs7~q>_RE2^Dap-*LBTZ2*%V^v5> zy!0e$J7!+wdI+p%(8zPkz;g$ZbOUZJaOG9?$O~m_THINxs-N+!@`&RHgR%pJQ}51F zzZKdPn-2AgO~rs*|6HY+WR)PAS$Cfx0xu-)$#+Vo7n3~olGasA0Vw(ko;4mm{t0^N zBXF$T9IAZr9Ksj3<z+mR!L91vB{?jOr_jbC+V*wi9Eq0{p5)O%6=0Fzs)(J^ z)rR4V*Z(brYr$$c3n{V@5sbhnEt*(MLUfe~Xc#I;gYkDj!-%*w#FQ%1k~BOYxS$@s zi~je&dj}P4h2t|LCROo6q4^BosuVlMm_nHGsOnwXHv2vVG*~@1E$=^aV|KH)$|mk4 zaJ+6vJ}G>g{JP78r*`}aKiRw`yUgq) zsmZn!@7^BlyFCEaP5-#^(?kooS;0@dN&;GnV%&Rb(qD#7oY{UGxE|Ir`3V_B<9&E?Us5P~362SapLihLi;T&T>4#2<(jhGHx|XfMKJN$QArg~U z>@quZ-HW9&W7Ad}6Td3YbI+YcInn`d^F&w^OHPGh0L2Q#QvB#m);0eOs}loHpLR88 ze^4$`-(qZ&Hox<>tP67xL{h?Qlr=XIq6V>uV3#^mXf3Ckv{kY_UEhG|AzBra9>qfQ zI=Zcdw4rdw>oW}tsvxa|jQ9{`!)0|AFY7ah$kDHpo%i(ce0*G{wAlC2+7AOjkw}vq zv~HY`K`LH@VrwvMVr&n>Jza5W#^_%H43H5{Yt$#@FTiIMG5ozpH6EE3WlhW!TE*Gu zK;H6)z=${ZZ+@^@cy{40{jz7nTWt*dvrOC_XkQ==4@weNUOixEhnwqRyj^-B9Kl9_ z?UOwD;nV>mD3UXP+Kh?K{?g)jWG{lxsREm4C{~;jWfZB1&k~|#(~N`(Cy4pCzG%-& z&E`0nei|J7^sqvvNImvkn$x!1S!r(QuODFTW0Dc_6vu?lOH#7~qN@kc7CuKB>HZ_> z$_ApQNx;EBzKwJq4Fgx9%v&_hiqx%ue{BNp`60DPO|o&ruU;T#CL)ti)(jwy8>Et; zXZS?qW`T7@Q`GqzrNdgD<^laC`zoe;NMEc8xKHN293pZLP8$YGI;Bm-R1_TYYjGvK_(&<=A3rSjUSY8%)B@z6&L)LZPI z>TM3%@`@_(ppQ=X0lH=#Vt|cyGse~r90=dAaAB&nJpdE`^6y~({eg2WzLoZ&uRbIOA?BPUi7%x#n_o4MY z4?XC5Us;cEli}Tr?|itMOO6YB&OhrEvOU`+=2|PY+_RkaFf|zWN5+8#%DMsjzafMa zXd^_-9<(b7?VK5nR6yL2H3cjuY0_WoBzU2{4e@no8I*C|p$^y=`N2v*Af;Hd9Az*| z5%xj${^PL z40RHiU?sk1W(NfcR8)&kMrXigy3n{aw!*n>AGu%aD6wNzN}`T=?jj@W%bayLryniyt2fS*(<+ujX`L^X1-PFL}dU zN=VfB$B!NeZ_tl~Pfs!m&i(YMaBj}+v1rD_b1@kA0YGnKGubbbLy>7kdb+8by|eBF z&X38te{=e`dcYWyb@|ulwSxj4sz@Le&0smXnI17zcaAaWW_Q7pL`z^zIx~TsyX8R( ze6ctr)tSK{Vj&eS9Sp_U7%Lt2@hqx^HUemYNZ?NB+7A{=s}Ahs7RFc_ml8tF8DNemc%$qM5b=#ei#k~D zBlQB~PbOY8rcF#Z+`f+B_mrL=j#OSCzwhZd zm3wF01m;_L`}vX)Ty|0KjsXC2iF5$gP4@uSeF75iG}iJB5v;^U&Bk=!NtEwI*!hw2 zu+|k49WCHKw-t6YBb%w5YtgF-fXP;~{ zNG{p6%a%1>y@$qikhZ#b5xmAfa-KSf3&GK#GwgrQdHUF@1pBT>Z-$;6%0Splg!G<+ zLbHmf$+!-5kg!@OT$p_@=%5Ni-^pUZ$Li|NGO(|vk9vwU)@=ji#`p)bfL2u54*Qbh zOFECS)K)_Xp3+(ZpwQhNBoQj`*#9zeoG=sV)`L$vkVIlW6E`o{D%KEsTzjN__L6z? zC+}5HcP4Fm!**I<))LE!xgeTHkVX78fFD z=LijqM~$=S(+kt#(oa~r>39~T^uFl-%=$csdwyBf%%8Y-=2Y9Vm~U90p;a;|zfX>^ zPT+3-PPs*04u`SmrZjO9X?aEaQ&#@WXj%CWc?#yjPkE1}!L6JmM=VYt74B*wg3CXI zg|Qe!;x;BeC!t$Lw3w`<0p=qP;ZIr^3Sloxee@o@v%ExI_sn@-cd#i>(PWb2$2(a{ z3o_4$>YZNGctD0S+n{mZ%3{dqNl=D2l+4%5h4$M2N)HlG3BXa-T;PCW9E7^;d`Uaf z#CETf`4se#7GW1e%6nXtupWHtG|H!4{wyu*h$Rhi#0`%Xh}IP}Sh($&(3`{Dv&qJs zHMiGAe>-dAx(Rk?NI6Po_-}obI!Ha}B#I2o`mLrVz3%@;P0N3>`~NseQcrv6ShLpC ziy$eYtk8-)?KxsO?FQE1`R^`ABiER?MF+I{K$I$_86SNfQsh-)t z0osy}YQm4UU~T3!fVN%nAjGbvd5%K!HgXw95OhugedE!=s&()S7h`c4^iC&_VBAiy zzB>NeI>{+_%;Zr82mKZtZnD(^6VHpss+~dLe(b}oRCxtu?lClhr)mOKz1S?M!A`sO z=d1N`l~-=QIy)phZ_>6wRsU4hK{Z+T*;xH!0OlC?f-e~al3lPL%$_P^!ogL^SEJUx z2T=D2m?7(di}!%@>VN=t&J@MlsDeQRpUn|#naMyacD7`0FMc#lb+sg{zZ)9kAoJ8(;Ir<^9<54z3z1-UYQPf(q9mVy*+SWI(g~WWG9AGdv>DAKMXnE$h z-5~uKkg+dT!*rk_Zx>{pGao@lRS{a&S*!>R-=OsZwR-}l5*aS^tDKCeMokUSiWquw>q*Mn(#qKxJU_~PdF^qpmd1qX$s7T# zp>hj$lk4Fx`8_&v`T1+`-P0v~JEx29p}XC2qwG?I zPRm2e+|_Y2mXDTjyjD~3lbnEWUYi=5r|)^U#eMFvWUZ(3E6{8vVe3xH7l8bK$)e$l z@+E~Le41d(M;Nu0P<17$W_c4}6SStVz8MLs;ZoJeGGsb$l4r z{qg(!Qg1B?C?i0}Q~5Mv(kEyiRg1EwBW-XA zRPP^6iXKty@}x)`D4l?o2t)703?!I&jH*+-h%gme9V4K1y=Fk++50(PGY$GB1%5%dEo5MESj?p2DMBQPaBX=iMpnK(^l9a1UjNWn|D-BN#5o(T zE4Y0*oz|8Lhj|-zFa7is{EznuCgrWF2#o@6_<%6}SG0ctTlCApOn4g!ze5#$*ANe4 zH9toh;VdEAf+R&+w$PODJ1gt+_js>;H_x=kB&X*tSEB`)A=M7OnSbL7Ay)VjfF(+;aO8BggNoe zj#-+nUk?U?sp+71!0e0wmHHG5&spaaOQds;6i)R{g`vI@o3e_q3FZD0P8D46$0V&w z3nU>~C&$6HFoLX=)7(d{I>##zCp9r7;5{BRRTl@qJGAgagRzkXv?R`uG1y{IVKL?1 zL374Ifc0Pd8qG*|zZNTTa-kD>ri#{kR8m5s+D2(M0Q>oH6K>!Zd9EQ$7#RtXbb(uX>!FNh?c?Qtx^T&6@+^f%% zSC0jFG=pOpf>kxRk!Ni2eB8aFcrI?nS4LL79Xz4)^y#gE3#XGN%(yho#o)lN8&=5t znn{_;9K8Mx<=)~5+&V2fr-NA%bB3SnakRnozyyj~RMcg5JuPpyQ}+n}F4{g;3F00o zDFxXkb^9ZqJdBJ@1Sr@p#W_bgU}m@Sjy7)BH#^WktqFLer00aUleQ5;g!tK|15ZKn0AKwD_S4 zzDi22#A-EQ2K*+$helX~;0r%9#SZJ}hwujFF_85p=&QWPwDoBrPUSoE5?!A@q8}P} z`()bcHx%6rd?D}iVb?^{d8Z_W0Po`!9$Zv&x@k$`h}r`inOp#|uk1QNC|aViPUU7J zR7nhH3#_W)vJxF!Ak{2is9jKp3otv^nu`*tG&xYklK@4%_3t8dBdxzv^kwMcNGIro z1#%^Bo!kY)-*NM%f!s1(mmvHs< zY(h*m$Hxb9CaiXlgZA!T>gkTE%n{%;V)@Sqbng(Xvp(FpcQ&rW z_khN&iFLjAD2okU4wp|^*(o_^Nl1; z7yS^kGrP7q$emgdmG<>Q{P_b89WPMHbm9d>5GB%A=r4UE9|ZuS3hLU6Bk(Z8XE)DT zD}2e?tWWKq0!jm>#-Ax(0<;7Gk$ai6#GColDYz@Q>ZGNf@kU7tCgZ0^X16>06=b%% z%sgu)AMfO-bZwu*>!{5u@(S;$@UYLSL%U~T;1WydK)#uNk&F_6Byx!c1jC56lJfjd z!6}Hpbt7=eyzBe3z6ze@f4W9js&iH-I^ekS!9AOpYidruw7!={cHY~DkdEtUY)6m9 z2XR(Wl&y^mh~O_<@}zkCgmdqAPN?ai^~?}0g1f`9aW6#}mZ-}3H(`}Evdvq)I?v8` z)5v)CplI)@>ASXVQFGwNe#R>9EnIZ1dH}3W0@ev@V*!PBJ_$1hPMnMp2k3YN13c{H ziN63Hd&p)S693P>E<*9SbYjx}+RJzwHz$=oq9kG35E8^pejhO@YeZss-zSInH_>9kJa-lFMje*KD7x93Ay@5QK+C&+!P z(oD2{BXmT)1i9Y9ATr+51nc(6BOu*iw?Vk+H(`zENNBN6SI71LKib|r9_sb|A0OEz z`;v8pQX!`zWt*fzk~V~xHboOk3uW>Wp@fNwQjC(yIxUtG#;(Y*hR8NrteKJJm6>^+ z?=?E-bl&Il`F;L+|Nc0S6K1(z_kG>h^}M#{HN%e@^-nC`*ZAmo5iu+;i1ga-=IRE4 z=B}+Y)hi#fbQg%2FS?iksWYe-Q7B!)vgXLWz(jxaQ>FU?3Dtc{86&r;yz%Z)%ONfSpWKs9AD1EMg< zC~pW}i0aguoUMt=_V$o&8K>!e8}xb<>NTF`9J~31yr)K}q!lrBy+z*P-VlZVKL(ur zH>V{?@H2F(9U_22j{F;Di00&W2Kw2=wCI$y~D5f*Ievq#xfgb!oOJ zAozn!aEY?%=y)&y)gFl9-39f)R)WONawBY~93KWSf$8sDL%U4K|H_e>t-?cy(G!s! zMx=|~&t?q=NDY;$6kXdxk6*6Sqnnfe!YqB)`9pFcCJcH53VUu&@O}548B{|+mzMB4 zkJzA2g>F*NkA4x7+>K|cTpc6qxfM9fiIN7m8^MiGk3 z(Jm}XxO3WPfnkF0lSj7Fdlj^CqHm3`v?+*HHp0*lQD!RxUkw$^2GPZ`%)-s3EQx39 zXh2AwknFd#p)xz*%I{9=!&Zw4+14(g*I%XgryzyC$R|o*El5KlopYT2u`UW^XeH?1 zb@ekFxLKGyEZ^NiEwGj0T#3kdDVt}TP?Ce5cqcl|u)gJAU({l&{N!yvV7SLouFLs< zfJqAju=05!Rqh|7%tWE)wuH_+LjdO7|CoNJ)gEH%fa7dneXVwob3?TT9>oOFMiqmK zujyL|zkGSaORxJ-j7uy|yZ@x_Fdhw{j(_OxpN$5%5CrkQtZ7|r(P z4_aHlAytyu0J`fu){~4${{3i_`FBH6(XV5`zMR=9{9`FnE{uQD%2kPtqFA6lb_{VC} zF>e)n!xovBdt>;HT9nBWKLNg%nBu-hhW;s+^P^eCXbQnqNC7^i*f?NNQiZ&Vwc|#* zH|vhC->Rvsmj}Xyo#L&dS6&*iCzp;E(7T+mg(;^%vSZO%D7((-z%J)w`ogKH0+6!j zpMpLiJ|M}qM&T|Wu7T4}UqKu%Br<>jAV zNCAUdb_d5yYeudIQq7U&ACb1>^$=m`NJh%{(LXCwRfV~>b5!igaZv9vLQN1jy$6X{ ze1XH*9hUQq68aj(ov>Dr!`9vEC zTPMM2M_vf-c}zw3^seTdacc(NP7)0k?#4?$T4O>;C+h(+eB>(~hfsd79{tq8Z6nw0o z-#p9Sjl3-g{FTko{?j2G;_sp#CH?>u775|6HQvNIXL+PpkN3g4Jl6c94nH;I%BI7h%wsLd2 z@8h;ywO~oKR>)Zk(=IGA#?N^Yzga_*V#Klet33YeD)0Syhx_KVdRN1njV7S=i#EF! z=zuEMVrrl^kY;rj1rzl{(D!)e1fjd|r&Rt_qCcf_0Scik0WSB|32e!bu`xk}eBi1= z+VO7r0Zk6%hAdaW?FJD%`TzAdqWV&{2#Rr#t2SPP>TB@gZxNO160ZpO5 zhgk%PA~3p+xg;RI=CizW{JTR3z_~KQw1Z7Rel7-o?65Fy_tnRw(DK2c(F7-DqhSFh zH-X^M`?OinGTxfQtbV?5rt5ujb!~;@Vr_536Ysi=`f)-6@#I_>$5n}l68s3A)J>@4 zK(eGOnm?doS8u+oPP9L9-wPMbg!R(yYTHA2IL|C}HSru8hHW(No1QS>5xc5dz`4C8 zu_6dV#!ryp?I6D&5hjz;x%cT7w6dS!_Q)tp4-j(K++_Pm*Ic`4_XdT1M|TblS?TJE zM+ls>RE0k24UceVRmgwf8uvol;5#Bf{}df)JwHFe9_A-VJW8MHzU*NeQ#N;<#IXtH zC0)Vd`+|dY&0|IB3v>(P$CA7IN&|ZgNi|MR$!|2ST+s+Cm&bY_UW0mp#u1q27B#APBjdIIa}rnSnJ~n6ym|?+yy0NFDdD);po3rarROY4;itjZl$MkF`Hf ze7VwkT8<$F0$Cj_p<6z=UPMuj+xcVXl^X<@L$egsxR-TBzg>ygEn%su{6p`oJT^ps z>fJcDi+LBT*4F~JQVVn<_-QR@3-Jx3iz<#xEbH`rQbLPMugjaJ zMuEEBHlH?GX=Ri<2^!kUxb8D@W`@tvY=tFrU%U4woj>&Hlz8)05`jEO=wfE_antG)=HAT+vLyfl4LP0nFRcBjp( zVLo~lFhG+fJ2D?ppA3IDDrv~Cp?Rx5t&1dSrEODkaS)Kc9!Sfi>>>_8P*~80A5*tv zwZa^n)4;U96#S@7@NAcZE0Tq4@WM#{$h^`S4dh|)i-I-3&l*Lpl>~0E>_J+e$owA z4}Gdv%D5;zDNmQQ+^A@pMG*GxoStwEcYF1j$i1OL$Jh4W<@IBg3 zG8d}j8Ud)MrLV1vRt@K8X8fXLzrmtW$g<4t$`^rjh63%_{b8qWa(DfY_+EjJ8{J!M z9Jkt*y`}Nv3k}8&AYZajJD%f?aL&<}!T^bED8s$W@ZLk|9fpXeJQBy3;A?Rj^5b(L zXXTZ2Rf|QzEs#IJU+EJ;jy1{;TNtxE9?qT0Yo5}5<2UhGx_9%$=kD&;Z`eB(POK;s z==c$V?>zRehv&j=3iEIQuQIfmtmmvvF>r(!b4T=htQgx%?7=6(C4*74G;C2Xx`dTD zKXHJ@F-7U@O@Yo%OhUL}Xa16aY*N8#@0s6}PJG@Bob~cr?!Rx!-FM`=F4wenyq>)n zfJJpDlR}&mhuvg#agunR#NmA)pV#PeA8^9&T7K|jRG5UJ>YZNp<1Ubv&DzfI$p!L0 z=$nzy1I=Z?F*X=G2q7>cG_kCI4z5-{rr>dc-bbJUVK!wQae~?iP#MaC!r;Uv(OIU| z4A|+=T^wiBlBrTXM~LP)XLY&i$5I}6R9?L3|2AOZ(}-T7^Qz&3JRlBhTD6BQ9^3^c zBz|5v=Fu^C4Xl-P_WsV^p>S2aG@{ zouk1DpaiR-lr1eet0t6#>U-9F5z!K=JFxJr_CaUuOK(Ssqo%ul?E1Q7BWFebfkS`0 z-4ds(gMVW_zGOx?%V<;{fq__wNA7=HHL-NxfvdJS#W>)-)bPL1(P%aEg&Qr&d5?%f zCJLT8wxb-+Jo)G|eE9~>sg)r6$OolnnJXVxL1XNFFdmK&Yam>L^OrJQR+en75~c>B@9Fmd?e?zc?_84 z=07oz-Q)wTpF^K(JHHD#8BF9JhlQ$8(4~@Nr`ay39VX_&G_ESuL60#m5_UXerQ9&8 zZm78xAh&C4qL@oh?{Ar@{pXBr6RT#g z4|ko7wl`{MsL$S6tt|fH^|N_2hojvi47@@POU|vte(jzRY)%;qEc{Cp#ZpAx^Wbh4 zXB+1OQj8F=5*+dzn%!k*YPs>8UGS=Lv=JNw@^)qkm;3ed^zzig!uLMf2`;$FYMj(X^bQUS;^EFpfM+@=ET&)2Gi_DKYSHl~(|X zzXl}!jL*?p`$mW^_nzc<1A8%l80&C_6*f4pSSqUB{C4DhH+=Z@6r?z~9aF?^-5DF! zGMdi-gt$2N%#!Kpv1^M;ik0&p#@B1zTdAq{@P%v{WO7;#hMoCBuY_r%r?q3?im9Ti z3xROX=K+|Q$54kw*N@O|%YURHU zcia`fFD1mE#M!qD7?pka*Tyla)$~A!YpH+lz+O4CA3l#5wV%BI^K|(zo?(|9b7K>| zkOD$Q&#`@|J0r`@4iaf=8VrlQ}C9eC*lf9q5nmyjv2N~J%EVf;6H{lsjjhdl@J6f?7=x*+quBw8AagUwkux=N}`aAr)bzd0J?j+8fdY#wG8^N>S zxZ>nPDNBYLlu%s{-@64|s?}z&HgjNUHX|15j2+Sfp;<_*Fc;?30Eh)F7yjVYu1j`b z>bc*4zyt_5%~l7Vww3&m`AaAt!rb(+1OL#YNI3Pb!EEDKQ1WRFiR`buwE$dvgzvsP zW1tqS7YGnlSb@knmx!q(&H%i9K!ZL_RTqTS1z&IQm!7r$>q9_S+Y@;9ivafPulHUL z{@C~I(lE%~FI^q{zv$|3X_Dh~w(`gA2}}TCci%1%%PgwC1G0;gnY5)n9!Su=k`TDq zWW9Ly1_5*dK9s$9Z^nVb1+Rg!`mS7>@8}^& z|JYiDmU<-43~*yKZr#7jD7+lkbnC#`Yrk2lN+CgAC4CIO+K}lNf8cf~{_K(oUSe;+ z(J5klGdr#b?&EM4WLhbe>{G!tH|m~RvfL_szirCQeRdQM7We%^(pM#J39>h-P&k+M!Jln)q|M$$0SasWWBA2k;?^6RN+2}irHzw#c*_XW z#4FJc8FXv12Zf_1PGIur^LUF6eBglz_F+=p@`IozqO{fVsvgy zz!In3n&kqjp}O2xcLMg1pU{g41{f6Hgg@r|Ry%$lVsO;zK2)a!>@&EoU{3KftfM?6}(s&aom|LHGCoADqEGasyRRFyuN`xSwDS? zq3jA{Rmy`dVp@0Ypkx(3 zk}7Ftojc^(acw%Nhp_W$>#AIR-<(H9af3-phhySxdTczQxy@g9a-+j<0)Bq)I@rNT z`Km)P^!|Hzex(o{Oe#j|(4 zZrLm^VBBNy*Iw`X>)7}?`e2I@AKV7?p8(pk5ncKvJO?FWqJ7L`mA%?8?mEz4FjJD{ ztMe-i{i`)AQq$6Kj+aJ68ZLGnjHVW?o_d@q*ndV2$6=KEvV_ND*vkSP+={^Q#g?Ln zceI^2;^nKKd)5uIfm_xeIpfW*%MVP@&Mk{t=v9SK4WWGvERZ#d-luS4=jg~>AkX?A zqNBckos~3>KILzm)6tCKSqfo{28Hgbl*J{HyILm>yX1WHN%oA|_tDc`&34|O0(-jP zpkXTh4&CGr+zC3W)$YK7^XRuV=O{kJb8f|k5N!XN=Eox=*8QjZ+OrK88L1s6m2CZDx{v== zMMk{Smbd0Y{-j!D;5q#WVQyuL6o_08fMj(V)LNQmQJ@3A*m^N*T{y%Lfc{!I=mL+l z<{SY%LHi(!MLoVuUtDVosV*K!AY$RR_{2w z!7*8Hio2tI7nFiD|<{mf_oVY$0fXBGf|kubLC5jP3(YK){a_Yot4SF$ET$kq?M&#^{= zeLiD}0X_mj+35?5Tn_J5TF*%N&cX@&`!;%g>haiv`Twd-S3_JIyjccXX0Qjq`%r;;2|kc9_zpf z1}AD*d>X5~$s+{kYYco`9jwIM8V;!72OK)c2>|aDs?wZU zkpq)fe zp>akLZhQPsVHxt{sI~{BIQVgp&J6`_DCME>=?-*<=UlY*M&Jz9mEFc(edJ+ zHpWDf83aiPD~jbXO{-G$=!Gp)qXbzuTZq}OtQ5DAf8MBdJySTt2&=?1KMuX8OoH$@y zQev;t-3CzaIr8tnT65MpkAWW8vZn(2_MU>Yy~kvOy4#xJJvijWF^;^!OdK~Qcp2!6 z4W&jLbJTCJQ34I~XeZ+{UA^aglQUg=+QgXEPI-HpKTIjJNmPzffh9tW+b~6VlNT`~UYo@<`s1N4@?1fV*|~y(H!Tf@*)r>!x4PW99Rh)W zaKpY*{;fZ=7S6DM?rJ#-sNqloZN4fNJ+S@TkrU5Gg4jHrOKR-)nGGBQ)?c4$G5fq9e`R; zou1w>-FD7tS6=J(av8`++==DY9NG5swUZGUnNNQD=mA6lt`79tE_G!0-B|wsHQ_U~7rrYC%=0DDtMPfYRX! z0T=rv1cxZ-VTZ0+h;jZpP|(tUAE+|2h;W$J^`(!o?vfe!9pwF2l5SI3`K$atN)lxwHva|E&$ElBZ3a=P^^YCuj+`H8)0)kY+ z9e3JlS9RhfY>hDWU~_IPLb}R1o#P-`(Wq!*&aKy7B!h;EXIlzvIt@(G_MOQg0Ilnp z!VpW?Z#7sV<6WwNzLNJFzpDSB4~M&DYgXAs|G2g+&Q>_o(Cg^#+=O)_b(I3rGNDj? za^{Pn`QG5YbGXArHQ}8h!rCbz&c7$(3lzr^y-&^*66MHC<({p!wr_u7h`D4J=cV-m!n+y4G>1(;0nSh;^Q)h7u>Y z@+NE^aDy>3u_5LrAjKjdX6{K>lUV_5BKEfHVp{T^0#uv^G-6x`8h`fizyFTsTqwQVYGh7uc^0|1a2N6ye#k-@8)gKNNwA`I}X3&hfv{?bZ|X|snqOG9I>LCTP$Lm!ACkoq_L^Gy&6z@y88 zDcOM#>KSG3*~AjeE--EnUf6s;rFQ`60pj$v#ABoRbyqCb2(cU``u*_cHt#4?TaMFF zBYo|0o_WDd$Xn#})v5zq8t7Cv^Se_-(D~_W%RfhL=HY0Nxc*Q+`6)FdvV$Y9)QOZ^ z2XF94J_WQxu?bLax%8 zyXY~96fg!Ztn$!kI!jR;Nj#^VaQBgWk4ul+Thldm0<6u=Z&f55Z(Y%(s}&Yzz$_Bb z&KosBapVT`F$te+fZq0w7eB`rwhSSaNpf8OLvbwUY5hF&(-m-G?13gtouz+o?jo0Z z+3@zUf>$y3lXTqnRcgUlpfaW+zzsjrN5D4iw z<0mrQ;hH`ecw_S^!G{KB=}!Ieo3<)F72Xr|2Tl)<9-~vJQ(9;kY^mg@54a-WV8MNjkF;!4rn8S;E-DRdlom|F55sVnvSD zoFoVBh7uU?NW};P_$K6QO&`X6j6#tOWbezfh$0qxs5Q0_-=E`G9jy3%s($&D&q<@| zo}0|7^z3^$kIjw(DZ*wQb9wDPrId8n?K=4dQO0UY(tEK3NAsTWOIBIvF#9f_scA2E ze&Xhx;I%2g`eTOp#%E=MAHxMw?tS+-4-q<1rX&_gkp!&vI|xXuGqicDA*WAy!`>WX z<;^@d++ma~?z?F(sGX4j)s7!)s8XX48>5WI!n+}Q1fx19_~CPenX*{qMfjJ{IVh97 zfho)O)FI{ycZYBUWN7iE|a{?-)Pq3|@6;BY9Qm$cXpO+DuQ6 zts0=_>pSl`&yB?j;tr3NYJsN|p7+*>CQEc1Za+DAvAQio-0sGuM1Q#>thY4y{)jMC z#02fz2&=p`zshPq)MNqa8WGhJuHKOI(CedJ`oPUwT3i!#2K`Mc8cAFXL9Lf?&@>Ey z;$)Iejb9kH?7e2QnK7dTzq8~X!!(FAdfwAtD0r{6UZ4O|w-b6!2_3j!fvY2W+@5n3$H{Quhh1B53D);B(p%o@(&zU z7hz9P2Y_lc98QJ-nE69&9_K1G=M2oc`Z-_^j#}CbN5>VN3a}7BF%0QHn;k`!KI;clntCf}#AXsME22BqOTaVTl64$A=>0c|s#6yw+DYB7DGNV7+_^DEJGLr!!Gq9u7ei>Q0yw!K6vS{7 z;R;L{JPRJ+DEITyhz)JEYa*Q%udLLdq=S1^K}dz7gPZu-aik)IQ2yEp)nZXMwtBW3 z^@)r+IVFsix_o{5GS+ZcPjVVg(x3EB?A9o89AsFqzWEFZLA9y1cQo?CmB{-3RzN*! zns|Q!y@v0u#!kU;U7Pp$@5f^E(86V0_jfdp{IQt)~o1mbY0;lZZ zqdibH@gX&ps_PL>xX^V3)eiO+q=IlCX}m?}vWZLpNhjM%g*c?<>}BuIBEqA9fQODG zc;~Ti4{laXIOW7eW-{n6?y+gS2(Lv7 zi)2Q=oD-G%ZA76_>)B9d))H;%M%}hc4aikvj!fA8_AemWH_1{_q_-d3s^<=9ys47@ zD)E6f(~0A+y@1Pla(tb#{mr6;46}3XO)JhXy%nmta?<6_6&zpNjq3BqIEFf9ggJFH z0<6uHw>D#t{hbfv4Nx89+7*WmMVAVYF&ICh_GF4njDrxCBCqn=Lar{=X%(RT7>!H!jW6%Gh zSL3lhylfwskZFGUt4`qq_@a7Ke zVqX`%;Sp*$;L=IqjqyZc6{?Oh&_tu9A zE{3BwH*a|r{bH@*0muSr_rvy`;Ofj=LdBEZpBu+a;WpcdnB1bxT5}CEiFr=p53Jb) z^oz|{zmq%vZjCE#Qd_c9-edAb^eg$S%pT=1QNXKpyh-O$1*mV6Ryz8FNN4 zV2Pvi?tET_+JYUnMxzHp*XVLog2m8CcO<%0+PUPV7W;T*b~f^P0PgUBAxu689PsDl za}Q9meDSubHb-uSW`^3D;e>bFpV!5j2z(DttEQ}LlE2v>mf;#IGKQRY*Qxqup@N1- za{0lnnX+?6w~)88gi0-zkjjYrkvCgMVRf8ozY931$AF-hmqQ3L1$N<28=Z(LOZIDw z4#xatwCUw&;58zCk_)OCA#%JdH2-0*;WGm9zd~TMdj;zuU+rsNOBi%h5 z9W2Qi_2iHeHehQ43m8HYlpNyo=hrQT7@I`2o>=5pfjYMW6Wusn%jF`h?1V$LIvJmu z1bfzALl-m6+Mu#*2rr0Pa6}7H(}DFmtlZGpB%{#h1?k5{G<*02l-7vQn9mnoit13; z?6~RU)*T|eaVf$A-zNmr^<`8TD!pnUV8c8H{zWa2T)>m5r>t)x{QQM32agOafhYJn zH;7?$w1px~`t_J_{@*cprft7LimGoT#7Zsa^(VufZA}(KdoqKYN=@6Q8ihs^6*|T)>2D2+fJK{i<|du7SMd8#UnWz3 zjDKz5X*=!PZSnRGU9=fc_?Lw#O$>hJip)WHa~@@$H>%FBemLQCCPp>xnbTH*HBb9P zB{jm@oY`Zxsa~7bY1Z|@9ZawV8WI_NUX9{RG;8YkZ;DkHa#usElYf)&m2jQ(3k~!# zgV@58OW)H%^M?;z7qYfD8NPMtlho!wji||&k`vdFJVG+^T4#)&I^-YxyE=c|Sk7A$ zi4>RQgR7`2!Qv<~V}=s%kM^!=K0!OdF-W&E zFRA9{K{ms+MoCSna^95^Duw;KjBFbB^d*ePBjHFSVSFHG6fkkY&}xKr1BI#@ z-nIzz{uz$stD^h}7ks($lA~U2&j^RFY2WjA;OPqm%D)xx|M&&HcOfRt6~@Tm;ZJlS z;dmxxi12;T52nGiazGEmg#kH0nkWoOj)MFAK*YmPOrlc_r`sz5Ur!=SG7GP9c0KfV zee|<#&a}Q$BSmy~kCnCFZj&MtAoUh-9&rX&Ns+Xi!21p)io%f}-S>VB5cLzhdd`V& zkD|0$1-bIl9~`Z_R6_W0lIaj$HRc%N%dA>!=7cfv9B4Ja5RsheT~vM3H6_k0z^U~~ z;G^)%aYZS-U{P2n6sUTbS<(8T=HgFZ1Y!)pp*r>);F>@=DzBIALM>0SJha1>;7Q3a zDCAuU%YI|QVEwDnouapt7D&hk!U)#wb!Th zhscZfM!H$Icf{m+F1^#ULyxMuT9e*x5<<9VfSPnbWeXM6a)`+`yyoru$mf`q=K7tl zr406%Nd(d+$$k-t^l6ywt8euOjt^GIAaTbgrg}^r3)WqDDE+abbLrBJBd1!ITvBxs z=4f~bffES4D*Rg@(c2`98XWf!VkH2tNkmq3{CrQBZ_NBnfIrxb>0vcYUtj<^QjfU9>0b-$R= zsFtUz{z%z&-^%C@+k|X2`~zvzRd|_T_1O-vwj9xGFj zXda4Q3ln&y-GwY)TzD=hPixnhox=Zx%rAGdAP{M);?i_t?UCFNG>*x}PpZNOmE8t$$mDDTOddlfqz0bG zLnjKp~gbfbi`FFS-)C zr7CM@#v?HSYuJh1VCvtX7YJWWc@nxBVL4*ru)Y*@2zhs$mxx&436BdIQwvt}Xb$XE z|8_dZ5r>p4qQsJwGJ76T(?{9^WZxz~t2Xm(xRioBaqle5D~s;dJANLj`504Wko zLXD4Q91jSPD(M+I5`SLJVNUMr^?hLt2(5E?quj)_ivFq=yYX#D3L};z^1!{ht(n;8 z$SP`NEp|8ok4od=5)k8IT_^KDf1$#G`JfZ$59*d1UQm_~?95uh-m4}8Md~JzwWS;Q zwOr}2TXzsN6PAC#09}u-wormbO2A67VFM*wyRezBn_~_Y7T(fM*vj6kBNAn+!=EZd zf3ujg7QIUS!FL>wE`rd2Eb$qg>4#i3vtk&adc?9Ge3jy?V*8EJ-FnyeZ=0+c9DAX< zX2Z&@=RY~K!FW^%_WRAj@V98KM zMH%7`-}6ik*|~Ov#6@bWD3N7<;NG08#80#4AC=O_Bu+>K27x(Z+7M!jO2ZENc?Jl@ ziN*#6#;=g#=1Mp=2?^f-Vxk$mdf?XrUN`aICk*d-ZlO;75esYCPUJZ1aP+8=^eMGg z_`@;-Y$d(?F(x~qCXw{_b=Zwm1P#)F2mSBkeGZh0gEi{1EJ=<`j<)7=t29LYPkYOBbB-pY(XRHZp=d z6O_K3*E+lM`NaNq=KOnSr_2Xv`lO^@@|Rxntp z%ufxfO%-u(VoEkRI||@Wk~qhCqTI(70(i?@M#NTz5W0XYxOI>yJPO8|9$U2TT4tAf zPfwtMQ|$7CchqgwaykVaciU=DK|DgbREe{xOKiyXQX|*&EhxQ?Aw0~?tWH@0d)fb( zz&F+4yytaM0kx)Rlbu|vXDXo{c~bmOZK97bw;_?eX3xOL$3f>_akJl{ zD1xGQ_U}xaOx$gM+xEVYh+Q^ zACtBYTVSyv{nF%8L}+{2g@fgfp70O{p>@bhH%(EIOr2m(PFoRS2(B6Fn6 z{&Ri~J$mZ=(xiQA($cY0}G+WsyWvz44~5&R13=v`Px-tqM1Lz}a4f`j3N6 zNPXVC1_;|umngpIBfH<-EAcH5iZu9XK%HECv9oG@>=xwZ#CS(PMX(#Irptc~vwm*v zAUg8d#b;1J+IyLZDT%WoS9A@Wsn;Jk6WmC{0(t_&})s(sBUo}orw&sFJ}EMs@b4f*5&EZqu@OYRvfueK>ygf9i=dToasK<^*O|1H&Xu&-Zt*6 z#JXFqE*925du>@H9H_lAC&DeTT)?fAcb@wB1T}QvtZqnA19Jf&yTq$dR^0_IaTLVO z85pyYUXall~2?wssCsX&d7;1yLn zuY9*J;c)zU?WP61au>~IUlxY?pTteL-$qHSMG>EyXm8BIX3m=+eMXX%Ws;OUBj3oK z`HftzD?}QANC=fJo9K6!zq|FBcBh-a4`}riJFEeRTrTA3I$SxN&_n|>M>`KAJO(|l zKJbqf(dZ9rlO82b9HzlrD>s#BZF#!5@CN4o02_v0MS2Bk@Iofpi$(UrIkr#Za6`gf zM}{}5yMsHh;E6zg#|iqPQjIsEHhHEqPM@UxmI%PY>W7g@S%_KZX8hy|2`~u71Yumb zFnQUeSG(Shm;<)!pbyKum74_cVBwQeW%4Eor953s1kQxFwE6hWhX=-yGyY70-J#i~ zsx9ThPc&Qf!aTy^HsCL@z1j)rA~-2o_idV6L)Vu%?g|mc@pbOxKdQaC>Vh)tHKeTF z2Iy6ybT2yIP9ie6u2<{5h-U955o7;J5!kG=+ddP^dPU>j#lw=``P&v=iFY_X336v# z4p&kt&LvM5nn($sdmV_gByJV`DDjGgzzc42@gnbloPD)=uJWNphO7uDh2iU#8etya z4MI2)nkT&4%_TXnd04yyC1I_l>saZuaizi6Bp;8N^@AOL?;U-$#ZvcM;P{^6K=3pP zfS(6r0vveQl8%CmNKbU9m01|KL9DJZDkja1*b!6m<-`nC{YGSSr`z{wZGM!=0FiN8-1|+nU=# z-HIHD`!SXx7UBwi0|+s1%#2tg!+?v+m*fcD;0>yo1r2w3VOiglAP@F>(RmYJ7BG#u zd?&z68eq#Hy^u`?sFrf(7wVwADoko|Dtu|B3IzLw0@Cwh6$bkvu64-JO(GIxY6^su zUio@vHflM+hZuEz<9%uJu8fe(-*9ZJldvIiyb?+ONuaAh`fmplnEF6IlGPuc)BeI! zH4fF;;!-tv1U+MA@~XV5X~N>{O~)W}ys7C~i>*RJawll5J4( z2CSzQpoS&^TUk_3dL8se=3C2_gcU-LO`+M6C#@ILm=^G$v$dO5NB_Ro#BtaHfhQ?J zG<4Z)x$fP^-bWA-@9^ZeM|IwM*qYtgh)>l!xt_)X{1f_`LtwSv2a9Tfem{RWt+zou zW!?Ka6H6^!{*#>=;pIZ~t_QEOsDh};WI$1I%Zbkkn_QCnWA0oXv0Z5kj)-^|v|Vq< zqt{3N0=(k{`0}v4BG~uP!yNJFr7;hZfHrg`R9-)Htt}V6FF!jENooks{yd8!QV=P~ zW_MS+QLN!&$BpuPB|`YhQbnJ@R`M!M|GU+u|IL|CJg;5yOBN1)yg7la5 zx@E+^PK#4r{(GmQ+PNn>^4PA(fmw<_!m#FHGCE+gl1I<(5A`m2ZsJ?wEja!{YP+*^ z_7j_-Z7;Pb>Hby&1{GziK})oQ2#4&PclVs=%;OH1&0HQLLvY>yj|;spNL0En*YeHm{0W+s4(<& zriQ&2j>nj6_q-JfNRK7jcdOd`Udvr&aS7&L+vqddwIA_g=D!^HaiG(*?a;ex-#&i7 zxRC4{1w*-?!9-(?!QuUhoR!4zi^Xfxk$yQmXA2GqbDs?(yQg>M&pb}|aN9_|V0$1= z>e%}`8|8nOj@9M*A^l$sB2#cpfMH_@a%c4wH?@wh+HG%e)jnv^M}NWD&Z>?MiCUq%Kc@C~ryfs{ z+2d}f`I~BV)@os@E8{Kj{w;+n2z(Kq>662axC0mKHfkJ{;hsQcBEe{q%SHA3inRI@%qvTZFUdoHgTX^=^N% z-LDnXoH?qxg9gGQz+pcT+int^bIh#8%S!>U=p9N|>d^hGz|oLXR(j2ey9zjx74`wU9UI`4(4@zQ;XbAd;Zi`+<+lQ>NBtG!w~Az1 z)Eh62g|ffpcquc1^U6Ko@mKSVJgPaIl$-vbw>HCDQKsu-Re<)>*IP#}c)HkVF^xV@ zuTkaK0^Qas*w;%hz_mKVhm{hTCt+M3{s~Y{W5yr6=ko7;w@s_yEf|6l>E+Ekr)O*OGrlcPyW_`AucLnO?N&yne2xY5cz1X&KW_`=x?!b@LW2$$3{)hzT{q zBSK6UxPwT=jsq8=1H+wFXwz(@>sJ4YvtdoOfUI`BsoxHH`D?3Za2;qnIPRq|w-YxF z)+do?ect<_nza~{jZ##-?TMF#uw09nYGTP5gV=IPTsyWjieTT9nmmz{2ByO-zgSmzQ6E_~|xjvk=8D_hy zy~Rw{biWeM&Q!7)vxGjHlm?a`BlS)#N4to&2Rx(~bCBnUj=Vzl>A=?VLU7~~tKC%8 zJnFn3F&|~e)g(&4SZnk!WMxPdq--VZkt_H5B#SS&Q)blf16-@mD}BsOnrod=K@(Xj zPTc31{BY7Edd7aY{efAwvHQ|r-)U~nh4!iJPx4x!k|4RBR?NEt2R9-E$}L5bKm5ah zxfQ^f7B_+;J7`Xrd5gLs$U~2E0gn|ecRt1UZrQTyth(LmojCq@hXwTRg6>+3U!j>Y z{#vq#@cD~@!)(^4iSD2QT4j#Tj47IyF?>PjuFq-V@O+@W(6!ms+}^%P$;DYC?2X2! zb2OGw5n9OF>24!@^(?WpZ<0-*v8SL?pTMM|lE1X7HRLJzA!_?eahbw(`-lRtw_26; zmv%z_$8Ug$voDVtTP%B)ZHmscckg=^Z);|nxpL*l%`SIfhyzYV`$iK`wKONf9ieQG z-WvJjEV#c0Ac-E+Y-93n^iB}l3YDn)Ugi4mX+1-2wYqU}@~vCBY2ptR%b}K_Kk5(M z$=jGLS#PofTalxi3ZBDq0vZMW#G<|r9c31#l$j9b+S04Ra-=zL?QXje!8S6t3t+!J z9fWq$SBJ1t{i{MQ zc_b!o0*jr-WRxF+nN&~S@hL`j=X^mJteWTOVl$JQ<1=h=o(J}Zt`RYhIf*bcN)?c% z{9k4rJg^CMFr1R`#xie&2dRm`9}{u`P&l}L5X#{tegCD)kbnPz50iMz+cY@^9~ll` z+g;uVefr?CI7?i*oHmP~cbI7d7s%^b{IvrmD{NYQ-)Po}hc5A@>O(fE%5g7xDffXm z-xGJ7@Z+R5zS_)$16gm)b!m7>GI`^{`y&GQxpO=f{17E)V&={AAsQnJWnpjNN%OH$ zq9kv_kp*6{m~zWilZ%CJ>p?N?Et!GqdYnUG3R5&$y;ml^5=YW@7HHq~-iX^)%-i$R z5QGYxEqjKlY>Ao9!qp!xko7` zbkk;KP<>|Mz^#t*KGB~2fv2~}+ZvLDjs~rFxf4?ip3{*VAHg*Q^;=QGnlVNWjw}W* zdVcF&To$K=E>;Eqy@TC$91$&5SY$Flhdcf9DlnbZNq(cw=yH~q8a=RPC00MHy6<>4 zi;3&UtvFrXg@Np{tIU83udbdD&*p4Cy!SN?ZZe^Z#thw) z_l>sE;Tl0ME#{De%<=ns5AX$9;MC@KfM+>e`V3$=OUdGNZm zM%YJZRHvFnV%&+d=4m=@q<-GeAVtPwt$H`G--G#*j;@**XwbiN`%}D+rZ(?M3wW^y zxhj#{LCRno@m)b>!5dz3(>Ui1mG#+Kq^?T>Zg!y-O`Q1~CG#p*!Xca}t`ql&T6w|k zb-YW`?+d`wG=-$7nsw!b`d4h@FBuQODlj9aCuQs)GqY~i`FVs_@rs9OHM7!LTKW$U zWGyHI-KcI!d*o0-3P9eG5IeYTzHnjZaqScE^x(6QDHT9Hf4bYk2h7mgu)va-j98Y6yo2#@bNf9l!&oJ}1 z_S4eJcu8LKbhn84722Icpkenk>__i9nNfaRq;_z@zMLR*?irw0mJnywqV5^TlZ28_ zpCUw@>4!mPatfNYXMRhPk-oAx| zA=ef=$Hf#NV#Jzbulzsa@JhGK!?6Quc zB&0-CG9)BPsVLI4M=C8ug;NwIj0mMGQy~>Km93CTwl-O^Ma)&mZ^myMM3ekFG13G0YeLQDGfJ!l z2yDyevKDx_`Cs6(3n1xCLbW}<_1p!l`vxTjJ=i!YSH-$LuibubVYaHDcJ?xD+9^-r zaTka~1+nN~fBs(8b{X0ma0|ELF{e6oM(;>w8or=eVV8VLYg|PCs&+4$(d+lHUut0M z^u*zxrE>9~|4pqniTK^_Eq1-_@2_dHvSU+qGISIipzBW<7erqp{YW&j0%4y?dHERH z{%20okL)HDrBrkTI^2Db*lv5}wW+w)Dd`jEp(yAw?ZM$q))s$@FUC)Jt2llwc1*7~ zda2wJlNG=fH%SvHLfvPu>PaMYtPFH&iIROScs>$zyhY>5Mpq@H_ZP384&8nu280i? zN?WGmp69cl zF!}vAXt>tT8#O*I^vcrmpyn7e8A$rA0h;CWC8z!sO;ooIoOAqouf1_t`M??lZRhn9 zuU9_aLg}VK@KqLQgN8i;sJ=f~rLUm9@Ikx#JE`;n-QoM!R2Ad=T=o$=_4r32DP+O? zUuh;8O@+8rICS~`Lk(@7r6MaQx!-AFG_xqMeE-YekfL;@yvTx z%J8Rv8b4r5COO;c={`%*;|DTWUDXj}qAxUL`&1lrvaRtQaJZ3Wu)$$Y*0Q%RUxfr_ zJ(}57)AU1Zv*-(hLxr(c-l`5-zS~4-Z~Y!CvYD4H0U6Au@*vOFLCFO$e~g=SAF6;m zox;u?&pxK`Ehj(eLg#J8bw_tPHSCZ^fa)7O|U<0XAPQFX_bdO>YB!AiW3(v^2^v#{sy92y^D%uo&F{f=U2xoCbz z#6u?V&l9*liEUl|!$Ijalczin+>HjKB{+m^2A#^lkA0O&WdJ=rX=>vHWGnGjgzfs1 z@WZCEk_qy?RQ|$q5F%|Lq|DsiX(hkGqqzgvoOJFJdU9GzQ|5`2_Pm6E#0giJguqZ% z<-LHEA@ok5*wgxX4wn~lp2Wr3!KBvK6L-R0M5Kde(zz+{tOQJDmu~C)wzlAb=OP_y zP(ntb*CzuU2>BmB!4Sd-x{V>h_`dVE4*}!<_a8yz$r|L7?+qv-5Ex#V^gk1CBOh#9 zt9-lb;Px4#z%;UluqwWe(F&pO-)vFvIizG`usTMr0YV0bs>x&G+Rs^Z19PRU529s? z&+PO!<(*RS^#kRiQRoE(O*lXqr~k)+uUYu&-iA&yy5^z%AtI6X7GE3VUhaqP>U>O2 zu9I5!LEt`UC3<=zGDl8Z8%7%+3}_!cL3krZLhW+Dsir-9A+d_43%30EE}=R`n=m&8-Oes1e)&*&x)hMVy0PkLmOe{F z<~SDioKY%)OgzRpFpNEfWb4}SQv(UZXD)QE(U1Rb(WL0jZ=KJP3Z1`;>E0`hyJdhh zmYj|8)AiYwt_y7 zaLQqv1JsFdDdd0bKN36kJ86vn`@;lh;CLYTy}LB{&+BP$K>oLXfwev3bOoPZgLJr~ z7iSLy-ZmN(>dBxicr}2{$F-z`=Xs}{l1laaNl100=azYa zNxZy{MDo7qU%vzy_jZX7gm=H+UgVew!AN^`HBR0oynMCNX(%R6-kF0yD(gQc`Db2W zj@p=${>Jfd)i8UGMVGMFFEV}0z+>#;;^5`zi^tI=q*gF)qwk<+xJRJm5HKp#uXh3Z zBxfHut(~G$ck@P^eRfEklXLI}SZYx?zINu;KM58}q{%+m#?*iQ$biiSXqFCWmNt!M zFQ@*Gp8?%^MwBMw6ARz;ju3ww5V*YI44$!fff|e;hs;kL6+OuANt|i4#7gnn0S)h( z1K@}=Klq^ceN!UvU3aFzC^Tq!8Aej%jv}5kjW0U`s36cCX!A(>WP11eN16VMB<@%c zgrT&!S)7-Qys^H6RDMwEy6Mq z53StiSa8a|JDD)K!pzwlGa>sJ2z$hH8_}nnFT0Friv8$(of49F{I$_Z`lS97*{%+s zQq`C3{R5@RyPm%-zuo>x@zDNv+SlLdyo9CO?KB4snRxo*mxrNhACt>DH6trSLy0>+PX}>a4boLK^!Lgws^3jpV#>UIfY)Fw4e(RvvH2dXJf|}6aBH?cV z!Mz?Lehi6g1|lvi9|(^%6P!%&W*U3FqrXiAKI<%7 zR(C|xjw0gOasBO`Fz&&(ntyO8oxV~l6~2DKoY15UH@m$VXvEggncK8aR{9&=Tl8JQ zI)oq~7Z```XV~*e0|;-5*5!6`W4!J2vXX?Cl10#rd&j~6a{&N0>n}L&hu1OL>m6FF zKvj;)HsWo**_%QU>(d`yU*2}3r7{kx)fV?d$adfVuT=Q{n~Gw@9z9&@TZ{wn!h{#c z>Yj$XToHca4j@eu^w{e0mOHB-`}UJiOazg;j)3eE3u(@9OuGi>Q<7I!-|+jPW9Mf$ z%{sW!MOX8nwnl`SYa&v*82@+}_v!_-&UW>JLAdY2SP_|3*Sh)zne}Y9^4WX7ltF|2 ziO%BKdA=ThpWTL0>MTqQUBFtiH_)1e%=9c+Go@?u^7*sSS!?qpg)1&R4bB}P>>H6tay@+TpP2oVd6&n9z_tLA`oMf^DQ%SCdJJo;2Fx>a0$c>3V- z5~IxadryhVE`%838a*tN+>uH4f^%#lSH?h1pvuyfBZLF;XP;63md0e;!0=+4b80@f z@zU!(KyH!&b-DpbwR5gnD7~wGrPYVxHH+94}TS1#D|NVEQ>jxfFk#Go5I?3|SO7&6%mgC|ghpi2gnmRrnf z+{aN0^HY7SvpLMdqw0@pk%!m9a%;ks)znlb=;)E^7Jt&{7IO#aX?17dD?wfx{EoQQ z?k`nvzZ5h>ZMoL6cn8)}9h`iRKB~Gm48%F)U_M7Z@Yubo4u*0-wR=O(4^EH!Zpv4u z*+VN;W>smWkmY~gl3+>O6FM#fsb9|PLR7QeB3~Tt)V_=7^%~O znIOr89Nw+F`#3s7b2wcUtBl*#IQ~%+ zRa>225*&{CKiK6sJH$%p+Si9A%H6IpHR8hzOx3NAgefAd(zk0boIM{G04G*=fPYCW zH6teb_ikV>XEP&<>{(TZNHXYJyvw29;L5UHEgNd8l(k5DcLiO)yBT2PcJelLk=W1v zwyeXfB(Aer%*tyw#)U4J9AUhhjnAp#-?cCgsK#d#*+^-MA?NY z?wYN8`73qa)=;4yMk9ARy4H%@(cui6RNbBx?B~vN*aRc5mm&Or2KqEScx*ywe3=H+ zi$ckPCpe@_^e4t`YnyEB!h0HR!#!JvHb!2(X@|>I}RSQ6ACfgb5^*H(P$q0o|!$b8gs9nGXdz& z10oVta7J9I(+(K11ta9TTM%Sd7ttn5Aj+W(R;~BTPXh5HzyxhTkMxkY43s>~*-Ttw z2tnHnF(dk51=5oe01#313=IPaSs4#avxoh~F@^Ii-_K>=uXe6K^)XFol~*M^ zH-8MX1FJ~or1I>X*693Njt&Tm zV81&dwd+7!aL#c6c0(j~11*OQ^7U6b!2rMm;Ix4M^CY5+6^8y`us8X{EVD6MZW?NG zhp@+f;ip{$Jq(~=wjb$7$FKdh1s?V-W{yrJO)Z#76Dyk6MON`n&WnGDR@&tiJ};_Y zvM~QexciHSu0)+vPhv7)rc5i0G6-*z%;YVj{m4jBh4t{DPwy{IFzge`4hnpQ=vM_LR&vS!N85X@+sf%>ANS#$h z+|c056z`v0k(78YsK`{>Q9Uav&St*m0Y6R9zyVSd&X*VbM+~prn++E8(Cec6#4*R? zd}ILTb8?ZPF=W5apc7}i)=g#Vr3rHaux?Ev0O9BKjkno-LdMTpaBwjQ1;0}V$voTO z8V!(1#gzkm%5c-BTy+axS+cLGV$-VVk58*qKi!R6-@Z00OA46QSG)4`YE@<2$j2p3a7G}1d@s0UDhQ+G<-a+*Ak=0%=TNNuVoOrrNuDe3dijICPLq$EaZ*_^ z%?rL@qwx1}u6E1jE#$Yb(_aG;-nuN$JYtUB9gj!f2o7b52zCHEHn`aAf35_*K$ z5`a`qZTTi8_%a9+MnK_i z-fxAw-Cd`7TY~G7@#hGa$^d7vKl&0Y8)9k`ul5wbzO})ugHG_&k2? z2xUGN$iImkHqc^Gq9#1_UI%OB739`T%-Wl5u;PoxO4$}$7Z;h(-%DAZxsCq_@p_S~ zcb(cvxw}s&Md_^Y?I(J)x3+gzc98xbv#UI3Yy##NrX|q12xQ zb8RpNj{XcW&G3OP;>DpS^5&sZCVf)-Gjh=9OTzfT86=`u^<6Z% zN6DLC3$&mBxO9;0T%pEUVPmKi5h(PdIXYZ7ezF0sHZW$*=9NYcWOyj-O>Y58 zrKeV-qk7mXi+Po57UZV+mqoYNFR>)-AuKpbSR@=vNDH zzak%El*@k#M59J@F~(QwYs&NL!D{k_oSo_D`HtsRdym?$@_49yBZ-j&*8e;_WR1j8 z7`he57Qv@6dfO?}pN147)25#MH&ahV;l|?eEC0Zdi_KTxhi0Hy@Mo#ud`L0?ppby` zuDk+Fx?W$2m48v#aW1(z8F3+v4Ut}*JU23!&0Ah_Z?~gTDRWNvdTq_o?%v(kzU z{3SG(8-T+*IRl)(9795fslTgflk@KO)egoVJ=)$KI$J4DpEi|^`(-#o1Z+wILw%_yFHN19*nE%7+TiWx-nP(VSOc1Hx8MoAaU;g$p)Bn@`3jL%+MU*pTR+WQCp_L{y?9Dv2*it=EM z0G|+~M+uH*#Q^5c@FKQQxRGFrJV~maEv8g+D=#mv$4nc^rTwnDYA4f%6=Ym2p%5z^ z0#miGfi4wJ`ww*5u{l5#@GsmK3qT}`pAmu8Pn)X09)%13xe`C3a|L00`@=ARr!c?e zV0})P+y!$%@Br@i2;SCc`aHM=QPeQ=NGRHw8M`)aCU4hmvnR5KEt%X~Uh<}8WLZ9o z+^y% Lf>OM}Z9*%_~i^7vbZWRZO*M4YR_Q<0A7#hpDpy_@n9z0%_vSDe*81C#wA zw;6ZVSO{N0`5S6V|K|bl$BZ%t0luyqD@9PfT!A%Xuwk`y_;UZkRNybxBZvC6R)+pM zg-CuSCfnymMs-N9-*$`c2q3M4)_}3dSM~UNTDGzX@}`K*Bu`8mN57$h-3j-~Xhtq=-~M{DIDsHa@RYld{Gf@uxZLdkG;i#UMONiEGRkSV#6jLh zyf&GH$uIBPO-&6LD^pBNY&UM)A3oPl?|O`}`^y=R7QdbGKn*Mf|6a`9y@mY1KfQ@8 z$n1Sf=mw=opXkxj+0XN@P8iK}wh}u;Pp%vS{QXBfY|~fZI_5!Kvl@azh-coTqU`0B z2-~STtk{S{5A$`TxYifxWqkE$Z*O+5K50&uO^n-lVC|>Dv-)Lm=V4WfkV8i(V9vL+-64`hc`jUDj}!nYSv{lS0eI` z=;l>~WtJ-2{%;6u{5K`{{;aAJds=a+*D>}8>uS5%I?-Fa!)sZRG?c#q6swqnkO2A7 z1=!D zcQA-Y*xo9O?atrtJCoNm^ZD-iFLv&W-XW$YO`Om8u=C~0JK~jc?_1|PS3V(V;|cqD z7ASQ>7Ga7p{K>4A>IhsEy$y)cAN6A~uc(+LGSa@5KDe2cJ5%dre(d9{Oa6BApRXXo zg;M%+$a|>G&}3Vs$p$TYeFTMxjBqEe9~5>t@i=OX9Hv0`X{_{`D@qzdp(Y?_vJ?%fWB|kI(%7|NF0xV5$_K zHFQkTpY<_c=J}$cSMN@2BaZ79CPTitT2RBU-u5eI8M_PXJjhQe77ngrvNup8Y0BU^ z#5_~PIlvsHI#6^~!fEu&blTXVWDDDxLBhd<(lrhmS4k z+6)0%mgW-Vly=L4P$no?)`ojjX4d6^9u6n^gEN4o!cUdPH9^Sm^`c$a5Mc= zi-dB1Tgo6;`_mg*KMDgb)a@^2#$o`7vDo`by1$Ci=@ArZ0e zU#~_=h4E76_sIDaN~4qDKxxiQ7LROqV(7mkWZ%0zKL`&kI6PwLhODx~%+K@!58WIWc~`c9jA@5}PlIY)NhxM8oI zV!sCX02bM_+YF4sZz9ShNCAGYI4>4XW~F-8bYb zLJhV^EzZ_jZ^@N?Gc~)V15@raHN;vg6zNIM)LI83I)W%q<4wqdg^#ZC$UpjF{)1fc zlBwZnzzFrb$=Vl_sH86_x~WQA4O-DT3Y^}dKHl+GwAU>^taCEznk@cpyKO!&7Q4JA z6$9Zb|CPC2&_VcC{v^cSYq}xZ@EP(Va-349S7a4)%H>^Ptw%O@>q4vIg7e42c;|v! z^HtjZki&L!;fU}v|Wb76-ggm&bjxinw{!07? z(ntpysM6q%P>?=u0oVTQG{DTd^$&zw7Lv72R}i6lBaZ)I&_|UU$LZT~U}e(;v4cFu zp1_ComzM|95uOBIe(v#jOG*Yi>5Y<-snh}BQ*&SFe`clr$Gs49X~Q9-M6cqRmjGrx zlKSLxSjo5v-rLI+=SOfrmGUp3)`UL8zp89XxPvz{oQ>0E$~q0SyH zrQqXRP%X8%#m`aejUtM|q?c~My0ZsqKJSmgnu?Kl7n%hD14JK0RF^p9$J>VK@?Qln z!B(LAIa+1y8f@;B&giwDH+?p4bM9}ft7~7mLwmEeeB`x?)Jxk5LUkeJFFr$DWUPS< zxf5SuGc1u0pGIdmG%MM_o#&q8VCNuT@AL89`a@xv#_pQL#m zO18_3%6>cSuw3?(Jvg)})j#=~1poi9aW?%og%|&f3kClt_Eh6im4XILl!=$RDdCCf zNL;EPeew-}$Wm$qbX^yjLO8had!oj?I-LV&^>g(Ch99@Z4m;Ow$Vz+IEo)}NvgS(g zFQ5EL(2n>MTWk};cfuC+rjgh?>EXyksWTV>dp&-Dxbjrf7;HarNlXGY;4MQ+^vn@X z1otF6k;*bu?2Fqm+tIs}yV)zBJ!{E+JDIPo`llWoT;MKgGzYn<7u?Rs1D^7$xKS_d z#~i`q1q4%GzRM9(;9+Q?cpThL_>%+bTCu;5Ex4vq=JI3E(XL?FNA-;DSy}m zS3=nB%zSaa0+>e-UJxKhxWLl$!L5AU5QT&H_4sczdAC>ofGfHinf_fd{0{kgO7N>K zP(c{soj_^@!Ois75s-of3vG}~y_jkK(5qH(A`$!wLDWMMe3uZEWJuG z8R(CfBdA#tzWE)8iNA(4UHaxGwAq!n2q`I}jfOi>NnBwk{^D>(MR5q(6GE2u6G#V# zAx!1l0BoNQo5U4x(ROJL?P@Ryjch8%MeB!ij=8{-!I{ zZoaMZ`U$b(_!%KHPTJZ81B@>@gl*~c5rMxL1{(*^OIj{{N_g0ep7Amm9$Mge=t}7l zW$m@fPHO-{a7p?3L$A$9N*KU|u)-01jU{Oi9Em8v>g&4#!B{`j+5roBn9f<);?0HG z%n07j;Lk8=NM_w-idoFXwxSW8POh>sQqNBovSng4ZJ@ zYXvk$U)(z)1`Rm=ora~)O8#&O{$a3N9DU&j~DwFc~l-v|5ksz3?{ zQqnJ`-Dm3bxBdP7a3M#lTjFuY>`DfgP)`GfEh+lhiFd>VQ$T~v|EcE@p#HBBItpAi&nybBNf9MY zEcIecZC6K%#HdiIx<#4lq)pAdGe7{0FwAPyebuG|v zc$-Ih>NDuDAj-&|eYNCcl}g;EEBl_Zgxe^j1g!na#O2l8a@H9gmE|EiMjO z$RaEi7i@grqMrEq(>uS3%Y?ni05>xYeJBEt7$!iA4EEVS!J?n_ZKH1FvknARj!2kc z#iwASfSgbwwuuhRzIO+{8iR-)_iNJTP~Bj|fOiRLmR5};mo=~bBsgaB<1FML1@J5} zmz`hg_KD6;ePinCe{TEr&yGu?4~&BLrPk^~<8?6H10^J&KC%v(A{M3No_&mXGe7AP5trLPN~Y z$lQ!2Zo!~ciG1i3x#N#OAttW-N)LL)=hNB#;^HO~j!8qcJTI@j&3y&_<@C8$KF6=B zZK-^``_X(&q3zo?5(tvhb7^g+^Z&9Bq_G|50xSIitTb51IRiW&*mV>@4!`r11q(X_ z#8T%agImdp$l$RCSez#t4*wgXXc!%)XZ9y^(=D!;;lJhtqt_O(wtZ{J#Q zBgI~4VaQ?#Ae?w>1<{2JiE{~=MdfdBBjeX9DILD53KJR$yc77BWW}34_r5a}iafjy zd#RL5N)MP`0h<0d_vXK!@hAq?0%C=XD*Rz_IHQ!r<% zgD$8gv4;&WFu2;Dv?)6UB;qwaW1QUT%`InFM0pnDk25r1<$K++gS-Z$@a$KPGik`PuC?=bK-K(a5C5;&`^65Ddl4v(wc zJ&3^jFaH6J7kB^6^#>VtjLZo@-mFrmhX?8%6uWi5x#DSSOH0KwYkAM557j(pJi73j zut$Q>dN1-XktRZxp&as-aURqqwDIkQe~0@IgctwC#Hw)N;H2Dla_nRDoCM)z+mUl& z1@+?sRkc6=2>n;Z1jV9yll_pD2-}xl4&0zjVT2X$)=LeR3ZQOb=jD?w`+B0Mn>@mpOB7DUW6@Y2| z0PSJ`1OGQ(exdq^(Az2`sS;C_;T^-9$*N@5lVBoSynN&ezL2vb;$)h)v%_AX&@^2! z%-LBu&ok{*TIQ))YH%T&Pb^(#Nvih2)Rmzga1gx(&5gcHzJ3hj?a9I*h&LFZ#SX~+ zra=F!=@>v7m?IfgKhE4huw0$&yt0$wOJ=o>gp;L;As=7j5v+oWcRq>O-?>I=bj|we zy$@a|DUKd}o^UTNN^4uCp(M5x`rqP2C*wUD3=1s zPjbNwL?+o1{|_v@3K!U_hux(EWp>q2-wiM`A4$GXW%eejIAWIQ|p z*H?86{(+P%LXI+9K5(zVBfo+zM0Gh`olcjr{T)OZ?`LjDl9$jQeD!EOG#1E{*>gndNAEHPJsS-OzVksiu{Q(h=r|a?a#7Sx6Zzf-6gg zFSCY`qWtOCyZ(B;?SJ@s^TcZT{+|RjGP-aCleY$r$)q6&$_~Ft(L+l+`F0!Y1NW!4 zwz+*hGU;q>Y~&$8@^#YhOEp86+91!qL7ypXLmPO?C$W?8ly%$$VWu8N9!6k=C#%Om zEt$mLM)_e!swDAb{pftHI7TVni(630s$)eSr zdOPS@hJg+l)+c>dMa1lA_mzxElin6H6F5|o=fgu3Je7^WgbHUjmfSOY`hW^!tJ$8w z;vo0GomA?-A82^}(*N-F%LV{fE~5->;X!y>V#?T~uF4JKOP7Kgn-oXm>T)~k$9X$_bj^q9N% zde}nXN^B~N8PZDH56E&&72%x!Yv@`q*iD4GWjowj`m!col^0`FDj!-Qy|7D2m`(}t zhk*seynp*WZFG*{_x9K&f=9Jky2eB~X8Ynr4vIW!YbmtTb_@ED`fy%C;zG^PRnq5N z(F~vnk??>mIxC0sDc9SFL$A1^=0CaUkxAz38CP}?#BbIJf&_2mvN<~2oiyG7w{tI< zQF5zOQ@CD4h^x+b@>qFTlLl&;IxtafRbU$eVe6Nx7FpM8>X z{#JyMJykK56jz1%bJK-_+3dkcy)1miaU=`Q^bUvDvf7x)nzj?#V{ zp-oBCCzDa#%983?L`MPtt$i0g38D&CGKuZ9UjIRrt1@d%i$7mnnma66R{;s2rtRq0?nsNJ-(lX2vV{d*)yVq z5!pUh&9}1sOQymz9(kS%YWvnuLG{x9Ty2u$qy2of?atd@6A$TV!z3ZL`PqP#a4zy9 z>eq3Kt0)VUVLO4@Tx;7=sBBj~K-*`WHO0S3Map!^B z$I5LjBKjXIjO=ydpfDEsWX@a@yc1y$GN4e+m-&|~0oNh*L4ZO4tHD*_vyMVnZyShW z9Qx1SvBpv4oGb4#L_#9dz~*u@0OMT_E46~iWR%H>-UDX zi_TKpk~XLyV@udtak|7YiaJm|$z(bb-?G6=B_Hus35Q^i?~6*^;(ia8Y#bO!ypU zG`oEFD)`F@Oo0;-Z8AIBeS3Y8R{AcFc8|Ap)yDLrQPo=cA3v6L$2W$M*Zw(wz6fsV zzo!~ri{pY16#Q8%TwgyS)bBYX>-L;|yBfa={#zqb3|1@Fl%EG!*4D&#;$wg{Zpq}f z`0(d8d~R?$A+hLQ+MgMGVh;(I8UPP!>J8kZ%RMPmo4qmG0DljeHO%;7eTD&n*l$Jf zclh)ii;`EW7$bKVkQ~*MfD6XS}0X4gG`VP*t8D2nKJ^lfU97ZC8-x4so1SBhyd=fo%k30fa&ub^vXQK9T8SB^LhSY zA8aPTy`e_~Xxub@xVdUNW`JHv>p!>sG3|2TnwdF_P}Rcp$bh=--TK$!vQ~#tUJJp? z-@+5yDvmYzC?Tx#$Rz@jo{y<3Rbw(qonDppe}nczBT=AHLT_yawkZYX8)h^`(cGrf zM~azXsuIL+br88U*JyZalcW%o%4Z4W$-e91bI!n0)nW$V1l%(4*ec4v7fQDd+(;qv z-ct~wHW_UFNmze~J~4PlY*@Y@vxFQ$q{Z?;7d|){RyVqJV>RL!p0vza#Zi31%$5GvGQIn>Sr#kY$a$> zcCmt;19&jhjrQZNg$#_1i`*fOCc!_n%)NHj~obfY;?>05?mvAS@c?8F?@s z-o{5WdFz9#$dWlk5p?X0iu_dg)hAvqnWEpK>tymgwbe4C*xpzp(lJV5Z>yeK(hj2# zues123Vz&$jxIs4d#k&c{K$Orlq?h{Vl5RWE6bcf>>87|>tq6fdN@-qcqw|zy~Z!V z@zqdtTbKFrvbHTvFU;r{fI%}D9#aXC{7s)Cj0_%#0L=_I6JApwazjMt^ZPgUaaYmh4o2ur}OPeO0-hY}5G6qZ9& z1=R(*(-vQq;;UIG!y)I@^Oqj%YLV9&!Vt)vPR|x&^jQ`kTaqduz1R2xoh1Lt-|cW3wa+zI7>ad zt-mCMXf}P{0#ERr;B@auB&%b)z}aN{UU?7bd(**EJ-v? zyvJu(kk~F{RM(J-={-P66$};gtq=QDa*Gm^664#mZH$c`x<)stWeSJtjdgSzHobgz zhxi29L;SI3?7a`$AhJktF6KGL_=m3ONYe;6DZqMi21 zyNA{a5M%^w+5zg7l*Q~}#PiR^tk|aI!7^q35df zy!_OSTxeEIV>>FBR%#zGGD<1mF+*?s1X6MUVLc4l6L^N>Pa4OW9ZO8z7cFCYFUJUx))T2`}Iqsz@E~&CrKBqHx+UOrQ zdO}^Lr>kBf0!=c!4A?Acu#$Jh94ZIA?oNlbA$E^*E0tiCkn$2qaSEy1p}HAITg2Sivh98=t$aoLV<4d!Cp zp$T0XX!ny4cZrTP+YU_E0eS&uio$shy~~6E<~*mm{CTM$fT83yy3Ey#Va4c;g%9_4 z;{EOel|~NBvkOfZ+>Eie09Cn@)$(ck3aG;F?(JxkU3Q_B_SW64shOvg-E2fi`}FuU zbFhFS=isAvx1)yZiwaTY4&4RG@7vmWW2s>Wow@B3G`I$cPSjoIPhp|)e;_Uql!=cR4E$Imsk%+~W9 z9NKIrzd=U97Ifwb;&w*me zQXM`AuMEgjJ%4QX)ZA3}9*N8G1&=QLLS!vO<6L~vDg;wpjZ4HE3@&3-Qk10YG|7EK zpUE4bSB`Pfe=?c=hAMd^IV#$u{8Riv{pZU*tS*}Qv1}?1D8h0ePE<{5CuarPlM5(yuNWqi_Z5_@$cSa;688wdaz<|8Vq|IOo$iSWs^I5Y!i2? zl+0fJQvajVL+yA=Qm3nutga4FK(HCviz)}1Y>YIeI8NtlM;3#B)}msvU$#9(KG-nw?RLGdJ#mI7lv?!AC*~0rV<&C zz78aRtU7)6tX{SGr&DzfT8jSrzh=uiy&^b=@y|^YJQVg07DC>ujYZ}WykPunQvxo1 z0--xiMvT!X1NliS`L_baq1rNs4Cl2jWX;z|^S@UD!r8{Sv++ z;8CUH=OV{P?ryE+fgjYi*qxq#Z&9U?l*R5A#%35cGeVu z0`xSD+ziIR)It+40WE{mU6fE@1t7Gi zMxYFuEsn{Q;)Lt1SwZ$>;L0q;8zJS8svD67u(CO;ygRYuldL_Te zWEVK#cSy-r^7q03_#)v}Ob@^|D}~J7L~M58Ci25AVC0`f3RVa>Uk)iwYPd1q-Muka z&n9^opXyOK`$Q*pv~GP_A|yo+DqGzKfn2mqUj(*_$Fd}iFX)3)x?ukU#xM9zW8eFQ z1Soej#Mp*mxH3xK#1;=R{%#ztv_5pTyWZ*pUnHf=W~6z(O4tUaj)6ZTmH%!C^xqVq z`_Drla#}TN+>39uVIjO31B|J)6L>BZwi%+qs{?oQ{|F57;cR6czU*Ue*r{jM9j?Ci zjkWyuxF_#sJRN+0!h6UE4Fv2?!mm=n+OSQa0Z)U>8iZb%5ACLve6buuz5y7v1=f_~ z7@W}1SiFs#;}f#PLIFF4rqTmaOY=|J>9+*&-^GasxCb76e=gy;U3%a;Pa(k?<6&f0 zlg|%F(i`T~w?{~yWCQ*x0bdI%ND=_agGyl_S1% z{*rLU+a0mq{J!9&py0BTXB8ke)^WJvw%F&g6!m%u)!PE1Tb?O))21Ga>A@L0sZv5+TMFdO}Jjbx2V>SKmV)M`GYG{7JZC;yV7M* z+2NYxpM;f%;JV%;CCJg9^@l091}#!J7TGwG;K5+)zd9pv99AE#U*W%pJx z1WkXDMG9R&PTsO9A3DqzI=Bz0K1^H*qb-o}4;OzxAAb+#bp=@bC9^8AMNEnPc#*5% zR+7jpswD}+y4Pw*%?J?4t!?5RX=1@JBpILe3TkT~nNO?BsV%cvN^jbP-=wTy`-Gd! zakp;tL~$iEo`A2KUzC7JC=iDn@+G;nF(6QSRP?CbiS z5)Hp1v$rrWx69TCig!_?f)~_}2kjQmEy_t5O4hv_ySKKf;99+FZzd^8`HonWf9f}S zHiLCrgUGHdGi?39n!FI`wO=tK?FepIXJBZcqnesB>q)9>bfw=xZJDpGQ{YNVlNqc; z*hJQWf1PQa9pdS_bCu~6;7>y%p`{g^+ASue&-XB+G|bqP5{6`7L2-PCy4&!l;n^lT z_onFWR9X4|Q1|BXQ10*lxFS1~J?m7IY-u4a7%C)3(nhJKO|~SJQps&evPYs+hA5RZ zDRL6-WJ{X}6Jo|tWXX&yH*?Q@zSpR8-sgSJ`TQQ=_v8Ei{PFwaoa(;sx$oC&PSI1$&64d*zG!zV_&Ch!8YGrUFJcP+YgQJNhvzAs1q6ij zJ~D10f?!6kGRo~I^Rgyfdi)|OtVAZqks^kLy{@vHg&B9o7};&Oy3qH*+{YWa4(f*m z=A5e1eRf%TfK$D==I7zs zX#x=_TWoVi{{aENVxJnWA7=3OBNb@^i|3EcGayXaj>mie~Id8{{vj)ys8~bYBxBY>);)Z7BW`*6)`4}Efy|(FT zVocGdqfW=HWTszaKchr0ODT>_7+EFND8I%s(^ox9`usM){uB)6V8V1bw3@Kr;~nDl z^uvUB;5Pe%`9&x9A9!k5uc!09QBW13N{Rdx^00YpYWN~X=!iFi_XfAD!)Nax^TP+x zkxh&*JuxUN0U8_ns=r7}&%cGsa|%?JrCf59jyV8G9$g1*#mtpv%TYh?g>*m6P#0J1 zx8?4>o8?7QH{U!X?^ZXbLcU|smUuJ$dlq4W&PIhYN`x-LnV@3&6ut^s3AW0|`A?k8 z|EF9i^tfPqT#7{jWCgy6z+61q)#Vf`#7h7gSEMPwPeCDVllYO*zXonv-;SyU{heXyF_Hc!F+iKZvMph9JY+AbGG*7k83*(3JD5$?<7lN1z68i4&A4m=YwUpa;L6 zT1n=qno{`&DQJiMKi>#3UkNGV5_>@1pbe3FsNcqUMxez%fkp+-)er(OHP7jg`#9?n7n@FQ=mrqZlK zpKDi+Nbl}jnLKnXz~3k>HO0&A*fWh`*Xsg{%|T-N)*dVZDHa>hLb@c&6!sGlMzDzX z+H@LNRBve@KKx-=+sYHw$Q<=6cCd=*Ct;?57+YC2a z&umS+89)6FOW+ed+-iVt5E!f|XG^R`&=0)_dEkW0GiuTo3yQEshA>mEFk3WpSzexx zmcoaHbmN8U#~;6)_h{*}q7(bmRPc8lR zSP-_R6BQ3ktJ_xJ8M*0$ud@2Nc%MRh#pSx^H(Ara+a7+JAp%<=JSh!2K+Lfvq1L{2i(>Mop5z^}T0?@#^kn&wPBjd{*>_ zv)x5)lz#9Vl1}-4Z&85kxnt*r_%^{oDRtJ-LV~sl z;a;H>&JN@7<$x&hn#rG&j($9JoX*(}gq+JR8u-pt-+(=XU{?vJCaU1P$K%HPI>QfU zFT6D?YiyotLMHt^_0N;n61YmgjziYX%pO9cmfH`<=A{H>(( zrsq^ci(HxWf)Yzp3vy;VN%uQn@GEHRuTJV`ApnVi)_6AqEjsXR4bbVQjxmPF_J0&%J!uXb93@csA! z{*_UP6kHYx29{cW1UhOcFg4JD9oHb~bUpYlCIRH1ge@dfIQGnV>X^}6h{NAPqWOS_ zhLn@b!NZjhVg-a&{(BXPz@}y;H=BJudd(R$f z16;8S-{RM`J=TLvfBQOIg-l2eY(P3Q%{^#5aWf41mIL=H--CWuW|Hmr_G&0`$rOpQ zKvu(HbgIMj0Xxii463p%m4n~X3Xm#*)2e&EdpXzWd>p-Z$BmY>x{VKulkF@DD-Z1L zN|cb!0Sh4(=u;CZpPPWUF~cdG-BbO&qgh{)?`ptscS5wz7?)!JP#gyx6T_ZyNuA@* z>m^g6RX2Wa?c8M>RGnrsCN#BN0QM9-h;76s2$T>taSCAgYSYI> z=p0bZ!`uY>riIO-^aa*aMjow%?A!H17=@6G5#;D$U0}>@D1;Mv)CEI6LGD5&Z3vFY^>8DQG%|jyjb$F0HF+m&*v3$Z0nFhsz?bt@7?|0psJuAxYA)RBu z$%!$yV^QT@v5FTeNncxUUtPJgb(dA!f%Bd(4;;6+y6&avBq+J0Y8c!zS+PM0D4d%m zSZtoNGnM`9_d7`a3qgUks0^WqRB%z&1Lnw)^rf&J;esG-p^_3|fO2p@%%F&`p$m)3 z*~9F#!zt>3`uCfV-M?O{6x@gm`<7XdNB>$JF?lKvUgrb$k(N&VgAxmRP!KnwJ#NHl zMx{J8-(LHolKb9xYcWyOxTQz~n!gPKkPdzwsrU{q0IgKW0rGd@y{IAbg2T|~;cJ^m zon-pP(ul}atxVT9smkANk8ocrh=RtSD`fX8{R0=a0m5Ek#i60R)q$^tiJLI9X+-03 z7wiGp8h@D~`L9!h=twR%7jmPFgpfDNQ%Ox7XnDdGh(X+_;|CyoaYla+p#B&c<4PXY zX6mNh#4wq-|SrY_}P;be%gwRboY4IPx~()- zm|!)ZlG}deYk$(ZTFRNH!1~?)Gf8m;a|1`z|3^{9gin_fHa8WQKUq;mYARWiiBIR8 z%^QulKCb+|X3stDYsEvoeLrF(K6qKlURbk-Fuq5gT1HJKM(SPTu}1d?9`uBX???9} zR%yiEiC+91>N6o`GLXd;QD7Mzl64w8TSr8D1LgQZEd|AgHa7Dq*!+3aI~{N^!al;5 zZx0w!{FS;(KOUth5aMTG2jile*Fj*TPE>TYxVF=7 z@fD+{lQFlyHm|h2;g{{@Z>ZdypS`F$Trv^5ri$$m652SUb-~RL?aLvyV$jmwGWbIz zr68da7H9MPT%hWs4Bm?+hw*&s6*UGf=RinUur2ltRjTzoHx_H(&HQCtMZtd0t?ra< zhTm#V&byLywpSS6bh;ri8u&9r0>T8$?S(x0m>E8;yoYCjVD~^S)wPP*H9*Q0wB-|x zn!cmOwZAp4v}AFtj=#+@NVghLvYH-ZesjkBEtEll+CR#Jz!#|fQh%uZ^7fBVrvfL; z$OBgjuQ+W%j~pQaXd3feWQw_5!{ba<07e}G0>AorWl ztZ~*mS)^MF?V@&{gflc>;Ci1$Fv%)qTDawh%mx!j^(wW9ytvDc2BO8aFNfKwjcwJK zW<{PUs1V4?q)$a}%&!A28wZOZA5+0&Wf(xrtAIQ2VT;XUkw4uyK5f$Jf29QZzu8mg ze^R)ah-Y0(49j@%80Vc|IJJNlF@RG$W1`qkFKs|KaSGlq>-`jB{VLtgbYcj+*CH_c z<+O>SAt{6zHC3xXLvXdt;7W}H`z<7Z#@96^9<9lszZ+_4BP3S9@%+k9Dx2Si@8@5) zlC;vu? z!>LYV#gBc(Yn3PCVrHZ*?|HU1(Rz=q>IP_w>`UPQbP{F=vHEKvTqiyO)-3>nPAVeh z!03h}nChTe`GC68&`2NNV#dBq!lt_z%evqI%jBr3V*6_pJZ`8Iu94+^tlvu zXfQMnJWgjh>+p(X~ELLjTFvSN+fLt zK?ywZp~_OZNxN~!kYnkl_pkO9q^$ikP3-d?@t#8D_9ED@=}5T-rI#s4!wNny)vpuZ zzkoh63AcuzR)@m6iaZ8~eg&Ieoe)YBGa<dxb}@5#qp0^^RatklW{2 zM1(;i)K{hLCcL3)LNevaQf)0N7P*^KmhQQ0VU$cszubgHO}CzArek*;iV#k5Y= z=c2j~PwL_hJ@(=C%V)h@t~3!sVTI5*b~C{jAEP!tEnWZe+|y~vvGE4@jcbSN!2kmv zYM43r_%{(ka2S3%-+f%L+4xc{u`3~*u&gH*;7c1n*O#ox*i~~jhiT<+R9b0Nykh3X zQ=4DX8!O;|@ZlvjJ;Vm#D)RUwI@IAU?uPEIx+mB2Q8EP+E1js`p5e`A+DEIAjdqcU?*VE5|4R}V#8 z`mQ165bkE>w{(q#a^dhjsRtdT7$)pW5tu1i3I9rU>YVD*NR~&+kgN125-d;o5d>Ny zCB+?vX?+mR1;ymm)CO!BYv6kT0!uO1RI=&qxN-%l*kgk06PLNopNkv4~CUbdTD-G8-7oP`7=vlx>@ZGuApql|a*Q#Tvn zcOJS-9)98EG!6>HfdzC3k!2#Ra}>_H1Swpcn;K+-n1h~qtU6c;lk3dYbXxb}8~xC?xe*l={vwV#oAj9DiH&mK z4DbfJATO2Un(J2U>kC{ebAMdR7Ktnt03bC@g0W-jOzpbH0mtkIpra6mr9S3i(^O`G zrJr#t-SJ8_%PXsu3-!!;6Dwz~GbJipgA$CAmr&+G0?xN}iqmQZUa}-#Hkl_1hqE`8 z&N(V5@SP{M8(2UbgNTFEd?s#nHDk-8YmR8F!;DH_#S5pGET5Cio9RzYGDA7mBuu+G z>(sK$vmB`x!%nLTl2)AmSQ6kl<+E{%&Nw@2ec5#EV8?KEv(JTj>kddKRYugOj^EfCG8cWjvyOhCf_1~1>$aAL&1&`>%l3cQcaL)NJv2KcjV-;2 zR_})`3Bv_LoZmS6JT-w>px-cC`(46C(9UlM5YQsU)MCvJY*%NV$F|G2i=Q>z|B`0= zIo*D??TO`GI^b|W`Dd3orGNp(W+mwlm$}+cm-%0AgqW|6&KI(E-}bEn-0(AqIvQs+ z^g^_Ywww{Tsxo~xCewK>x1X=}ip};ozq-L*NAgV#sVPba_IL~=_t(El?plH5MmnDZ zGgkObXxU$QamxonLkxH^u)M6_M zdq(L^3BZDWa6G9JxoqHg2ynd9yCXAq*zDiyA(DTARTR3~PAx3*^Hw2QPf+^>9ghCi zO=cVzz^Czv_(k#+niums5*M_Ma7W{(8`s((FI+=Q$g*k3)Vl%D^f z->iPsP9}9{wU*as_dQb2G}0l?lXzB%c!zRWkTjRTS09f+hbo~$4y|n?Bm*FbE=Zhb zWs+4bRf3j&m{VUu$p|dkod029&#d^44%L?ZabmiXqGBe~I2pY@`~rlPKpl=@hLXqT zd66d_G?AzyaQQJX-UL|MNNNy7aq^9UC>vaRg-1Y(`0sba{BJgvnGORKO)0NhiAul} zx=5{P7h?bfugGf#u_E|w>qxpf*XDNKZ8w{<^0~z|=A8#NYRG+;UFkCMk&LY{Af%Sk zCJY&1&67ZhsO}Z!eFBec{szFNO$Tl4;kyVBwzbsmTToP1NWzsP2}wKN!60%eM&j88 z*+1jzMg&M?{+eFxx!2~D(IrY%8c>^w`=jH0p#=N9c>*J$KMiUhq{XYSJHgs;5}{Qn9R zq`XBSA7+dxDD3@s6PTbTf<*~N4|qag-2r&yAd0no8=MF424)?0gRU6u9=VbR~j+Ra6`qHG-$Gpyq)gIa@&!59RrIvU!y zAb4hYmp$ZSOF8iuhM_gyD6m;iKWrg5Q-{XxV%GPA#33lK^7}5%gOV7f(b$X6bM>?C zM@y^~*_*5-6a34icb>xI9)!$2F@kKvp%SPuG7^0n(pc)LtU;c6a6MJ!Hz9YYm`S^u zgpkR~G*Wbd$|*y?tgR=ave&53)KFTnO*^pzb^YfNk~v4|c;=Zd3jY+0>+2yUV%6y6 zTOP0wflC?^PYS8!{QWGmnuI7u+IU1j_u8`jq*+StuAkOrBwJNTXK;LmJ+EJXKV4u+ zXDHYMRdH;nA*dwcHH7Jnzo3MY)eEv2=0T~s{zS&b(+sX0ur*L_$`6X+&41n{ zAVxq<0U8T!P?$jPhw7e zPV}s0%s4w#zQ<9jo{Xu4CU}RO=Hb5KBpL6iZuf+g1Y%4CE z9dE#QbHdgAVU!2!@WeO@pt>zY1h@t955)53A^9%&nyH)rwuM>5oL^y1OOg6nA7Fv! z_WZD(`-364Rz>B?KqsC*FHiz@zKall3xKHmouFhSh9>CO)Rh<7PaygfHJAwP?+@7h z(|5r|be>@N1*b}++ZwAgm_zKAxwm*MG}`vk#icK zUO`y^bts=rmF*>rIiuqdL2JaHK5X2R)XJX!PD^h~?%itAY&NPu!qulpTvwO%CxWQi zmMYA^iLFEx<^OIqx=-;|AhqR~3Q;ZK!t1l><018slb`^^ymPalwir`_M0y!6EV~_U zjQP`X2aef`Rq8T66|DiyIXR#1kI%>~E{M8)-b(tMe0;{RI2)}?`%cJO!7pk*w!il9 zEQ%y^gpge5eaAm;xl99B=X!TNh=GHUJ?w%l=|hJVpF)in9YRuS`~i+G zuCWTv$!YF*+=$=wKstMLv^h18>gKxJ>b+a2QR$mp#XIX_XR6K$4^Q{uOAG=c1jr}t zT}}60Hsh)vTDomcP&}V@KdhznTR`?$Oquw8LdEVJNbf7Z7azazLVgw(12c%m6}dyW z`fJ$4^Bl3zLb@g-O3sM5oqN-DC$mMciAq%kj)ohiT@2gWs1*paJUB$if?Fu;)sCID zK4R5woWxag6CSr6QDl)ugOjHbm5MG{KLCu{Q#spdlYbJQzX@GP0`LRM$aa`4M+?R+ z@HgQ>@Uld)oghzt_r51w*iRS?0?%w*4KK1S3~mqfA!HZ9H!HBz%F{@sDJ|CQ`}RJS z#cocjMMV`g2Im6U=EK<_`BwYMBIU(YjvY-NJ6%zEx~{xUf^I71`M~31oquY|7TE#h zRC|;q3=?dWxLI0g&oF&V4@L$sFR{!KTN7ZS;;)f^W16x2>&mGh_dfo_L>it;!j|y}afa`boohHAPA+lbdf&eV&hAEDujd!moA^>H ziY;!3X4}pw$Una-P@7z#v0h0fJy1we=o%2-Mc*3!QACi4vn1#+m}!v0&y%5oH2LK+ z@%=On4+l7_OJK&#N6RmDoy1cR738D0fJd!AkU^p}pxt_)sqBIm0bbL0+KfQVtaA<^ zkAT--Hy$nDr?*>TnW^Xq#-ItMOB)Y98)SD6 zhQG6L@;SolrelWXBIH;G{+rN;Qo%3im2~;Qm~Gs{=vj7L@Mep#TmD8(Z*qi69xpz* zPfR0!e#nvxIFjdRZ1Ov_y?<<19lrEr2s$tgyPsf73~j zzf-s{KUz;1I+)cQHpW)y5>ksA^l6~`=pe6uOm_4#KOxA6O^qTyLMxc$z*pTJeXZ|8 z_n$hoN=W*WqvYd!VW9&_ff=N%gq;iwfdpR4XC1O8G7!js_$%1aN3t(;P+k z3%n;$V(xk%8nC$P55nu7pYen8@^;841S$}mlQ7@lHrgY<3KSYBUBi65*vJK{yotK; zOxWf$uOx?d+p&2m>=Rp#WIRh+2UHK$6mn30?E%`vPv|3YG2&dQHVEPU^b~}x4n-~$ zzT6+8GPn}g#Wu4jpo(e+Gj@l6sZ5w?rHX$Go?D*r&-|GMI z%@npe8PoRT%khkPd=TwQp!z%m1Q!y%W&TS)b&3-y!Zd<;XA-E z40QUl99;d(bvo!g zuU`{$VMD@dWzkv2$WSE(cN9V%0^atj9zqth&#>APSos+Uo&XuLy(nxj8Zg6DzOodP z$0b3n131IMH2)LqDA^U93jjN+L;qSgwisT}1XW8E-=Wru&YppAo&#?4l&f1Y6s0zr z8i}vD^;*$0H7#J|ron~n$)eNhFT7kq|ArEC9>Wip<=5bP{c;JD0wk^fi44GK`p?MV zpLR9{vJ(EDLi2?-_7H|aM#`5`4j8wj)-Vf=2Y~;!GOW0D744#+BB{yVQhcYJ!q@SV#P8hR@6e0*nBv&2b5aus--$$lTv1a5@a%sr~v+C z68w_JIl{!1P`25i6Q}>V)*_64>Qr|BZtV|;=(9cdO<$+`bZw%9RQi@gYQL_6vXzs$ zq)^fnJ05mQDAR7j$s9@*VZg!5Y{sr-b$n`jxkAry<1=j%R65#yK+0=rob5_S#o*I9 zgex$G%J*VCS@3&9a#6Q{h7yoi6?B|UCg!JsUhy}o>Oc8E)B}Sk-<1t+`M6+4U^pQw zpfD86u`cDjd&)gL>|lGj8jWnGp6yboF-v1LdXMJH6R4dC5&Nv=VyXp(BOhC^`;HWfz|hbj`xo- z;ZTIfL)v=q?1qISnr`gR-A{@h*7_}(wYS1IKJ|!mAeVj;WUXscBGF+=FETb$;I_}D4lU*Ih_yx}qI^tHsmlc>E>i|8xXLT_ zecv{!`27Z-Q_8L%9nGZAEux%hbonMrbN9Xdu_9-?ZmLYNq5qv;B_T-)^rKRls@1#y z*87z2@iVzsHbB#}L_ArI0)2b|poT};5AAps~&7-%8$XjRNHHPA~Mt{RMK0y7k-oI&{oe%#=Z^CEKH z74Jt0HQch-e{!pC+_M;HPr=`t}p2;qE5WP!iH(wIY!?40TPwYchYVab|{i8f31jafp9 z)4CM}4R5x*zvHp-l&TW(i1r9T_u%x4Bw#VTg+VJJhDa!GZ|^Ivi!^C#t9{cDZuq@T z`(=$x)72<&5O~G#z4kdF|-h5Ob5K*zZ4QW zmx(rnvyL=jQ}f)ICvSrA=dTyyYyOO$3;q7*e+^gje*-w5^o2pl0)I(f7}!HptW+UV z?Wo;Z)c#$-;P)f1RLqY7FS{WHiJw{YVdo4nedVwA1(S@y99*5rn}dt;pHf)2%T)O3 z6fcW8g>{nNI}YX^^Y?sge|U$3&$cA(FKb_Q`ABn4(MP~Qzu*Ro7(*R#=z&cs6xgMj z86*~PiKBpNb&yaEhps)Q#(bS5lyUY9--p{ZwiR|%1{x$&_A%p%se>{+^FY~MDpjYK zyn2%AduP|XnZKMj<9~Xztpm{iVIdMgjIqMev+XR5e?xt$)BxkV2dBX!92wa;jg<2m%*-NmwH?1=Np>%!9rr`bP{VOZgWltU?;c*%;IF7~IRE zezY5V&duJfgSm4ln-kV@m9L~IKgkPlUo@{vVU60$tFw>Zmq88yu^xdp8X@%yX~0w& zAS9tn(@P((YR}ns%jqMaLWSK%%AbKIx1QVr*7kR1W%D@AQAS^AM_Vs< z-&}DyS@h%Z<-?`o^zVS^RL{h8#PC~WmI~PH6e9T+@}wP!(#7S4iA@O{DagGhgA->H z+IBcZ%~e+Kc~2S!7=sqvV>O5nx`!DCdiD8&QOyKseeQ}o$ z$-R$tnK(tlRyq(;>z4!iL8c!J@)}9d_Nu?e2D#XS+sp%<@fH|AQ@3JBgsI&>T!Z3( zE_lwi249uXl{Sfy;V3k$@WCygq!BlfgT|OKRFg}MHS%wk$nms;-xC&*+d1UQyu+_6 zf@eQ;T-hM{^h1DNO2?NuhR%_zxzA)?Swoe-55kp#LVq73>Nx0W%TVyHHA3Y!!g0tz zI=g7X=)j43WGcPfkGDytAaDQ*B0E!GpxCnSgoFxbKYK$2*ROM^BI^DqQ-*1M=i`Qe zRSdaLfE2wRCq$7pRs#4oArB7Sr$P2hh%Z}RDvu1|1JX!6+@aKcM17>LjPhZDhm@@Mce)&T5I z_=F2aCCZWEGZYDQz_)=i@&IJeadAO8DWWfw5>;-%Dp-|xQ%1p}SYNB2qs8|d&Nh;^LeJWGh$twkCAAHeGgiIp4fp1SFMVzN4|V^>x7T@$Ae zs2+A-ZhpijnD-Us94F(-e)b>oMxK$Q-Wx8f&vvGdIYzz6kj$FDFi>$CQKslJA&XRQ zqH=anC-s$K%NKwh)V1XwE=?m!j{gS_KMkI{oWmdIpLq?A>oj2I#_b)G8xGX}&b#=Wn)^^?ubO(4vYrZWex>`)5ZkreyS1A5?;z$X1M44Vfv@8n zOynv+1nS_^row)Mv$kV_wQIJ3Gf+PF4z({OjFPQ6JNhP*yS9{C8?}!(m=Dj` zzWTve&3GXp4{)B=Zd01?uEj<>NMjxpZv1W`^5{Y@>ZH9=@In%13WfVm#I-wvaHMg^ z)Jiahaw1Ahvjfd4b|B}1DCA3fZ?)NY_-Z*EIxiv~b1CUahIE=x%hkw~*8@KI96{mo z3JMwdh-?JD6G`LL302Lpwmr3wMVMk6;t-vP?Fpb@hEhUXTjUO|;cyg*~2 z$Ofs3wH%L=s$3ZtcYR*W#1ZQwU)8?tvwsn?H(XXwAU0myWMQ^_M{q^41R6ngp(gIoH#Wa zk!*oHee<)P6QH=Dppzb5L)?yo0j}2yv9#G@xI}O9(0WMGy-j}TfvcYa$sCJ);3);^ zLrsrDXCB@fhLHYqv{ywwQkBeM;||EAlb~D{a^HToa+1_- z5UdsU(6EesR%}=aC0SdktjcrRcydSj6l5d-PEP$v`LANsPmjhytcIUMeQB*?gv10a zE2|0hq_@yEg|Fv2-Y&Y;V&{MCQ0lrl#aFV!U#A)*t<-A5C6P)O;0@b>8ng?A zNVyrXpzM`z>Cu*oSbD?wCiKGyZ(TucN0-C7_8pHtwavJ^`c=k7k_AbL5A&&9K;_uL z;N$`jT(bk=9-oFgz+t%!+23hoFxNh+fQ`_gOCQapSCc<732}9p_eXwM*{YAh^}h){ zJ{b`pV_bFGy|k*Zw5s~nu>iH20M%&c&+Dq}^__HY1;2~!N)y^4*bswZ2!_yjJA!-! zI$2t}IX@fmc!nLXQ|fivW3+JLY=?YV%k`e9mzn0a77oVD_K zpo+;^k=xC1O*x<;H!KI-I40;6ii|U0=ks)7HIBi9TaiO_3^?+jS7#kIAAp*5cRuEt zy`X8_dXrX0+v_fS!~Ax&a&frt+ei1Kp13w%*YXceSA z#OFCv`+b3hWFqA%5OcGcx(COx3T8hO{50jWC_gR+2rj}8bn2^L!b$~J z0iee&Y$^cVslPa#a)XV^w7EI#_qWUq@K-ZQUA(p>+E%~CMLP|Y57Bo^tAv(Jxybm7 zVLIl)Zvmge2ZgG>7qpg|u88Z!kr6?g?^!&^^b^SwF4-q>V4C}C)nIpxaOzi4l;cgt zNMczDY+;w;MXGsbUI0Qd1cEa#6VTI=z}{4@%y z-58r!G?2ip0B%s=R8AK6-lnC;seJ37Q@1QPV5LrMoT`y}{u}v2 z0E%J=loJhd*aqNbRDMq>hy9z%DR1r>fX$lm9u^n`V>#$~UI}8s<+Q?@s1d$LXHg|@ zq#l~yc-?Mmd24z-w(_ z#6n1=fvRjE@xNt{gEGUpM92UIhuH_oVp*L-v4||La?j1DL#5GZ)WXuGd13dKr%(KK z4mn>rKP~x4peY}Mc?;;tf=)!yyBA?|cu+ZvRd6N`JYfVa?u>KV^e|UcjnY5?`eCJxFF`B|ow`Xeyw!U~7 zy}k%wzq^3YSpAU7h9BAjY>dT1DwFC!V+a8)KP0%FIqZ%qp#xi*(2*1%8Yv!%O{Vh4 zV0NN7u=#L!l0`=pgT(IRX556h<5x%WPQUVzdilY&z}8cDjYi0lJNoPMyIsH=cU1>6 zlI5MctfX(2QdxEp#}N(On?nmXU!$5*v9;;`;6gdqyPmhNIzQk0Qb*w>ebf!=wSkVq z7OgZOPxr<{e-m2#td7tJ{cc$_pvyYNF*!bV;2bWeh~m4tjD*Y83Qo3eh(S&rdV9ZP z(e{j@q_sNA<~+G0wZ%Z+ulU1venE>t3(pj=56;AX_R zT~yFsp+&?l^1ol1Dt%7c3Xd|mk^;u}r$<5YiGbI`FEdqK;EcBqO~rq@FGCybqQBnj zfry*^UwOQM|8M-k|3PK^JO2kC#J_V;f$${kO0g8zomDSbBoS0>Z^s^~^aTi`q5DC&m3|P=$Z91s4la87~ zr{f?sP0^G(F#~K`2C(uU&9mwz49g(O_^=ry*DVrt$ll!E+*mqXpLKUq%V46JL18rw zVcL;Vg|C@%eS}z3`q0L7hpSh7TtZamA04og7OK^aZ=6QhE&5FeDq`v5=lEAC-EC!L zP!x&)&q!x0wv9O@gd(-IW;9{H3EdH(nX6Fib(12Ey@2($ST~>%s1lhv*gI}^PGL>-TKJLiP2@BX>=zDtD zt-n^m^_h*=?S0=?8-!%!o69Z`#v^LhP`^n5$rXW{)$d2*IH+fkI6hne^mhm0*veE8 z3;v*GgiI~)z%@WknPJY447SAT##ZptgSOxPnl6G-@9z%uuqybqA@8tz_=XSje9m}w zcqx{|=4Z_Km{bGxGsrzhC4hM4DuHK886hI`V9p{b2yR3E`z0YHFRIS~UyDMK&TtYv zo;o@w-mXbXwqSEz{`|Yz?+?zb%CmxfNJ5u6;cAgE zx;-`cg)|iUl~J%_J{QyG9DB}u%zR-i3!-KJ;jwg0&TX3L)f{V`ID;DB!mOelh6?pKP^u?=d( z7==|v<@nVBo;e}Gh4mb42DOFM(pTdeUioC@?HgVS9lNixhrb>CIN=!cZlwl;v$5S% zcoMYv@D0{v&JGF`;{LA8-=}k4fEMnH#32Ag(W?9B@*njaLzn;}oKZ6LCI|lR(_j`3anYc2ESOH$e zI(P~$XV4-M$Q{ByL+)d#65&kdLh`Os9T64`VYzZRT!Iw4e|WV35J+qd4Ql{nbMosg zNb@>wO014hw~#?RsSo{drQESXSf_sCL$MgHuxf1J=?A){Y9RM8d26Ko6P{+9x_DZj z=B4w#zb2X1iOof`{6P_mgHO&ydsW!RxGGIhf(JDcYJJP#DnRM?l&qvr0qnmi?Z?Xb zB`E6>`H>4gGiz)w19-q~%EQYKvOKtE&uzs=o)-FCI^}tj*jjKcNPUahx9&s7%D})3@XC@^fR-aRm3TdR;wJh^n-JvdhCKx~>{J2D02x0Dku@4230JKtvhab2 zf|?hi2W5S>gvx?H15kGDeNG=h$KWu9y$o(4bKFR+$0{fsh&Z<$>~|~yNA?nX-@+>e z&!~@f6`ZKIcoV#{{Mz2{b9Zu%8cePub3DNk*Cj%O=fXsR>kIPeH$I2N@?JmKBd|Hb zVQl~$D&H7a2xn6U7jtYY+#)LpwI@$fYQNFn)Qcv4S$^&4A=QbUenl%_Rx&^KKzjoQC`U)vfsz!l(6K4zQ_Yu3odxwbY7-L+ z0bYL+{4GxbkK@MZzb5{hkPBqk-K4M~588mn7r|G-Y=QF#T0dQYRbxFsMiijGrqu6i zDhlZdkRYG~i2zJY&u$RZ_!)ATJI^d=FIqp3!-x)$@TDhpJh#)cDN3;}9KWvla+S-Z zVZsg}9M&N0O9IxIP($;RY)4G#T+fdk=g|)Bv5R~fK@M&r3kNy!EP}SidqINKYu$&a z!^z=p&jZz~%VbHK{?xhze6<;zKlo$@^#aN{hzPovB+pj8fCZp6V36J=yMUxQLFX=@ zkKe#XIWFbY0>5&a0xpilC)H+PMUFHF)Rp@BV%w)oJ8^t)xw(eR_h+Eg!~xdAj}xIr z7MzG4_>RbR5MU_+jUvASh^+nvq!X^7tS173#bHi#D#wA|0&-#~(sCJK0R;4pqEgI? zJ0A;LqXH{A%eoEATwm55VvKANT_B{Zc(crv|Er~8;A58Ez$q@J%Pv&Sq`AbPGtOa4 zzE^6{K6Z2a=h#Qx2M#+!NyVFAk@8Ofu(W6~Kj$YafnWr1m>D2QlaB@gq5jcx%j$4q zS-ma-V_iyZPI&HzEeG(`6}%Hc-W>YCer_wz=2|(TSm;2N_DO7Yz2M;^v%#k5S2_EV zfn(~|m&wW?o9kgV*<=nVoZGV-AfxE{elP@_pFeC_3?XrYrxUc0D>i7&Nf7L=U5KGBqaf$kM(Kxn?a`{4OXin9 zJ$dIPbPCi<0M>96SVPwSS&jIr2=KdH+%pI^BBa(XrudO3mL$Yb#`L^Fb+ey`M2s-c zmfc3PouZ)s8L&$yc1^|GW1R^xrkc|V(3!4vIqerlSP%C8x<)<17uURh#Al-DM2Cac z?GT~t*ao@q=<$RWpH8UMA`FOz-6EJfyXwRTQ~rqxr((G$j76U|B$hSzht zcTbUWqcV{|Z5OPGWns44(eK5HW8Pyj%Wm#oeC*(P&zJ`=`vouS=AxH?#fJYXIK5uPX)0$ltgzow|AWbyQm2ZJtxcb1mYF=4Wh8E44UL!*w*j^Fa6p2T zS8?eV^S0b|2ZN+#>eTXU*q*`^HW|OLqlX{<4ISAGxkk7(q~P;F7(a}t`%(XU!^QDQ zv&rkY`QPWO-$_@g5%ambIef$vf8R&rcmV)|gKOir0H3$UmGgb>?)Ip@ZS`rum38Xu z5-2&yHeU(iR$@j1?BP0~et{7~6TU#15kyXy{Ar^gD_ocyF91$(Soi4Kw#NiLDX zS0x&2)h`|<57#xi3~y)Cv6YVrNjm3Qm0D^AMcR8edDqg4qRn0>_Bom?yrX0G1!&^;-u?#~#(${1+}*Q0$7Nz|4z5`%&}_UE5ijiI9jK?+GYaWhmx)_2v(5Jc zyI4KvaGwWCX2oT3`_CNNxc8SL<*(YT6Y5u38^vzRD;{_+>!oq=T>K04wM&;Bt_kO8 z#vTr(+8ElquWc-`G%ScaIQi<6WtxYlCA-=Sj*~b!;`+(E9%tMMLRA&W^xjuwX>GU z7*}o=)+XkAKyC$N_%atNHx9z7x=OsFD4MU~EeTqHzVG7HR1DhoR7|el>BY9+R=43| zo0ZPlu45wBm*O3Tkh@oxt7ZU|+u8|uCvbwDtu~wQVkeU*UuJ zNS_|$#P!kk2J|fVvnTfNx^pmH>7;%M>i}ngFf(BXB+G+!qi+R+xrjsfIll{L&`=8? zW&ZDcCf;Fn0&WIhFe+PxyK!}^=~`aJ-}`s#Hy$>1_!8r+r}}GgV)n(G1FLMmEC<+K zJU*nh3c}4z{lpxQmti2B-W&W>*U>lhF*dfjlM+gOqFXr>Mu==zumPChLZgLM2YrX7 z&bUR!FM1n#snFI@D5zj1DEo(puz6xo?~D8kt$(fc^D4v!6+u|N2sD~`PC6H?f$4If zgya3@wVcv(D+7|==0sjxGd;rbZOH-FQGbZ=U$Q}x|3UBm2ZehY@t~zg@gVw2mzetn zhX3>;!uI$fZWp2rbV8yB`UWx!kO3Rcvo3n&`KVE*a^Nwi#-TKk;jX2htdiDkJ@x$4 z22oGpfmc5REmHxP7hE$uW*i-T;q=+9@J|7pkR|v3h|{<#Q~fQ&CnTAX85vSHnse?z z*yfKyIzcfV`M3Wj)INl4Ho*)hiFT?6!ny%H3ivK#$$Xkt14Vv&{|@(9-!tOr#`@oO zCAq7|c4g8#jfuH%p2G?O6sHFgWZX1`5GrV9Cvx!2nkw>`W<5G`6Ro{7YRb#v1t8@w zkfL#1(QGWZlz)xlseVJvNrt(zt^L)L6ZX<&?vMWub?+Vz)%yMqJ4?lckmFPmIi#Xe z$tWR7qJvOPr4o|b5=xdsD94bL$|#i-CgspU##s_dqMT+>Mbwav0-t^2;O>v~_O_canHwXVv8^)feAYlbUdi~5y-F6>}Y{2uB19ArLlh&@b(h|7nT z>Dg<&9w4tAnCVH#D z&?OYxr)U(s%5Qw`y=$PSvmo6t3sk@Dn>c{bZknhapiy|yCA7Y3z9=PPTEgfV_Xbe=Tb~z}Os~-ilNP4*U@S0zk8S$pDvF_EW&*kA1YnHY2Iqu|huWx0UhR+m)l6(#!jPd4t7A)wdQ4JiW;u z_ATiC)&Kg9jwx8;c8l0BJ;{RtT%39lY0w}5;?UQFtE#d`fTEMalkmWtY&V$_pAi4S zUn%XI^x&l#`$L4>8=OFKR_R?Eq%1k=;?Yy{tcWRT!7pt^yvZL8kkU>d9=`XRzVSM^ z{o5bHynR~K(NsWIZxBW{GB(}SfXP>bIva#d@PPMjkU|&*>jhvLm2|K#Fh7mNI_tpS z^{e1r$OVX^F}`U9q1^!B-8g$Vr5RNT$`@`X5nZJAJoQ_RCpPh?mJ6~VZlH|YRk9l( zHsuAU={uD*O`QVa2}wuZ85P5`e2L43Sc6JFPW;7**@xS&V{O2zJr%$M=%G#tfM$q0 z@l&9K_>NZU%b-?LfT-Wu@WIPgb|itmHQuVVcGqZSi+v|y~f&zdcI ztyy8S-ul&u_E|0NzTd!yF&wUJ&mGutLhu?S%`YD2&O&Mi0^o?{Qn&p=zaKo2o>tP7 zAhB?^PA1 zs+0wZW7aNEP0$ZhKoQ{6n*u()U){QpUmK+GddQ#qf)NIE^Jt4#)0EsuA^|?WX=|do z+g_ZFf@wrbTRlfq%(N_sJ_Wg@Lmr%awx}U!tYr(^PX<^vkVa(_Gy57rLQeg&gjoMw zDPRw^s1wKGq?nNy36!xOelVv7pHKUDZ31!o?RMk=7YL|H!g_@4^mFOzrF9=zRa>ER zCc~}J!A>^wt!csn#};*!=vZ^)di`CQO^D%2R;IcZM>yD2?vIM#Arm#^ErgBXXl!nS zHq>mSsG)phdR`uhD-J#952!4QMjz!E;_iaa%$2>Z?-G~#E60b<#TU~uVsvn+Z}8o@ zCpZTR;#~c^=-Q1-!VDf(LA_48CJpLnK9|93>x}T#Z#yvV>bX`r9gp|tUpO#4i>!3g zzjajfB=IhFVlnGs`k5?0vd!llc5jDMxj|3x%MU97#0z;zBX4PdA`gxt{C3dVo;d6} z1rF;MubprLZtC!Q_^^JV@(@_GA_~7k{xsSH74p<7;-qpDY5d_ffl)r2<81BE(bgM> zRv5i|{g`qWrhzX*K|;hp=agy%*s;bov6pAMbjx}_aw{lYmDF^bu+aXX}R zCobmq$pO`5++-l(TIzz!vAGe-&n2ru?qOp9EF**G%O{PXqLl{2)H4FV@ zypcMfnu$uL&D)*%au7aaby-7RY_X@pKbtehrvvA1Ffi zL(rkpJF6NCRj2kQcl%c8A>9oX2YkCDBEKr{VoUO>(r2*m%lP}hJ^6CVCAc-n$Pa0z zHaT$b1eQ=nkJ4t-h8$qh#T^}NB>)R2njLxAhbx5CBw*U0G{_L`;nU#fJbFk0{b5LI zyMRkUfG{L2K#cbkhiL;Wz0ROlYr~9fGTrtrKCn);LcWkN|L~2`x_pb7`ZMbRfZB1W z_FC*dpTjMdL&vXMk@I?RsY$p^zHp*HU$I+~>u6Q9C=(rYjO|@PQwliyu^_+u&9=Th z#Va0dt*dMbcU<%6$J&)8X$1XysTl$-oPFSGusnP*S@FpbUJR}jaEi@KE#kC929S~=;={dd~)t_dw6>!NVP?T)@Hxmp{%8(DjPeRhG z8oS43Sn{r%x5N{w$qBz}6eBL-GMG|VP1Irh7;6qAIfMqCe`MQfAQ)-K?vdJ|9kxLD)Uop9@1QBj7tDEkWvb&1+Z7#f_I~M zqwOPS^xRK@?M?biE-pV7((^t)>b$^EtUv?#I_)JL4fVn$cKGAXFr$4g19F&7VEFX5 zuU_E(Q^1~vhI&9c9U|af;6^d(3Aj1fpUZ1#j3m_wFJ(YF@RKycwOolpn3O2+$P|zJHys2MI`W$FzOqX?(r*=J2cE85t?L(ix z$tSBFm9`X?mwu_`8?APRdm5NGEG>qWUj;HXRaDjdI)jXM~&w6G$#UZd&MirHgqONm>63fYdX&Q{`~8JcJr)L zM@AneI3{g6tNbiNJ(5>UWF3b9eA>k4B2T!x5ouZ}h!jgiSGP-pNT;LDl_^@z+{Y!@69A3WO#yAXbRn~ox8+0~cpyu0a$TlU)*FJm=3Bj4J-t0t}} zr7{46mwuzL_P`FTh!cSwc!uCWWj3LyD_EK#oa%uJ!=jrar7)^ur&rq(e*S7?(iyFJ zi0&myp$`ZAyq0_#Iu_2*J=gf|wQ9y`sc)*U9HypCiI;$;=Vk{y#R2q#=}&=V9hfi% zOg&daBcWJ&q|I`23woIAgV&{M@UG*ltS=BEjCajHbD)^AFq|>#fun=VvlqlwbHCfl zVN42Y04HcC(yP~V68W-&fLo*yx?kgxkAS?#jOFq#@}8%^8IbE+hMd7@)gq-P$YD~e zbvcr+S$fB^a+hCR*{HSNJ$D7R^(m))DNA7mW!AhLUPt|r@^1x$^PY;&&u2AY^Pf*f zvgt#KTnTIk?|$)uYR}~~22aLwbEW>g8o}$UR`xD_cKzV_qYl&XmK8^7tUY9oh@U2G zD4dH?X!G@>u?_%ajR&tSj<>C4W;kzQl93!Q1+RymNiy>dTzd-b0KS5yeav+(UA1HZ z>`;*{!NdDR^KZ`N2U(2kYsQQlwMp+*Y*RV!FaiK1bPa4#(h?W0a}`803_>AV2Z6Ty z6-|eq@IYO6OHxq{%Ztuc^xr@GB}!ny$L0S?`lvI~w`)fFeu)~OPQLK{?KenXsQMRm ze0CB`BF+}wVHn5Lf{==*6TUVOAaiLU{i!YF;20x7T$YABXZ{qpGHmR{^0uW^rl>e& zkMH`raWW;`Z*R9o$Qr*|t@gXY>TeJdk*x@f$qsn18Xx`~g*0{&v;ptjYL{rbaGW|8 z%nPS61{=_24opSv;efTMY|k5mGU*?kx-CQFwao>edaW)Qyp?;m^2H%#@z7B6502QU zU{j(v+{=#kU4^Uy9-L(}KP&~0N&q8v+KX?N(xZAPFrk=^$Jdg(#!Lk=@=m_-=%;*n z-bDz*Nq7rUvUrx?*WNfww(I3_ruIbkjU{DsG8;8l=8A~5R*QtzA@`4xSjVR@na5lA z)Mxj2-2fR`WVDeq*a|r4=(79#$ z<(C+FZ8jv{Lf?w_)|yl_pkkLaK*medeS-y9n7w~UN zmcEx=_`3!s&72exPqYXy7+)dD2a)R5PCAJ|v|mUQ5B)vZUm?D7*(A6*@wWUhd8O?0hZwpeabgQ@`jPYCy<`D zpk;djk}K^Xv3AiSU-9C~3DW&0k~SMElBL~zZKUq3f4i{g42)rKzq&x@PKLO6F13F) z5j9B0B^)1ZDI6Q$S_bf8l3_t{xppGi;Oca z0#wzaAC!qMuuJ!@(@NDXKA$q&=+xqvCs60QYR9dY2R=SM9N!;L{(Ky?nmEj0+jZ7c zyTvlE(Rlumyeo@rT2Fps#%zuTBk2QBZDJ9dfCcH(27ir^GQv)a;|jLXv-)t&^AY&9 z8gi3H2BxL~8^5DDC;j7a9z}Fk&X>*K&wdSV?Q^Tg{D7Zo0d#V(O7*;*nd&7x}I$sVRYJt)tA@Hipq>$OBgXp!iS2=4vyU z8h2KEDIb%*;kqmI5qKOFz#1z+DJe-NhXpQ%(#22K$`{w#y~@A*vUYy>ar2tt(-0d= z`48G0M06HOtVjcr6)=v<*ax3;ZY)2)Y=8As`l0fTtE4zZ3-fr7jsU2FW4NcaE80JrbN*sPY{13^HjRvB8=A^VH!z*MOoN`hyOlax8EOO zi~`AS`U~-`EL&r~s6hL`82Mw%C4})wA6@EOO`YvVQ7LkL>{q!uPO;T2hptzgK6~?W zpZN*~vzpp+cM}%1N*Do+8L4(a&V=8HTckf!Z;2Ts2Z{5nQ^r^7+;{4Z+KblYPD>=5|cBPk6)vZ{u9iJ7A22E8r z^3g03*2f$KqLy5?D_+$BVbN{hs8eQCRuS(FjGZ#k1*fwO#H@e+HNnw+_UG-HtCuWP z6iU4mOzwB%LMoh}Y~Ah^MSorxT&(SvzMGvea8Uh2)@q%k2)i&BnGWasInp+{i_BJ* zEMd1p9rc6FgcQDq;>{I0!<7xtCQj(UFxwqT*d!HGdKbwK&?n~hAtkfi*{oZrYwn`$)@AdANpQ(G>?7|kwcD!wTtG?@0 z#qwSwk=s(R_BGu9HC^y|_GufW&GjG?hy zVWZR?#yeQDjKWJaSKCmv1*ihUGSvo($0YX|r>n!(G;dfqoIA)!;!SDC{-Nz;`LbXl8*y633Q2xMclo zpq~Xar0K<+fMY2^++F`1z>3nYS_ypfTnG(+ZuN-lcWPN$Q~LU| z$A|To^<6(v;i%K_5mDX(l$7dOtCs7PyR$@)Vx1|xrr|;3;E92_myM!b`a_<%c!9?K; zzda^(8xsC?we$ZYR|mlZYS?0`>fCdI?L^j2VmRd)B%|h_+iEwlWA9D+CVp{le^9o# zbV*RvLErtuc1>zJ+>wJJGfnb@cQEqG>)*!fKH))o$4mO?6=ED|Qs)4_0LnFu(CmdS z^2HvMGxT{eYv`Og4qPh-v;bCe4(FCnJ(}ob;u( z+m|_q;iot36DNj3s&?=5$RYXQ-u$rpn-2nIDw zTaim3KB;2Zz*EuTy?))o4g;TAGMV3!bc-(O)6n*bQFz68&m;7Cm&Wa4%|_4s4N-2o z-(No^uUYb9EAwOPs`B31@6L46q>*j|U}Qbm4(C*v;f3)DqrW`{OtA<*pc)d3Nf9SDoS<Hq>>`-M}MV*bEWg(H+u``cR%iZ~(vlqX7m4D)(hn4(pN1?W5+S{TTp^$5_MRs8&B3V8ERYZiLHFMinO_8d`Hq*w1r)Ozc+1;M!fptEFw`)Kc5A5Fm zFji&+=tzUFU3e#scM%FxP5$0KEkWW;LD>SdSmDnDE&!g5fCFh!ya?)-SW9!eh@$3h zSzqgp8B}_lQ?);U6K|z*p`>cv{*bi-)rzVfJl$RRTsYAoZmVDeE>%~nfW4{i$N;njpeE!axqljt zCmFuokoM#BAIFo(76lhr)*6o>K&JY@?^i#|@c~xYyeGY3b6Nc9Fe5q8aQ;J5d=P!7bCZ3Y>eB_4Kklz4 zQo^sltG{qba?09Vd4+g}qS`%Loqi?4@+ZS&Y%{jSgL|#$hEg$|4Vh`1n+yHhl%K&$ zJ&-yLg;v`5BIIpo4g%3UXyICogs!b1O%o>?xU=?yP&ZQtpkNgCzZPNKPiao{N$>F6 zP%5PJ;-JFf=8;85?K6n|mbaB~E1x&^7rh-`HRpt66((A32>YRAp=&x>(+yoSIT2+k zp2hjy2i97=T4L$zGgr#tk)!JF9y-w20UbR=?VxGT)rr-7Cu=`iUe#^=df)NWh3A4! z#S5z-_fl6~gxE{H@dkiqqH(+cOGps`6!9AYgfMCz0CxG14X7-ubn5qW4fx!5uty&I zIw~&^qK!`-RB#aJXm!#4%>a+o;e29IBC?G&x?(nzy}GcWt^fMd)-4S&m5a>H^`d_> z?F}-_CryjBBY$*oXjY9(>*8RppMC}u176k0CU2qkEv({muab2X z14N+!jV^t$h6jGyy$5E$vySq3c#}Eax8|c^*r`3%*P`VY3f=~I&N=mw+$#z0eHyg2 z!d~03NVrf3KdMG#D~V(CsrV&2#9SD9bchkORK+XK3nTtldxj5PJ_&AQIW*x)OG!)YPZN;{@n90FKj5l z5z-)qaIzX-Lfgo#yVUQ*ZaSs}LMXySYuJ`hZDefK-Jn?h$S=|EWmAs*oY!_rwchI6 zx`drq&Yq)kG?kxAN`ur2Wdx4HS^Y0&_;zX+i5Ceii3yPRfR94K7iFa6UajpRsGNb+ zqIDi<@}PX){s0Zs>mEi#4|k1(nW!c>o`sIK{^nNC=kGVg-^-72+5Aml&b)LZVc;KI zuf5JndZlVGbu5sVF_Zq%iYSe1Rj;B;6s_&j7p==cy`73aF*`jxo~xUC=`2oO{C(3A zaU-ZOxSHoM9YR5kx%lk8{-B>WlmNV#;|p)U2ahJ^##NDI0zc4!x)X`*ToJB((V^)weO=@0IarN$v;DjAo!u^kzWvyCC?%TjUhLpC^SUZuggOktDc$Psa zF3a+)O6np?53IMjpS-5U#-QBoG~1$4?#0gikE0oasgR_JzUD9;id38+vg~ObS$@57 zP$7aPSz;C-M4~z%QEUHvw%vj_#qfXUa#ud&c>Ysh)eLr@!TL{l1lpm7SX`I)h~(lG zPR}F8rWNKl8@qIIRrikSnAQr}c+Y;@y`u7lmR^-r4`d&6e%y-`_#-2%^VfFtQ_Tuj zXy3?S7IS2P0OZ{>?v-)k+r8LE(L?|0Qx94Wjp=E_`5OiPXmH~Hjr{zd9CASDRmPdSGcfJa{w!?JQd)g2>v1d_6Zdxev{J()lCB{5$VqTFYKRr$duo zZxjes5`*M=GPcQ?JC`t>X|n960PQpqBY@oeE{ESBvqgP}R)Qv0uOv+k6W9;H8_+=x z$8nbZUtE}uFU;YE6R@VZ`9B5dFGy8Eq#IP!kdE&%W5jWV25Fov6L`K4*2d>1WwesM zLa;xZyW5SF?^QOU79Q|oEZ~nht;2I2&{ zyDlhY5E`1mAKlQ3)9=16^jFMrF4x zggAG5TnP;TeKj4`_o`IY$0e+|W`5+Ytc*)0x*Htbl^({9Ze+f4RzHw=*=khAX`a*t zS@J^?Thwc41=4#T*o1hcxS=5KTI$_+HQHfn7f6~5d>L5Pqx3JzlcMY?<^k4Nwc5hZ zK5?@*dFj2$?%urUYV8@7B`bHO#-(nE#O3M)*39&$LrSuhhOTvuLv`sTp+5!e_-v-` zd^S_)W3~MfI{AzpF=W8q1}aP2F2v} zm^WZDmOafI{Pqq`XCqOY78tGO?FnolvCfGzFL23_W?IR+S>(zfO7*vrqsRLV2XEbS zTxwvca-(+P;b?XzEooN`_&;QHkr(y=rU@7$mAe39@n;ENKG}lW1imHbQ6ot%4XqBP z^t_gkth6Q{vY7kIfi=x7TP`0>SaCYX;kI`x4Jc~iO}*dr=mM9Xh~q5`!GRRctj|yrS@Y#HH09HV0cJ5(HHeTDLuW{8FVW zlF?1e-ZrCcrxpIe%ZT|C$HN%1zjHVNG^Rt%3-ovXmyO(A2p43(W(b2QO>Q9wa^nkN zZdnAgIcb})N5bE2fjxWsYd$elN+F(5L+(PrIugOHEvDDgIx~92%P;EpD3np9F|Fp_ zgwBSF6wmrwhPl7>t2TKmuj+dt8BczDpTar@hhH~l8JgIAJ6DspUlrzI2tcz+iVklD zt$X&!7Z*IzG7=eSC-zta;M$9q3*+v#zkUGToile;CMnOie5?(Zu>o7St(!U(ayeLr zCyFSnAHkI5xQciKN29MgPBLHBbvmc5K2sxXg73&)-z$6MkA zTi(lg*Wg&{g!S-_ReM26B0${5$wt&M@VPHHA*T+~;X~j@t|}vx(&VYX<`n=f>A%k_z{`q2&xzl_+fwyd1rNHc8zR&cVpVrEdswM1q5PKh zc4z#sU^gwc?Pf}EuXXr>RV$B(zpN8DZ(AxG7o#?hG@%3vZu}wWW&b;R5tDzis4An@ z6>CkT+0fKjYv{PNak%UC7UWf(a8{7YIgQgET$u=*u$}jEVTh3esEDoRa}xEc565kT zMczug@}AV?j9x;TRzb9GOFj~sQGrZrIZg2{ouZ99tD`QjeV8M^u{m8m*yX2_-BdP znmR)jSDjL_B6Yl!ZxU&Ilc@H)?bQFfF@@r&5~R!I10*LGftgOb$-rmP{s~pKQ^X?g+|Oi{cn1qczXl;#O{&)UfavRhz;Iz*LFJU054> zGEl|u;6B3$53P=G6mN}}qfM0G6qdpUf4M#PgA`;I&(hG1>XSXiP1!>aQO~YJiQ6u3 zy&lkYSuF{_N|Guv^X?uII1#A(#yr+@mu*2&yzKCk2AQ`d7lhh(io2wqu`u6Tbgiw; z`t&-j=#qe1LnEl{33lKYHd@hw2E^SS?VzP`uoOTsQTc%3`4zlhf2L zTI;%L;DVJ7kP4UjqkPLZeFm4qBPq|lDzbO=Xb$lh)F`L+Wu6zh56D#8r)pi%O4wiW zY*mhbiC(3^1-nC1%e=Q=4YThIU5)!tfLBOOmU|*V32JXWSjlFnf7g>+BuNk7)}u1D zOS(EYICg3rFR9wPKLv!teBGao@9D$>dgUpk_gClpASZyxI*%~xmW|#5fqL3b>};U3 zsLhBtYXBXI0D{GbQ#` z2$rtK8PF&g8i#5UN{Zs9XEctBWTF$NktW;tcJapGN4*M5J48N z1DEy?42LZiXW_YEnL=BBL(O&+zwy7`+uOF)#bf?+QDw7NRdWQZYy^<|{)coh6B^@D z33%((f$NBzI}^Fn2d&AC;9rG;PTT1X6<;l7w5iD+OlMk~cQTHnWA!P(QjuugEqGQ4$b&O&xD>90W@CB8{wT! zoQ4GIzpBvE=05qe+H9|Aa=UFBzPYgLhn7&EPv(Hs;I5e#Q+}K2Osnb7|H)woA_7vP zdi)y)g4rttxX4H&&VvcM5YZ^0a(4$9ejJ*pbJ)>z@?>K(_To}@)9VEZmsElqX;Dzs zdDM~7>Lsp5$n-V`b zU`hb#mqE1`P{rlj@)ljEwLGJ}wZ$dRfd}&u(*r-kDPyMn{HrOkfr6>d6UWwhbyIj} z!BUUEQi39rz07gylnOjl7+%5e#0|NOzcX3POAK7>o_u~yV)vQ?x16EPHxq|G>DJ~M z>C|jjnWXlb;R_$~A}=dKPZyOts|bC4Du9lzQ@9snCVB+9+xF;~|H&ZZ`Cjc%Fq938A_8ghxEQ-WQ|Ip!FfjlvzpVS&L$>m)OaNug9 z+>(RDMnkH+`CNmJuQlsZSI*B|p{*J@x>Gm>KM(_M22T;*a|d3nCds3byvTxh4Nhvx zIUA!{>D_L=vE#-r<0H!cOGSl`{#IjMeRDr$k(1CwL=B#e7<>El-XhhtOYp;b5`rHeb8l z!g2Wh18|pr|0MZu4~p$ZN(rRzx@2Kw=x78a9KZ)~ z%pb;(MrhPW)myL!`V0cp_FCfC^pJvlVNSOnYa-7f8JA*B-pz6@T@&!~==0$3y92!Q`MAF!Zbp<}V?a~R z!(Ks6+aqC~j@Ex$ZbxeW!C`>3L@y@=Z#C8c`+P9)Gp&FcsY^PFCr!meHs};SW27 z*{OLBQGp+6GCrhGieSJowu?o`hFdhrABC^3t+c)qJNY5+adzzKZTuAy^FlY0CqKrz zMhC256~3!LvR(}#oq#Gb{+pWc!`Nhs01mq{&j2#uf3{rL&9q$ScMxzXjF$r>xvIw- z7^KT4Dt_*b#D+E6g;%Z$WWgFa)HQHk zpMr=R-_tSV=38T9{zgIR4#uzDl$=ad#9-GN0?@0|1Capo0f19>V$NEM}6_Ka}>@x4_-G8{!|U= z18s;1IGSI*WLf^|eWzMWmJip5pb=ByJ3ITsjoU`zZHsBSJOWOz;5|R9C*QCoqAM<} zLzfXN>Y`DgV`>NZNn5n3z}tYGO4dTv}) zkwYf>Y^2*YQayS1>s01Pmx9)8^6 z6tsdM|Je$X;POkF>_TeUq{{u?cuw0Ri1@aAk9`Sp85r1tJRi>YXDO6JL5J~b|HF)% z-#&QeEKOio>8|%s-W^|7uy$3IM71L5>4bzJmqYwwss`q&l@PBmB~GkzhxDf&;v56_ z%EDW>KR~)1P>1+VYbw5+77PBtiq~jp&l`{GluI6SIUT$@EdJrSC+%+=pYaiOu&K&L zIb9cTZ8ZoxMbl;>jJmilZQYt>G)Z*I4o&Lzb8SA`=J8cMcEiQTpDJ=TfA}BT%Mp;55V}ObT2`V3>II{nZ6V3PIcs38QQ|Mt zA*8cnZ`D>62tR#lL!V$I8Y5E}U$cq*38={dPg`qH`^mVvBQLK(y$cD0Z0f^g5_TZ) zj-N8H`}2|!#=TTYKrCLw%c;g}u_$W5;rysdPFPr^sb_YMkB@@AxrC3dozmv{x%C3z z|JQSWRXOgM~Zlv z@)cvf!$`nbcHMg=|Iq};kQFykhi*rApG-MyXXiz#wZ|K&Xk!bmM6T)(sKT_dE%A@k;(IpW^BgA1~(Bn9$ZE<(x zol1Zm44mjt2DG4qrM$bHf{&TVBc+bU{_wf6-z%6>re8&!^(A2Whaz+3L3@u@-BJ31 zD$dTa#?|3n_N$($NoYoLt|6uIq~}(OKz85LO~&M%e$5RxYM&{6wj`k;nv-r|9{&80 z)IE#xH3N@|17;OR_@o-rbv@@&Hp6JzukiZ@xa4I%Wv9qHUf2ZGBD4X6%xbDE{l`@6 z#e4lz1#4JkgBRA9y+62CcXnueC88A593%rUhwuY_^Y2KW{7n;d-fQF?&`Uvgu^xCI zTaQyGhSJ1_=fA8AJPD`L2|{kNNa;G#VX{JCiyGlWHjKCT^JL0zG#PdXS2X!_`s$sA z1-dJU>O!Be{Vhpd@f>7+8@87vb(1w^ioUQ(>ZIN8sPpwbUhvpU*nI9l?#tsr%i1kPQaEk+ z!ppoAFA7^G!sX=@X#fegZFlkO zZdJ1~`#N2xiydJ_JKrA<>soZ*Qg?+)s<=CzZ4L$Yz0tkJpbtWB1h#-LG^D|m#LReE zn0|UGm5%DzI)6FK%~aBIbum1{^fh&~AR0@3#!TS+5%V4qNB_E9%_K}^t0~-RRe-N} zaRd=hG9&jK4yO+HK)?Z0kp!aAIkaxBKDq=-tRh}H1f3$pOCAohqB?shXyv#pV63_45Zb}3@z`L z+&fmPRGE`2d7I<&*nC6l+a=PkpCml4I22F*vZCK~_KtOi!NEo&%~M0CB;?^#mA7<{w0 z9`AJv{_i>jw!^68;k8WR)j-qQOkuf33{7#@ z;qMQcpi}ACQMT9v9|lv5yDz-ibNy%ao6JRSTTa$4TDWq*jNoEguz!=K`W8aR5t6n5 zr)?7MmDs^gY`N^X35zH}9`@q1&4LJv5rXOSsl~Gb4zU`lBv*aZ&oMU0bRMvCep_kp zPSS`ZD(fO}`alFMszPuVF_CIv+M=)%^E0NWI zDv?fcCb%|B53u(kO_@)Se%&NoVJcMNx46l+>6S@Xb~;Ybn5DmZETZLQ-n&@&bodB?~4FD<_HY&X_JZY}LPMZ#I3d+nK>s^-tqg z`>T?ZSDNmdnG#3cJkWGm$4ni;(jM2RCiOD8;URW;QlQfHFQqJECE^GQsAc`hzeT;Y^vtXO_>>hF(Rxz zEF#;agzE+_MoN=`ajb%LS!pD07+;OXwjS`_`LaoE{wGgI7mu4x_CYsSZN6T@kI8ts zrA0e%jI2N)%>(PUg+){oWmu#iGn!Bb8^d+|Oz2%Bl>}*2sQ&niD6`MY$H$XO_QrcY z^b1{~BDDEicaYIm@sX>bt4Mc{*Yu&+)k7T&eXangQfOts=bcf_QdX5j+~`sy#yGik zZC7C5Hw)JvAky$$fx{v){$cH}>$80u!mxZA+m zNU~aR$>pHAIv{tqKV#)HE||_N_X; zIT^uD(W2vGD6-T*LyXL}tnAqKSvSYn_VEGt1m~*C1kG1T5036NzWU&)1iv>sQcKYC z57Ju_8&rQVXx$$aj-?zD!BAhYnimsT2hjk(2&OpKC;*zMOH`E^^oWTKwqhrm&l@{< zyRWrc((fu$@oat8QJoe0uYvyD?1V`{J%zNm5+?RVRNZ$R7{VpEct2HZ_(?PN&v z$;FMs*x|?R1J9i{SN4Pr?|-@F`1g>$5J+9k5?&guG@sE+dt>`01D6!#QE2X7Up2d0 zn94Z;huLCD{ym;Twfj=)B~4nlrd@b+Dctbf>GzL!2&zMHNd8ceH<<}OipY5q`w2am zIH}Pu47F-5)ZWXm;_<}iR>aZ$McYWf?5Pe{fOn1N+MzkQhj-gpj=S#1`RLYsO2~NB zfu+i|mv$$d&A)O)W_`})=Yk1>l>!&}(Z)X_^Z#=^xF2x*kNPHU;ijeu!GRM(S$8Cl z>~C)8|M@|r7~HL^Yk38?V9{5fop|qOhMUqSTfKM-K7aabdm0F!?$4wg_-MEnMv4!> zT$>ImbUAqvK;ugy*TkIoDUaFDVO{zGG%+%X&-A8-ZgrDn+x;wBx%SVRcBFfrD%|n# zo$U(E>qDWEpVsx{gWts#9x0Qe6Mf=fx+=f6QXmj+So**i{%{FcN#Xpf69jjJ+rklH zopdN4L6?d%&pgDN)l|{kN9eP@9l5WcDfv|0lFU1`yywflw~6bN&j%N#;13mydVdwl z_cq{CU@xyNN9lOElLCJI0g`Zckj#XiiEanP6LzQY;8`ISw?fAl75`Y>6@7#!s@@QQTta*bDV_x~U&_~^@a__>`* zIy)G%T-onCcRNAi{S^c0JQKfzpLvI+0AO;E(vJd2`E`65=yV&fz}9!Eqw+^AR`!#b zJb1h1yzsLKwDb@8c6)tQ-p>}n74q5YPS4%fu=Fb8FQ%NmA*lRJtbLbwd^7DD2@U=! z&?QY8CHusZ2IhD{7;e5G28&dh54G*(efY1OLcA?`$dX*6iOIs$%VyE{uzcVP-;N97 zm$wKvU#CS;;5cBaE^KIGmgYh?^W@fMoX}sal+QjnR>6r@y5AyW_N`=0Nd3glHRoIs z$*o#|4x&h`GX(5zVowV%C=m`ZlwjqOd{^V&e+m@#4MjZ+1jzJ^5KB7x#?>jgyt#9A zuJf|h3n1+*Aanoq^)<)l25vtDD;BSh26MY42H8_!QcLw{U`uz3kAhwl8w!AV6%D~~ zG(McN4PJ`5mWFl)XRSdH#Mk6u<fJnMuGWvrz)#4>0dRU9hg$;ly37b%j(q&Jt|TaU z{vgg-_>o*rA0mLAKNDcXdybs6ji5j4eZnGJg9q#R#Qb3mj2Abgo_Z$ao8>7KuCBUD zhZ{0qV1Umwvt24LZ>L}+{FAg19w3W68e~g$< zXFF5Darp{2PjCumEPv_4XOmb4q;caD6pkabJZK;hK(O$)(aULsP`LXe+Gs2(mNdC! zkj$}!D|szzWsqXz4)Q`AlT*O$s-XbC;tp7ut>37CP+tPQ(-jh(^!|g0Kzw6#{)M$e zOjZHCC!(4>3sZevaHk2jkoj*XW4a^!FmN+R{rYU=FQ3g^ z9AU6wPT&F{40E91wA5q6e_%4~4W580>i&b4goOcLiJ~?1=C1XEbS1-Nfo|%Z} zLi;V>qI(o85rI`fSq>k1M$-uHeCGadCPe7EsfJDN-@gZE<^B|SLQbQyHldz9iYBVq z<_|^4T%leY!p?#wd3n!!W}e#b=G|5i(~0hRgLswWgdp^VfG|To4s?PYtR0`*;-ggb z5nBB?QpzGLkbVT803fm+Q24j=WE3QH#4Ct@Li_%5LZ^%P=ZG5M$c~6Pi%8V6h+8kY zCpS-qZ_28#6dBDr7Q{8GH@NMS{;CsvlaM5US+2Hp(@iPaG4=i&fWx1l^R&4wQOmkk2@au)Ta*$FcT|U15gs0BAEKl))R?vQZW<(xZ*rx1gGS$R50`MK`~4le%NOrP1ayuWL_#X(-m=Uj9I1 zOwyyA)ULSRtA@vqKgoy{sCK%2O8U|Mo*~S2J}5agI22bUVI*D+lr5&f`x0Z9#Ki=z z5XM7)wO+Tag2$)G=Qk^Yu z+xul&Je}-MFBzOeCgd$BS1dgg#*z@YeLRTNV76V8YeC0fWsFg$ZlW5X&s@q#lcG-{ zOL$@sy-8vnbtv0tto$9F&XN@$5m>KREY%6>m?$^)rW#Z8XFGR^jIge^Fb*bOPzbxx zFL9dl{>LZ1E2AP|<}YXOTe?-G2V(pYc@EQ^aePhH$D79==pgPu#NuEtrAx2_Ox7Gr z(2}rMSR6PI!NJz#K7aVAG~fVnY(~+wBcmHP;OfCB9Lg;-7dQQW@kM=30S5rFIHlDC zJkM_NKSt!W0xKPhsGzuw669nuOC2}!g)bymh*wY+)Usu}rC#?;Gvpg`+}ApVZq9yT zaq5`cHsR7mzK7J%G$}0^t(wfkf;TiX<-nj~+Mo+`i)ArO^cW~+Vtc^3&)G+t$GIN8 zVTOK+E(TT40VR7B+H1Cr+t?wMW;shErCrhof&a4I|O$Ui_I@hdaSR@;~N4Chw!HqBg53(;>? z@A51xCBzmlM>o3qwO%@?_&F!<^UJf+d1alOgWrXmON{PW8f1Jtd1*o<6fG#aZ_^U| z?L^(i9pax-a3*rp7B#5t@?x&$u0UlcLz_(247}Qnul$(pQI+ZT4Tdn*Q%ln*L8tnGg|CTjnu3hR+@Yp=Vah~&HHeI?IYKGKEO?KQkF(;yc2q-j z9_))W&7C;DSsi2xEJ4q1K|C0ONL+}cwnp&gf%n2XiJ+f}|G&E1ydBFt&~89e;Bl9J zo}#mMlG}EblAmp-%t|bO(%s*CX+Yg>EZStZj15J%;`Q_Asux-&jn?lJ2ii@QM+R^T z56l#@?EO{9BGFGCR)m$;;6>kqn}*<32Mu@$n#~OnvXvjoC7Sl?8<&6Ik?306{_9q3 z4vFzrPX~`G0@5rZe~9pH;Eqp+{)fMk7q#LfXg?*f+>;_O*`CRImMhbXP+4Pa(J7PS zZ*lu zZ#br(pxL2qUiiM;lRI5c*oGS*-D^9x_@D%EE*fTLCif$_6liZ~pa6&BsS3i`BZ zAubfwPO#Y!GO$Xq2)+PFM*GJ(o^$qPr^s(H}>;k^X$CcAz!XD?Crc=ydEHQyz$ zrxtbS2Jh4t)S+?MpOKWy>i|Cy7W@==0h*g{2>yE~P-cgin_z<+W`4hU?QG<4!Jl6vy(JWzYzt`nsFy{CSMtJt4# z^`an6m`2vXY&Jk6LKDG!A2#3DGoT}$s@y)f$e}^%?yg?*iXfA&dE?%f{}xJ4EB~k< zHV9Vkd+1{T_!a`L@?fcHu?8hs+*86Rb0d?k7Qsp%T1)SyaGNZZq&kmL6O4#e@$hy>2^V-k!ZS(>PmbvyOKf_PTb zVh%!R&{1AOi_7VpiCCUOG97Sv^!5zZ`<}YpYacrChfKRaih%j^_slnEcXecl4^>D9 zk)p23Ibd}>mI8SwY6&fi47bs`0S_Ey)kl&xDa9C{0=W`5JBcaUkiPairH9sLF%j5f zM;K%~*b)F-Q+zDRf{nCc`^jk5=WC*r3v~F?NPB^uIDBaOb$~;vA~ zi`r>m<*lG>jVm`U#oG#9^2jmoOt(z!JAZk%*Rj?%HR4RY&9m;C8E;UDbQK{n*? zuDQiiBlQ4;=zMyr+C!{9$F7eXLfwS#h-0uK(O|Ql(LIDshTdie{1&_tdxutMvAQlC zIeBL=)4R<({RpgaTk8rm){|Hw8IM*TjRavQ4ptFphY3sEPUb8Ij2UnWLZ|n{2?A`7 zaWJ4}Y7PV)!Tis&fJ^%pEu7%!AXGYraP1I$01EG8)Kz$rpgt1dPyZBpj~Yn#V1@gJuiW;_HCU{5lUM=TsB>BVOZo$Qhs60v(gf z{2unuiL9oZnYyV-+T(W31QR%;dNhW$c-Kc_M>$bmM=#g=peb|VX}vr+(TSH>W_7ai53#HR%X>G z>MG?62Dkd#fc+(?!$>f)`D8`av@s&v2RqF`eg?wBjsp{zaEWwF3=>boZd$b(!Le zkM1;U7q7Vs)idm6&|zKR2ROBx*oX`mL60%J0kX{yP4v9Q;uZ@rU&;am#jh3s0`F~$lTC6j;ZEx z^my_6;H%RnoJ?E>SnQ43(>9e{QOe6)@VYVKJC4yr-eGAnE*tQc^-k6&2sY+j7`Zso z!hG0n4OX`Y9}zNJgqM*$v{7W)Ja?qNl#{JmZKV`%dIzc7v5ENOsc6>UWt2!xQ8xZu zmHP4Ll#=ygmv;}!w<8yl@Z)20|14LfY=QdJB_G|Pq3BPwuKy?*|A=3>6IArJh z!e7((D48Mi4Cn5Xi7~oAgqp9}O7?&IdxYVxn~N-vg=6C zi*3hJ;uiGHJ<{TLT;novTofRFLLG{210#Sfyz7r(I}BITi$x8K$?=bDaUG=l4yU|M z(^tz}9je8jx;2xPWMv=0oFRSPoPOn5+3UlsE+Zffe!;*OK)HvOb}@6G4AP%C=68yk z{Gf+065}+}hL&j;vjk$=)PU1#7wqj zAV7w+1#LjNL2hfm(6^JXd$XXUD{TfOqjr}aZ^pt5Eo38fX4^Urm+~hGuVPIphzCQ= zuN14t-ZQz;U!J&O`+Ett)@zBLx{I00Kz&yaf#6YXtm1FSheC?!u(}uXK|}9ms>q6>c9GH(!rJ&JA+RP4k;K^pL;{D#Fj2PtqTjG)s>Mw$W z8b6cXovI*yk=_h04GDih;dh8pkkHlwXRzuLi{B7SeFG8-`o!ICXz3B2RmI3iWAv;U znrYG<#FYS9libF@T*U5ynnd&W-D-3NLe9F+$7t^7ia+mt`7zzh@C_=rj2wGvuQ#s+ zzM3X{?vMEIzZ4hw4oml}FCfPbASv2iH7$&Kd7$Y7O0_BVW@QifNEfDGE?A^ghrL;e~a&c29NPxoU}dBWGM_^bCAiU32<7<&G9*`E5xCkg0Wa&7cA*>`G z;M+unHBv4}o@cn90-j-RB1|VI-rQKdPOzZr=@7R2GU%^23qGcUSM-p*c9dMPutK`k zLPJULZ(-`qh@{g8kjBSFo`Rkqrs`_UH~xzfbTb|kxy!#Q1<}` zc!?q@O-l3R@MT4gmSL*MV!<4KV^3+B$&lRa>)vIC=_;(ArEJLP2(52i?1|e;47opG zcKc<3(2Ow>6W02|I1&0_%?d`j(}r!Pqk-m$)^GP4LU$_qQ0-u33uK2M)K~T~bl^r2 zIt5gI{N)$SYr>>V7p9tJriN9SiHjnk?EHp*{p_51;a}e{e!Ed2y_=iPJiAqcHXsj| zv9grjqXdoitjQp_W&*lUA&r#PP<$5>VhT=}5SOca=iJU{xUs+YhPk20!76yk9V9fW zCj|m|0+m#7xW-@X6Z=t^6yE44S!OE2O#KsXWBw=}KCC~iDUvdf>TwsMxj+9Cq)w}J`?V|~ZW6DBw7n1Bu9;(4{no1y*GLr}sc?+epo>pYx}oJ?+9$ZuQFr!e2J zJDTEo1~?c1g0hYw{ql598?P{um+%7g`ZtDq5iiRG^re)5%x(n~Zi5Df6zdP^`OO78 zpz#)5Mgu$Pn2ueL^ezJG>%Zk+*XAKZTIO(q9_vBFgJL@8RXhYwe)v73anK+?`90gk ztJ@Q6BTvRuIRzND-rsw?TZP487h&AvAGT6?=aZTc+&%_MxVlHSBb;0wuF-YO)MY<( z?rGc`yLV@o2SG#J%7+%}pF#NFlbOAWNviKzIN@Ov%culAj=%<$M&yaz5Y1O=b~xjw zIoiBbn0ep%i^iIx$56Lx@pUnVA^@*$3M17Ai-?N$)LZ8CbRv_Q3 zI`LFUESp$)il(t>f=?A(X7ms$M#3r2zk=&FA*bo?f}@0%>WUbV5HMvY=Rfgmh!kN} zo`uE(li|A}2kuzUhC$NS6Y8|XA?Sa}41M;R{pEVQ2Qd%wkJ&?JtWY-J2x3ZhZ^e3b zOefO6N`s_Lr|}+9f67v5FVYp97G4oqz&hw2U~4GI#VA z!A5;UmrF;!Hr#L#MRG0S>%Y_7AN`Z&{yj>p6v%XPK~HknknmB zEmfr)SHGyL@-NMqB!4O){NG7r|ARwPr6~})Y3c0*z=%5U z_3BZRUjBCStL|`#Gz2A?^k3384gCmF(5cY5A7s2w! z@c9_I4uP-VJk2RCEyBm5B==w(YES~^(78174kq2B8fK8z*wuEk)MLWBf*E*YV< zH={;cE)^ahLlM@&iFLzatt)+%k^Z3*Add`d1>?s{=@Ls&&Q;`T@O8KuxG>#xqF{^r z@YtLg>D)_wCBu6SWsi#dqm)%8O$X@6p?+3ZC#iKB2SK@9{KrB=6=p4=4tt@q3x}uN z2D-cIo-VQ1_9LsC4jwm{)GoLbT;bUqO}|%*T=J5Z6{Ks-f9y2v?Vg(3H}zg#S!kmC zPC{>RUU_b( zAs|T_G02OT832012c$NB&5bv{fYU_QL=`-Je&~uJT_&W?H1*lB_)RYjzrI>lJ-M?M z;{E!;7A~ZM)ctayB_l^_a>0CA;pB8sDVh>I<(!5-i*<^m%tn$xkP3$uatc8 z4C^;y@#wMCN;W=Cnuw=yl!OdbguE1wKK+QE2EUREk6+7o5I)G!Agh$>Jvt}gTF%Mu zw!WA)(Kx?;)wFpbQv$Att7tZ3&tbu(^9Tc4pXYD_Nfj-ZBZcf~XSLAypsND&%r(dw zG?je{+0#i2SahCY5{J|+!JlsuROv24uT_a zu{NQtsu(PGm+f1)CDtsS9CApNvRm;h^v%gz%&fQM2Xk$6br<(n#q(m)cHf!)>CCQ8 zs=d3k*Q^?*0mI|Y5U0q9rbZr$5nbB z4o+@ZaLlqM1Ax(LNo6EtLgN;MZ`8vn@tXa5xT;kRx zuG6%pjP0F$;z^NL<^tAN#|E&e@JKJu$+;k@?o#J1+bI~ozXj<7a{GKmMPZXDXp72W z?Hr?WVj|dZub>?LUtSO+H03}#~_Wpr*jm9;EgZfIz8vRFe3U_^qjaeD# zX6mB$B&J{Nm8g&v+;U>GX`R)=CZbcwMs}B>d)Q*46Ko6LaEQeX3=u0}Hfh!lc5@$) z)HV1%F1sn_6nEIuzy4-*d-Tb)v)2yaTW_^3eCKSjr;rbGu0ncHumY84^VGEq^oLD^ z>p+lQ0AGCbg|YHgSQ@P^iX7XD2MH|6at#?==*BBpQ`O9iIlKE|{Pp&} zRTX+&x^vHEWZ1n}whicPqfzOHHL{jhPL_M-qJs*<#|Py%$%OEDWas0*t^4Bd`Wwo`e0+$-u!* z*s+oU;Nz7A?M~_s$?*NB;!llD(>;Es0;KHzlkxBI>_0l5S|~w_Q#&94{4=XSl==4i z|NnT@XRv=kuTCNNp`0!YC35t$mUC!@%ThQ7pIE(XSp$#Yia)_xWn;j;cw80)W^`rk zkWR&q++^u~6)^VyUi`-D1&tf{wjqwb$r{h=?Y5oH<@2koNn4P%#hNiN4O2 zzDss%S(wja*n|l`b^dl=LM4E6_f#{E%ZHDEEEGhRrOMr zaJOFY0?jx|C`9q0OVNJj`>0}eG9%>h86#P?a-F4zqOj1$$nH|i!Z3p=W0%OBryeJ# z*b*u-f~9=?$FqH*kOwhNFs=7Wk3YsYc7#GAXdB9ELiuo?0xh9SP!aPfy#T8Zq~kAc zcOKa~rl$Gr-6!eQ`*&P9d~SlhbNsBhHb|VR7(=nS!#qCfRMnMG3L%1{O=yXB&O-Ek z8bStNCUqg0L`yI4s(t`RFlmIHs|A zSa0@Fx2Hm~)ywlu2UJdpSLLu^0gyr3WS-LGyskyTrpd{9YoA?ujO@(A6I*BZu26UXdT-6D`x3RUFrI{V zw@fvulpd1tJLTTU`HH2sLT7FO<})>LW#o%bCDNa~2)ZJ{QIG83_10@T{1)$v`lh6+J%cQ2#@@t=@1#LsM!y?`@?qP>K2Cnlp5*Jngb3*TI2IHNf>z=9P?B~+9IOSh1O?_ut6$dEr- z^4#ngct!L5kzjwT4r_Wntr%MMIJMN~(E?+%aqOP2M};ScTXxO+zV$whsDq4I4Mn?j z9`4qZKR4>Q0}q|reKG2hi~^-Z_pP~%@4PI$h8L0#!KFijxF^iQ;rP2zlM6$gr+VDq zsM!@8u=L8SQCRA3IL}isS4%s~Voz#cIT0{&PtkEk4*W zZ*j|_4&Tpy?sinCH!0O-=88R+f@UjGqRBaTZRc!cJQ9KLiERpTk=8%ZC!H(EU*{tz zj@Cc5@2-=Z@%7FVikDCAKKH@uNwDU&3$iYX!pCsG@-CvOlWXD}ba}M}xmJld`RaKK zblZU>I3D!QVd7hICej9eWFUO*|2YEzjbTB*))G2{oWRQoZ+@1upk01O6zX}T^3ml( zvJd4R1}*9H(cW)JZ{O}Flb#y-@%*Z(_suhuV`WG1u}BMU+?bs>Bji@2$J!~7j2dw; zmUB_{O3_Ikz3@>s9ElXN{P_6v6Su9ohJ^;kH+n5Lxj&7a7arPnHQ7*<@R-Uzvke&U zl9!pgs{XTFT)0SX8ospzD|7m0peGuZYrf5qg;mkUUdD2Mf>u!$jyid3#<5l=(FYhoLWcA363e4nl{j-nn&7V9WVaW8BjRO~TaN9G1t% zY`idY>J~YZ9sWol$Lo#6XY83L9w5&L1q<2)@;XH7wCBlYqa97B*kB5}n4pj^2E zJ(xg_4Hn+A105?{TTm#*9Fh?II;riag)eGpVi)S2$W`*zd;MJF*5QP#m-8Qn@6%nr zNxWj-7U00x4aiJ3@|=By8*qn(9uzh~JzAbVZn(OoyoeQ;GYQX5fMdOP8_zt-Bhk`x zM%2MKavzr`pWY+qzbZIS6J{KCTvu?#6E{%e2WyzVkA1Jzn4o z+UUcwaXHKa)%eD!>4on0yFE5LPW=)WA+GpR0vxx}Pk@v+TD&wSr!u)4N1;uv zbakZbfXK4mk(0VZ26UnjYUj6gOk3!1qS|0&Fpla{5s&3z% zNzPxMY%XcGp5k;(_Do%^SZJGyax69aulcfCfV&}ImYjB(gz%;nb!bI-E+UYFR(vxY zDZH#8CZj!~Z`#wg1}f=ii}JjML>gw6yI!+GThP z?KpIpTxCqAvAIV~h+E#p2w$ZF>wgXkZ@TIq2@Yf+!m0HfvZ3X(hkvx-yi6`dxgP1n zT}G3q@L&znAM6h3vxGvPL^$n?p{48>$Z$tI_~;H9lS{)JI_RAM-hPc*DZ(mH7ohE0 z4FNY|on#N>-0jyDp4Amq{QnzgDn&#}8+aLQm1V0Uon@tv0l1CDYc`WaANTwcyXuG! zx{%`%+DO@D+-+!eYtIOq+oFY^u|hCtECsuKR`W**v$A9aoA8$3p1m_FA7kq^``wkG zmA*u!Xu0v>0V zt^`hb$a6CCK3>Lq*!}fujQH3!TE`5Uw%pi0?*+QSBGN*XzVyQi5L@9}DcAv2WE#Tr zFowRWUv_L92H7Rj>N~2|^ao{iAg|`DcY!a;6Z{6+ki;S|TVi*LZ7m;@Ibi55?+H7fe@m zhm_UmMTY)PF(0M~phy7fkra+{4uB!wumX@j5FY31WjsBHCL@>8)3seeyzAD&o2WZI z1k*6?+^qjFi^h8I-q(nV|&lG;jdOAx@}$-Bu+qvtUgIkNb(2ZRl`^$jaH?ZMv1rQBIzB`!BK7bjF#7Mp|3owcbV#Tvev9>_L-w78FcbS!D(Zss2QLE3 z8_*YY6YZn2MV<`B_k5qPF&kKU0aVpp%X*r;`FC7wozJbJ)v-&+gZ(U&+|I5WU5e(El(A=?SnMXDxk~!{tB>VdW!g4 z>IW2w9X_i`s;Mx-)VcYMe*O_mmRPOuJpRL^Gu_jl*0_DzR6t|K>P6D);!~SHOWZgx zd70RNN~}!#L;A!{d!M5gIkjzTdG8xa50qqEon1Y_@Ix~eKJElbwGv)Dq<3&|+4Uy$ ztaDdegw%GJVR&eSxgVqobh$JunTyp*3zvOG``0<(hQu-`PK4A6@Gu0Gli?tXS1yL$ z6}9|TyzJB%*%W+dvsL^%g*vU&D}oJ=q^~G9_4zx^z>L0PoY&0y(qg5xK2_z-A;Gu9 zHtqC1Q~BY_iVX7|_wx^nJ$4nxU*X5f8L9K2a!r;l1h~+Lz+L{dtP<(W2f?#K553XK zB2*#;jy|ei^i@9+cHFm>u3h>!CoR74`f1ziSq4{M$)1WC7e(ROT&R-~erytVnF!1i%(eo-XW~Z_Q0F%)5Ko!nbt)LoeYl z`P!7$yufsOlbQaDD^F7G-LG75zHQRDA;$QE_<7F!ag;-c)+-r?`Dh$b86D5y7}AUA zz{1pr)oJIO&#p1mKFY-klUK)oEO-5y?6f^vM%BQwyy7yf=+{tE2W>WH*GZFSJE{fDkL7Ns@H;QP`Vx1<2r+Huwc-2JL@Xk!;5vdlJ`>d{2QwDnmH! zJa!6ZFJ(v8x`XbO{o}V;{RYAtq^dhj%1;b8Y|=Yy@}2*VYYk}FA}{YAP-gX232$SZvhsRXi+(YtLO53W&G0$$uC}ng=4*n&+TsIS zjh^3Z-LQ9tfj9+-d6B@-|KO1TU2vG()##9=bZ!a|pqPVZi1Tvb!rv#rQ{6blvn>F5 z_eiuTeqp#FIi4YeNF3*;xm-264Ws#pJHH#i0-sjmO9gf*K9d?hw^-SGOskYWkT>~G z$yW*ou&X4!36z^8L|`s4d=G*aV>^KmdZjD4|I_IJrd_xT=N*QI{A$@MhZyzR{JVG% zugxOuN=a#vx#nBrGkTvjUnv}oOietu{$hqy-V2Ivs|Wy$u!xRiwr%khhtahxVf0=*uN$l+(aW? z2-S7qi3x|KX#c}Zp*CwEVzML!o&PJi3LPG)I5fScwm>s%&n~5A{kp|XWlvmU-Jic8 znMWtL4reZX;Bp{k?sg5Q*-}!vcp6*_ZFKLG>gM7gq~`_ucZ$3P?S+k^@BvyVk3HSo z$=jLF9Bc+OD&oMsDNM%NANs5#31_&AKwq)Gl3tGyOUwk|?5NxXJgj&o(C-0>wG6vK zmkj-B%q!S0kTR)-axQ>}$AjIi4;ZOSm~BcEZ1%Ol1Dk~2tGLZuXTLjQzh!o>+V!JB zN20`h^dgz7$PW(j6x1B!9{ztcfe#pB!Wy!U-oMdw zCXheQ0oWQ1X~!q{rC`860it&v2ryt_ZVn;Tvdf7{9-IY8zd9lR1HWd~DA?y6|j2Niv2?E@~~FPH4Z@BPTy zW<_IrY4z(0m`~ceDqE0x?-zU})w|a?+4F5yHCMg87rmaLvR<$EQ(HEw1C@MAU9b$x z@4P?<$?6eGZ8i;`6Ee~-5(|2S_D^T=_P#2v!YUJhU{BpmV20-H@RdwN^XBDD@KM{s z#m?p!=K3y7`11V51+$xRgS5FZ1S5 zUi<~@OA@t^RKpj~t?lX0HduT*^LePIU9tSCQBu>3 zOw=9Pp@{K?K$(x*HkS(^#|iL^oizXc3RDsO@odZ=&jvB)9l~>1dnC>xf!q&>!o`I5 zFHmP(!6NkQPLShUg_jU_v|cxlo)Q&As`^B&d271wL4zxQ&_Uiw`Bp?nq5D_Gb!!*d%qGi&*J`=e3j(-P6N}XjhHJ3WxxnMimEF9mn zI-r>&udsH@O8d?EpDxG%(abdKjMZ7r2ecNr z1oc4bsyUYD%uVr-X3t>OL*L1F8a?D2^2Sv}Z0c%u+dSbh#ZWRAir<1GBcmz5HQ(6G zaNMULHqg^0QRvrzj)B&lkV6?NUtkZ%6IF#q)(Zt+;>mh z?%RxAE`3q2b@Q0}rn^p^iUw{sBbdQ^bf>XJLX%}^uQYfRcsNBoa-(PB0WJ6I-}|)dgg6k-$Q| zE*60cJF^`5G6!ax>nc8^BD`izOGOKS(PjAaxW7J+qv%iFNE!)n!Lis_GA2zI-qPARxUxQ(0H_e%Cw;4HJQ=ciGsX7M4J1 zKwmgL0?QW%*W-nxTHz6x!ct>$fUZDjA8ug>Z=J$5JNIC0ecAQHyVV&HiM10_w_g{T z3op={)7uD@)q>qyKCV7!KHJCcScKuumziI^g%Cy3?h7^W5SZ{>2o+SY!^blJI5YHX z&Me$iIE49fa}OW-4-qNN{}!f{N&&D$eJ{jJd^WvxcXH=|Mx zNiClxqY-FYJ5-88;5k|NWB{A^1*nk~vu2+?>TUGpu58vRtt%8F`v8~+3EYmq(4d{e z)ga~9A+N1=IY`R>XmBPihMr}mv<9_hmk;01(e>F-_4;hxM%k>3wAU6T1I?K2AvSJ$ zl)nUDlCXq(eFN^?9CjU(Es9=VZE1YZdfF3v@#mxJ#=Ji5f2`IPKh20mSx?K?lu62K<==+|8k7nt}W#6gwN`Y6R(7h4yg_S9 zfCF${2}`QBd*BZNaJJLvNbpfdJnWCl&-xbRr!WY~)>nMlfa#_vL(08Ec{i_f9 z#4LtlU-7WOmYjtp(?VhllA)1;nOgDlL$>2`Qg%yrdA$mFnG$;->AZm$z{(?wH~P<8 z4P>BpYivduNEj?jlRfTDFK#OWlhBgC9M|S~7BWW~Q?V$QU@9BCb&?HQRH%)DhjlXlxN4J2# zXvK>+8QT?-9SfhN#O^;50SPWAZU5qw1AM+|D|i|&`VEp9*2IG&KX(eTliKh{M-bgn zrC+rx&?!#beg!e_=Whq358rR6{N!#H|GoB~rVazR_kZUiR<^~b40aE36G)l+g5$2g zQ-JAkpNW0VAv44VOO6<)xAR26p#9#Z5|6_Cj}tY*3}`|Lkaf+(VnVZV0jP!(`l2r2 zT?V0~Db{QRJ#3VOqjSE$VGS_C3Je&pqg?&-2v-AOU4*N+VBBqtp^t}akuonDa{u^c)(~esQ-yGD z(EF$2EzqjI3mcl@k62*8c2j?}Bqjmvj;AmT;oO35kSrCl1eNif*_>p!l_y_kq@;Rj z{;R#1%d_))K2*N$F1ar*`wj~V>FRg#Q_D^DzrEaf^ZnQ<>N7uMcVlgjQyjln?I1eP5dClIu^*0-W6T65criam z7(z}bU$d!?lJ=&tR9sWs-g!4(aXNHm>0AgY=lOJvLzxUh#H=C-I^tGOF5J(FCq~k~ zMUF)YxUZKod?mpOzS{srK<)rk0cs#rT$@U$UKSiCi=iA*slc}0r5PFoYn`}_8*UD> z&k88>cp78XA~8|fNAall2N2vAHT&L*4XuOkDQF>O%tX1c_rK-BvIeKZp$sF-w$2qk z!ghF`VW-gj=OjH$V=8sXZF02r*<=y1c&60SQ&;sTL)+LcSV^c(zJvCg;h^eA>7zrF z{1L_&-QnB9IE+(;HB|F=;F2xWdbOe)L+;7ONY0yCZ4Q~i?@pwcUUa;5@!i~aU*gZG z?>a><6`!GC0wuBxz;OTCYVZM}k8JBXpQ9waitNRL>8TFkNnhvmGbHiQVu!4N3USj5 z^(Bu=m3Jt4iJ!YjZ$ALdzEE6yJo;%Rub33C8_AH~e#oJhC0Go@H=vJ^?&-Ypd?tmM z%wa3nf61BiHnysJ&$EDvr4b+Iu5cUkVbU0xHi}Sp`0iS3BQY-eD=+$>6fD&pv}8dJ z_)y+6V23)?(@2^T#eQohSDM2<1G#MBjqaR3 z7}>EL4c%cDgB%OUDInZh^<~atuDP=K?-ZI6z@YATu#}5*1=L?GG6N52Ajg;hv}Oa) z;!Yg|Aq-%HdmDQUhvpaW{bG!78}(Xb=}Vz#o<+`hLZ3%{fll;k$cudM9u2SmZ-f00>)Otv-tsSl#k&?-qX8`=}#FhW?lC+WRsV`6CAX7C0lIfF{H^PsetkV z4mOle&&rYH746)!EbN4Nq8HpH03*)69xa^OFotjmVAcMZRj@i{4nS&(4Aj8CwdfsD zfK>9DXk1%H*diNteztU;xlyQ~up%dN-?Yb@&s2HT%+*?`G%aW>=c(9&_Hlk{Als$3 zqj@WhRn+XlinorIU1&h<4{Hp6`8y?wew;$u=1B8+3DBbB&T*8yR7~$NzUzIrfjPW~ z`O;zNI`BohV1$BN)0`)6JO^i9dcfx=+}D*~YUjN$Yr5E!yxla^zk#wTal$iAm-~u> za3w+mWQIiBvWSf+f*Qb*(ZoBfd_#@P?P^F_cL-8kY`x!s&&Bqz(K6(-U3i!vvG9Pr8{sH*~7w7+t(9(cBiR4Thbu=Op*X z9yzn#4rJPG)Gd9MVDWM3%T|AL+(_&ew~w@hmMQs@*u%yi!q0v*^kvX3xG3fzyTB_R zkPviu!6CYQNBx_ZdXsOyHO$x=zw@ zC0$RKfXdi)jQ*Hw3Sz=b_Vi-tns5=GwyB5X2@UVfz`mMR3C3fbs1b1cPq%+-%#;Nc z`V8UXS74yP%z@j?lyK+4<+=B*cF$qh`UC9iXNA7pG+JG?qH{Yt2#g_GD^du&)k6#P4-gb`sDwBjH|VUR7Hm#uJK_Hj;x@Yn(KRIeZ% zN;iYiHei9YRrERDnZFf!vg`Sl^mRn7bF_MSFuv*FIfn%cru)Q>{lPmmJy3WY%|d`I zxUK71c~=7m`I9X80%y^)t`%rpTOJEHbs}ZjxP46~!CBG|e6))qoL%d8CtWI6pR`+r zbJjjI^1yf*6_ru`NSeDY4oBi|U?J)|jyb|L`3c92{S(Lh3`5+|5gbGnj{X5aL;z{3 z=YUPWv8Zk@y@!aYeX(6sK#%}#H7dP*3y5+$uB{$esv*fMep5`Gr~_G-9!=Qs*o5jW z!6Cmo(x?XUKti_8<%XXRU2DrNuhH>3PVQ-1+acfM5bI{Er3X ze{uwj$bo(&Y*Dpe9u@S95?od2g^M zM+y0(P4B`4lD^xy389C<~$A$znaIoa+OOq9(OnOFYm8na-lRN*wN6F1q zYH@N?Un(RAQ~Cm2#T)+c_kH*K{c~N(|8Xi{kWK%J=b;=allfSw2*qbQ5b~gTgi)k~ zv)emA&8&Au4gHZDE;pt3c28$(&OSTk<9VhYO2v3mA%r5045mt;ocy*CtvNJB;{2su zwF8;pas>dNw^rJAf{zy5K_waywp#Qenzxopyw;G0H-pL{-yNFYEm?hq)p?ZEtN$rU1KJo2RpXG|xcU#EPV(h#_L{+-$KT=d$ zb{r66@n67H|sybWnt8zbDuS2{_BHZOEKk(f!nJbrFM zG%3j-RABCs)~2)xK7d4}l%Ocmg7#{>RR^%9Cb4lyARcpJkxyVA4WF9<_LC#5l9Xs; z!T*xKgoA$HJ&NCM$d-O+OHA_QECh#oB)Az}Sp47Y0W~8iK3$1W9fGfs6hbaSEE^(? z?)0!l#izgensJ0H9`)>aoE5V=cxA9vZn)}4g&P@P3Pfl-C|3FhtIBEdcjv}9>WF1J zC?`M?IhFbJM<@R+75IKTTuPX))+{$(*SO+&f`_kW+`HkX&vP4IPCo6v@9m^qj-0MgEviSAg2Q1ChYaOr94_7t^K{Uu2WxTP zTEQc-9+4*O1xx&1A##3F_$;gKHH33hwxxA)L~Ce%%EyK*t6ke<#;62aNyCCJpyGjW z+9HmTY%ZA789~J(cM>QQ) zR;UZRp52sQtgq^kGd{|S%Q(~D?a0$(eO28PT-7w)OZED0$unEUv;Zhro;c*%ys074 zpURP>*!nM8yKQZyg`~1{l7gKyV2Sht;(SQ^U<@l7dkUw4qn7L;&0Bw`oWwx&0*0T* zZQFul&~BC8L=WDoGeDc@dbIv<%9>S0)1RGcS^89?req*S!po zKVu6)8Z2!C%YX%5y_Fr-+DSh)Z`$va*WUyV+N%dWt^id@T7tnia=76GqSC`%=$>WZ zaH4Su^z9$KycfUm^7iTDi(w)BV25pd#d+TOjSJr$?vc!n34c&WUo#cUn6oqe=ClK` z4HWA!2V~T1hh1Z5opQQ2(oDm(ylbchzVOdx8XjZOwEb9TDsy-uYw$MytZCn}V%nIj z)tu~Uw?DcMJGWk%vFx;*t!z>J8ENs^hQ$D=4OjQzhCAUf96e^7j|b*athFAO-SWO6 z4-=e|Dn=yx%4?HKCa2LZW55FO6Ke%;-2Bl!4L|tL&}&Ema0nEWV~#aJ`vvORv~g&j z&Az)|=BKYc{dSJ5Lhyzox$_IOWs9@Xy91Z>JzW&!PJUXkj$sGRb#GoeX;mDCb;+V| z0HF=Jq5J8ZE1~r68@%uF;L$9yc`gNSPUP3SZFu&!zw?1rScHOUrNPV{1{SYAW3PjI z$g%I>X59YZyxPEn94+?os*ByTGMREh0F>N@5i@m%+27>oJwVuVzv&W?N7g?S4EkL@ ze^U;c2fSCdRE-EJxFC$bXoGM1>W|wxVqQn61B)t|jx~>?e%Az4!I>Yr2P{?4ZZz>9 z3IoCui~hv}9gb^=;d&ebd`9vBHGyrHs00F4L+C^swllF|>l+rXwQKF2r^d49nyj=* zXAhyVxth=M!F)RP5KY#7g-y}51jqVS>n`h$)m7VPX7zGG?D$&Q)litmOcWDd1C_Cb z3GNAFax7a7`IPLFioX3`|2D|X33@d9Zn|yuR0GC+NPd`#gQX$mgXY*1Lr0#o}tWN7&g8R9|TjsS*# z(YHT@>G&8;?h=t~`hgd~o%`u0j6tP;#~5%feqam)js@~fZ3AZ5WZmZgeF4kRZMH)O zoCDO$3W@EeN!!&NS8^ z-~ROY$?Dt}x81g#wJ(`Rx%_eg=cJB!qlljCH@!KO4yY+SeNdNJ4hQC;G&z3*aHSWI zgJS`9VEX2=o_I(_>@I*!x(U+45_2uDhgF5T1#yoUYS>ENe(c!&y7-#@4ub-(DTn2B z(zhv#zmz!j1-8rQ6^ma7w*>J=I8ihT)@4-*x8a&DF7X3nFmWr>@!vs48hMWP*fO_V z$?r8FVgRspeZ^G`Z<2gW3XZ7Sbk@)FQ9ZFKU}5;xnX{}~H0`8ojsCx?sQV{bc7OD; z_+S0=XQ%%EP5bYvf6&AKzj{GMJ=H~j{;7qn{_nzmThM=cVgK{uTF^CxSJ7HGtbeD2 z?HVV4!2SL|Ga>$O|NGzk180Ew=Uepep7^Xkf_#dnc9Th?`)MyX5@wSBh7HK=_9jnqAbR`; zstE7(rIim2211gK$?p`^B*&Rs#QIX2(Yo--ZgP1ahT5J4f;32jZ`x?Hc^>>W#8C*8Oi7S;} zu-*OLz5DiImttP&rng^w6Ix_{%jO-+SUIV|ZTgqSkUN$1$q;a~qrZPpxF~@?|1wW^ z`HAp`1ARjtk>rO{4!k!fcgK4;nrCjG^x(Ghieny&dO}^dSgYOt`tX^=iNLKf-UY|b zJ5N_Hpf0lME?0r709?-UTBO%}r>hCGv58~ud%n^s?cG~_JMw^0T5}A?-fGkMae5Sm zwYyXuzgJXVcs$uv%=ON~4yX}dyURr(m11vwab;^K6a-4NL|f$c-_G-Ljhij)Xw$jo zwZpP|anr;itHKN7r-ZFAIX+yWV!OO zec`cqG{te|txNr8ZmV^^z&GCgIx6Zw2fV0L?*7~POO#Z5tId`Y6{9MbEckq3!g0G% zl`z*-DJh6D!z5zim0ZV}2GT~m#-~zJf13YK-}fV#`rrKg8<+z6QSjkEl0#=vO8(^3 z-u5RUjQ<>9fQbc)?Qb+w-%VijA5EV6pO4yaeWCH_@#vH12K$6BC&nQwB$%yhW_^9R z_J~aiIIqXJfG={YZ5`d(HV7HrsK!wLGSAa(nsY}&Zr!Mr^ygJi1ZR9&rfC=wul*_! zD(bE_c_q8qrCV|5gEgdP_#94+zIBXBmHo5%zs29N56mBx(XQa2D6sIgeV1cEc<@XL zu?eeOm>b_}Uz)VAPNP@LzFN$VGLK#ZY$hQsj8OAA^b65oOg_e@Nn?TEaj$i<30TI*0DA|csg}$#5rMm-}E)Mt}pIB z+uM`Px=&iwl)1D>%kAj>aAeM*nRzBl=Y?G|x_te{#-y}8ANzCoj*zLLyf2fIXZD;& zck(M`K4lc6by|g$9thRgdU{p8g{M(iu)fOyX5kq1u7isfZDXum!0Iq*u_feW`3T;v zP(N6Zd4eVJ!oa`n#B}4^z{im&^%K=UB4G=H-BlI@pkq=w)Fa& zN!^}F2g5(>%$D0k|GZMSyyP|3?a=8vx&JH&nK<)dpo-f1#H20{%gt`+;Ri3$9@rSq zc}}~+3-RCLl`Btv$S>*d=Qzf z6?=baFKvs%z*dVEvv4#Lu|j0G=Mn49ylS_8etgxqp3>=un9()9i~)K(IFWoaGJ|#Y z)NDxfP>D}BI&`G>IC720>Rv5y!7cfYID;DkNsdMyZ`Pt=K3#Cn_Laq<11JAn+mr3J3xs9Sn$oAclmFbg2qRuZj?*N-?1q5$PQ%0hJa> zKtmwqo$YttGuJognwc}_nsc2WGw+|YXXiW^pJZcYW=O(tK;1x1@!J1>yb4NmHaM1b>Td zR*AR_qIO=W+0QT}F=OJF3jx{VS%mM@jjRBH7KZMzd3#fqkqqdD8w2tI&;1h+1r5|X zssr;5(#7h(vJ?ArXP(Ujzu&tgxWRMzI|!Mz0w7y_ywi~+)UZDsru}$%DPDwST=yJYd!lGc zSww&_a2jQ6ViQ;G)uNjp&>IwidAil28N zYz3z(Rg|Nnx>Ay*!LgvId6DH!Dfb!n2$>1!^0Ph&-ef1N6}fIShWwf5tznrWq50#^ z`Ge-$hdTNyb`L&FJq(2of432|VQe3|8(eTFwkux+j+7d;ycC>!z+GbW(j(&c5OGUOo{gcJDAF+cr zYmVut7e;GLm11k@=S|B30wHx36s{}FW_^$uJ3T6f;;2SBBf>ZZqhCW+4+#Jj-sO48 zoNEJmEq@$|oQ!_xh$s1y#D^V84*47)rv2+rI;P*EK9NuUq;pM)%a;KNV+cvG$5Dua z$k!p6Vv%eEzMlA1>(TpeRM`1V|{Qe}OyP(djM{6k($z$28nlqGDLQN}Q>a?94YMA`?rBg=hMYVuT zmT2bg?Y+oEAQ_P|8vEh58ueoRot#q(a=niLB| zYQ(mQ!m+f6k5Q?Z z*?F(Hz4IJrR`zottNOQZ{l7l3{dP-c!19~jV&9HD$k_t}C`rvndZD64prk4?)Rq(p zyq{>_@A_4qY67NH*{?%i3qES*d1t%H==058FWPEO?KC4lN}GV9a2F7L@E8hb>mT6C zQ`Scow?z1+N@`7n&o3CXygZEl+Buh}E(*=MQ2ackKN`;h`+UNi0Fq2z0X;e<6N3Dz zFYfUQ>J;1o-xnDer|r$*l|#!cZY=Jzy;kI}Qq}<2wzDEL%C7eNv0AYq2W~*DGo9?j zHopZB>Ihdm+Q_2^o&s{uDe~TS>uuuq8@sv{H`8s?&leCo(+~uf62*f{FBuHQ<~l9E z`T2f|eVUhU{GCWG@*3aLq|d4ga#t--`O;>oQ`Wnrc_Q@+U+K)J){(sGh8HeE92(l0 z!#${It00csi@~`@F*YwsN*3GSys4nSCjU%tlL6a|oL%Zb@*qUuA|n)D!g#M}A5FS8 zE(?f;+421x4gNZtNgLqoPlZ|V9!d7g|AmJn0$4p7f;ZZE>Tq$gEp$~rvL%#=s&Ozw zhCZBbNvoe$Q{;Dj7S+PUti^FT>_jkM4c|k5T{qm?qk#cA_CqZ|J55+Q4FVWdAi&qY z1q#bF20)+c9wn%(j8jyqJr9#QU-@%af3nma?SNH&dZ0ATeegS+KmlmI2*?A7^Td!K zuaUdG4b6e zc*|OStf}P-m0cmvWD~=QOA#UscVXDHd8X{_ z3x=dE;UCE3jEaI!E?Y+dy0G^LW3qv7a>(lqf!;D_Xa7*~f|x?rQ2XNKS>d&OkB@eJ z91)+ze8M=O8ZLQ2!933PmO(Z!HHyfBHbOIvSJXYn*(bwHS8XltIsJfy}H@N zu2(wq>QoxNqk?&Xr+kxFu*P-9vQ~wv_^;`hE5rmVFui);3MY+T;0TJ-`dQ41W0vT;5*vGCnr38tR2}#D1rkXF)yiIz7ZBfA2E><;JsZt&&rl zN?v;5+EdFMf^xRlw{N5D%-ruu-V;BcZdbBymIRcPf&VSOzkNWkyhD;AL}D{*Qm{zY zg@*hR7h+JYGUplkjQR(PFI^roav#V1qqT$($c;m;JRw>3sx6A+KW-hSfKe3aq zwR)tP0pE3hO#0x8*2nE({$Xx>1%(?)Or~aP1B8uUmYPjtrj0m=j=~jIb{Oq9A?$5> z{*_L*%Im^$0l_Y3ZL-of=5C&>jILUOE*FEljDjPcU~ekmqK$!U$^yQbJ6^Vp@5+OH z^``1Qic>`=exPtGFQiTQ4N}i|s=Me^sl_IhcziG>?$}#JQw?dSM1L;{9xVZfjn!|0 zYo?}VS<;5oq8hrB*bPjSR9-L@J_KqI+ZA0%2+1GY6sGJLhdz~N6p(WIwqE6J?RWJT zM(&(RQyhKI5><8dR+#7qgeEYXAh>BLexJrj4?=ycUTR0}-FtXi45OoFj#Ly_erRlb zFY0kjp8dr#TRAvA|Eu~E@)vKgz=aE%N!^AY;DO~B!a$p`8pS%!&lf3HlHG+lz zk9(e?Df^x+27_#w*N>>5tR=AmJu+y$L5U@lqxc6t0oajS3~Uqf5b zEZ2Mb#>Sg0ZWTCSC&%YVp5U>8`mCIk`Wi7&AXJY$Lekm}2`x`4^s=I##sx|FvhZZ# zU;;0dTN`8eZgdCqJOaF}=edps4`0`PI0*O!eVx}XB!Gv+L+Z`CAhBqn=?3OdccY1h zYM!ZYw?=y_JbnGO@uC+SsXxOEMc>Y-ElX)ne*#3U{cSVAX3i&@6Bg?TC0O2W65Oa_ zUe?M%^+Uvl?1MkhE>c|VT$z#g)5T*>$l%pz%=BbwFwm-KDca>E@i$cwq7~Yz4d)&#e2KJ4-<^+9Xx8>Zm=R7_0LIs&ASU@WmZK)&WDUt zkkGPR0JRaVo^Vb zu7U8O`4_fqr=yka9|DKO4o{5=$s^|a`UyS1F@HS~Z%jDWY;iZ~>pMZalPBY8-fjHk zv&ar+T{+t9q1v`HSryKN|K6Q@Yve*8ij$OUI9c92{!=|QO4ZLyEa=p8&M*D^j||Va z9P>NQnNmSJwjN>@3tg^-Hkif!l(bs>+_mB!frG8y!KxR>9Iyj+<1IRt{JgE(BayJ) zL+@7#)iN>#Aa~&{1{bu#HeW|{#4q8DmP60)T^*xbn;vA%BqY^{l3BNjW^LV{3Y6ME zy@v)Q?&Z8%h0e=#EHVRWb#5}-;s$TJH$oNvqXMf1(-(l&cAd%=^BdrbxBImC?b6L? zQM$G1Z_wXd!fT{l6u3HV;SxCSWOV==ul;S(YhghPbQAejL)qSz?X!r-V8zUO*LRj> zfO6>`MgbuX=LE*z38L%tktEmUYRA>sNO^>W9)8~eJs#LkQTjSFyJ7HSQReCS`UUua z+p{7grCmD15Yd=~mnBbZvaP)oLP1*Wr-7pxZ^<(%6fd~v?@i0s44k$@5qaO2krGOZ z021sZz10uQomskP@ujSV{pVl~omw$*)!D9&3icV4**W8v5&A)ncyo~0>}x@drX7Ln zP>Xa=Qy!3-pS*D(t#%%|5D%LYf0?vu$nAU=38_DS!dNk08_p(t#{>xPt8iqrHUfSAJvgFSPUz;g#)!v zrzxC$#GJ`eY6Qj`5|gi4U_yv}DvB@V^rG>jYsX6`7X7(6Y`rds-nzSmmMRXNM>zHB z2V*Ea5-S)A2Wt6xLa`lOp9Cu;kyi`i0_GiANirr=fih1^bF}oyIv;vV0_7$=Su-oO z%HQX8z`0pIU01WYIhUCAO{wP1+&ZF=_6do%Kt(x1XcsBABnZ~E9*=0nSJ)<)K6CJT zG}iU%w!Qv$TQ8p43q1;dyxd+ZzvP^Ma*>G8#k*A(*^^?s4wKMT-uIICbfOXhlj3h@ zFqm-62uZFHxhdZ3$ami514x zSoK#lTcxgfOoYFeT02FrZOUXGlwwginQmL~TXOs);9uxe|Ju-&^5adhycsjIq?3v_ ze5&6$7g_(=N!70VQUpsB?06s+ok_|BGxi!amj*%fittl0vdfddFfk6|ZEvHzc(Po@ zY8x=ks>O#tm?ODT=X=g{$I{0baDBt%^}Yp0Tp0`!x%3GM<`7NsTy}8Jkv8UKn73hD zJ#8qq%Y5qu*`VcKf-ihpsiW5AneqIJ#M!>H&sD}P8Lvy3j1x4R;$EWmC~WH{tM0fS zU{}k-I5s?zct6q@O*sF7ko;t{0_nz=ExY~Zj9nb+2fWYhr>hmeRC5ZxN`!4*FERjy z1;juqnTxixRL_P}BAmqPMw5-Ntl*Huz{wgK+lIB-iizDsy8d|C<;ANO_OIvp+sasw zPtKoUNmMGpDfXO?pHr`K9Li+BwZym3{%{7D{#ho4)A*v7)WSxQg@x&WP4hpThJVc+ z{XcpB9{}tBn~Ht^nd#gAJ4Eq#-%w6FK_?@pAU2Ry7lP5Le}|fU7EakJEUtOE;8NPc zZoy>k7izW_H~q6UkIex+>ieve7{qaDv+EnHV5Y`?KdKG74D!tyA#oTtOdPw1W>bPZ zMQ3@);m)R`b7RboOPG`R?svoX?XM@cL#9Uy@YZZI&+SR#gnRp|!%?z{`5Xk(jvTi{ z4%r9sqz9WY#fDnQ+0ir?NXuM7u`-?+pZdXg#;#d! zUgaa>(H#qeNvGbEAK)G2@$q>?t5P&HqrZ0nN{y!&=tyEC;0QBjE6mRx%ou#f6bA&C`i zTUH@}>&D~;sL-dTw`tM@uRgJ7O-l`Rbjs#xC22wG!l#Z9sUhh8j(AE#n<4>u@zE@3#TuewZPRlgpZTZ%azP+99UEW9d z?H^piOHjDBR!`z?k<~q*^_aLe75r9L$Z(HQ0EV$Cyt{7e5i8OASx(|KZzVp?VdAu? z{l}VvGq9r=?>2AX)%pU7JaLK&-o4BB>Eao}kJtH3$p?2QkSo#?xSg7>6XkXZuTbR? zHbdP>=f0}>38YnW2cxnu-Z**UYwBCfaxwBKO@!pXJp2@7i4RAhq0hT>o+mYEyopH< z^<+(aXK+7FBPU3)KKGJGhvkH`Sg1YO3N8#5#*W~=Ref43;AK1E%_1qRVyau7Wm_T4 zON)kzz4Tpet1)cnZyP%GAhjkc4@9|e7%WoXAjcxziwu;hW@6)e+$i3cX&ZMt+V6D1cYq)pjq$fZB$w)_G+96E;FpK`t*Y&0GQr=y9U7@Al3>5M&&K6~W9H(jp5ukoblvD2a+#H@A4#fP68( zIRR)S6cvyg8W{?8veSC8b8cE!j3;_>T32O?;o~y{!-D=_!dvS*B*~jlym{3Q>pXy? z&-kA3v}HLoC|o=Xh3g zar`|aG#f<{Fe9rldmz!;JtbXG4UdDJ(?G{!mw&H3%J zW(yjWls_8HeN;z*>%u}(W@Pv^pLd&481yk>?oT`IFKO0f%(Fga_}A%Q$R8{vR9G$ zXql0Ldo^qv#bx2wyoaK2auQ*rK(Yj~1DJ{@6nuxtb@KdZ{SJNbJr76gi3=)PwbRT( zDD=0DMJvGuQmm#4!~>PsKA`o_cCUIO-ClM@hA300(z z=acuWTAw5&yeK)}`6Z3#TJ{YEw~mDdawzDS8EP8XvI~(6;IX^_REwZ3@CqP7m#s+Q z(Q!_m*gTOrEn&?=6Z^sr?wUPmW%i_+&xJ}y^=h1njnp`V7C?AG$GAWF4UnL^a7=UZ zr<#&{e=#;JH2?%kFR+F|JjZ(jH?xpk0mF)1JGlHHo^C4d7EWce8k3r_p=*{01hKj? z?$Bfi-c&vk6tF{UAwELtA?@=^v%Z}?&pbJU(bMLW;;m}1H(zB5DpAv%qMCgj210=y zsNQB-FlpVuQ0JC1VhNznlxW;z>7mrJ8f5<BMBUAP_fLz{5nC7Q$tTm=RTDiqsW=Y=^7*3A2dwzy@9_vOB)gJ=cUj3H-} ze`A4TDFK{$ORziR5M~QcA=fXWdj9oL~kS6g427N z8f!`EH>Tua_P)N3X5nSt1|K*~#cKAn(!K@{a330yy@N|=Opaxwgf27>HLOiy8D~6h z5%IMlDX3?eb=(Y8}jxk z=)ZHUx{kBR?jcY-=(M3Zfe|j-2f(50U#^g?5!yu^imS(jL9Fr#jp6xgMBjAIg>)ER zo)|h!<0I8V={E}IXc7|??K%7M8r4hUryY3Ctq$exuhRKA=3_JflN`htB339M{vUDK z0{{sss7^=vbu{19?n)SN>~igW78&*Ezc*+kYfRU4wPP{BAnvvnkYpUgceJ_j zmc)G0<7HC_h`1@+1iWgnhgiRFoiKXWU;~@EOuFpiB1l#Av$uI2*J%{=!|C$NjPcM` z$P25bcl3Zwi&WTcW z_v&-4T)$z`*rG{T>sp+;#iL!i%FMw2+>lws{mK!aM<%9b1CKx%;^i!8qdQ>lg#}@n z)KU;<7B@rk+FBeEpmEO4ic6g)mqrbUFZOP%z1sHvWkzFw5RPF}x(jA5z=E*f<9?a( zd0C+q395M~D&E{tnOw+f)Qrj+iA`<6*-&E(QSzi|LOEe-B)|C6nB+i_}OSTvzK9cdJY6WhG)I8Y}+ufWK;1$9mvF9W;32X zYFX75`f7sM14pmvx~0z%H;gYv4b?pmM_k9&R%>-Et=Ti!S{rUeb5jm~!$LZQ1fywP z9Y-$=cLOSMiTMY`{0KfaUL{qX*Po#jcH=g4oU2=7gnKfZqR!iXLfuum49FG-3Rj?Fs-nUUzE9i9Fe2Xob|&zEoc(3B(dZ1tlJ zebws>skEMOjh;W8gYK2dcp-ujpR5}Hq^q5#@b0Wuw6{7D?FCmWqUwjA3cxQp37Wr6 zee>(Y$=24o{;nFqu~IOqJ$gZ8bJ{edb_Y;U?b23BPxA8$V=I~YI9HbM|;2VcvgL4-2p_$%(82cIehqvZ7LcgXc+)uFk`B9B4 zM>6!6Um&1>3)k`O6JQ@2D^~9C@g)CSCTa8j{tg^;uE92Yx`lOaU~ohz~U2 zjd8YJA#2bE&?h37%_8f`-^X`o;-su6PX@^^ZibC}=7UeoB#tmG+~3W=j%G(XCmVQZMHeHn~S+ zzG?5u6irw>iUD3+*xe%69S=VU-e(461?$13_Y1nAhsX(;bdsJQq)(ot&WXh&*AHk9 zvPC+wDyJ~i$;%L_4>ua5#vjnc;?={2pZ48Z1=BXd8?Heuh290EE@-T_5*Dbm*n2}c z+N4UhG+&dhkJ;pGdbbXeTi0bHcXszPv$^pUq44ycV7Be*mEk#1?Bxkrku46POwzzQ zCSmw>)we=o6STeHl2cd6Q1kbp$Bu` zbzG(RLbqi=SBp>j?G6$K&7MLWCAtN?fMBxYI}F}o=inkL21&Hg262RvA-F#3x2uEn z5w@l=R1x;{3C|-FDBh&6tMA*qXaeuD_)F_=H+;|x3||U<+IF!+Yz*u)K`CttR!|@0 zOcRYconW~PHUPK^!LA3*ManJoL@VHb6!{f_0K2L{l)~I~z6Is+T8ZaA&3kkj_wlTO z8a*E`Y#APk<{8|nwhKJ0JDhO4`=Eic2fstedrsj7#9_~sl-Y`vh8vg0@IzhrhthW#Fiamwn_k&D> z9&0QGRW1%j>U>@J;F4ptmfo2ty>gRjZ$sty*Xg1!%HIw`0-+LURqwYTz>8U~97T-V zzD|&jD5xK|^~jfQfyjXhEo6O{x0u{o{ZMuV;|01+`xX%AyUfmh?uX_3(+oO#C)zBD z3Di6^2VxS8RKrdfMEGa$FAPg?^31&b+(3u$*tLa+X}!zZK5bQW-h0o1(h?q^b`oUi z;3W%cGSZV#4t)+Th5wO%9`D;N`rzFr(%q1~r_9SL@sG42F88l5Tws6j+z)$g_0nhm zN`zl#i%1aoA<&&Ej z?jFj*=eROt<0`$LYo#gHUVPMe?1|K7^H9i13ePd3G_?{6b{ZzQIG$~LxY{Pz3du@Z zobR@ceR|>ZMtyy4(~B#9FyDv^VjMo3(0sY= zW3D3aW7PUdq7F5u9yRlX$ZHm}R4<*1>$Uf+&-7Td0-oAtnYc}1%PhY zIX#%voS!H_8SfxFj?y?bp{%}bPFN7+Ot+yH*Hk1}#bUn$+F^ zVyHNxTq(kcE~O6Y&c*-_1jxiaQ}<_W4< zI)q0AJ6*5cH4ZDKvwmIS4c!{-zgdV~j3rc%k5H7T>1b8TWl}401v-;pN5WK?*66?T zP@NK}-N1V+3brL-YMpPjw8dMinT!5mpWxFkN*7P9DIjKmP77c7 z%R@4^res0f*gNma4UYMtl20%nKuuL1+y00;%a=MV*P*gyE8h<8)})#`%t%sdeJD;u zr!E8DAa^;`*$khQ`I4qA5Z#dR&NIh$zSP#8_~?ku#Q{mb-DibCs4C=g0<;dIW6}q4|5|sZ)3GK?(e|T^l!DHJ+Efbe2Aw9_tPcBADF3(K7Ga0Ox1k()1w}@ zH3}JAqHwNSga1tz48>0Ovhg8C9h6KjEE$lJf`XR$LFm-ZTs8W5zH3JJCAOERMvPwa z3zk?Po*-Fwx5y{9={RGTSQe@m^IqHi&Y6wiv2nXWc+Ya=NJfOBl4D!}-q?3G))El? zk#pr_>~Of{^}1>Fg)Q5dsKcZvrwakI!fmgr6JDJ-Ayl&{volPR^o^wHlIn=(NU_UH zvzLuvaR0!PHcjK%kBv>NF^;N&o^iKgJkM|#GOrzfWSJWOz4vC4QI*<0$RW|ZIPsNX z8-vOwwrJ`;I_=ldF|2$@^T90VBN??JMYK5wA%f$!4!*UwfRQGAJJ^agjg>`6H=*A( zO{xTWK+(6?A7-DR&t>Hr{jUn# z|K9U|l0x@?C2{kVV>gWrzn>JqjA0_7KeC#bViJ4&U=PAqQw3kw<|_aZVhCnA3c~_- zd_9B$BzrM5j}k|p*@r7Z9;L*jec4KMH(HB{x>t<3iYNCZpIz}9t4_UFW|AA&=DNij zK!qWd_Rvs#Ni3;2z9es@#PYd&eFMoP;Pc zcSQC5~NoF0>ar>0C`T`HDW@^xN*yrJv z*4og#nilRU4#?x4>xN}=+!PXm_N@I4o2`8_-}yZKW1K>uu95##R=lj%?`aUF(Z?0V zp}uv!I81al-QaWZE6K9E0`0Zf7wqqnrER5>sSJ&rhM9Zr+95aVE&tHd=d$>%4teD0 zYEEL2&f(4a8(OJR>VrZ#!9sUIG5MIx{yr~0w>*6w;r`CM-{S5i2j?(mXsSzp>^c)U z!M-4n7B9%2SGHk`NOC@vUD_dBUERWfjp1VNeX1#49L9C3-OpBMy*TWev|?16m$4)> z%go6uMlru$Nfx(XynxtOj^=CBdnH+T{O+~(a(dAs@RH%9f;uidBd8pOGdy6E%mzoKjJk>g_ckB|fZ^K!@k7qndG>xzDMiheo$ zKxcR5BxE+_2zhi^Qgric>^Fv**KMCnf=V5QFZaOPml^N%3x=)ht@9yD;=h;)c+C5& z4bSJ6Qjd zZXe){ohc;Ke|nd2QFBfPfPGQZ|N5<^f1Ko>-~N4)zx(9xdGfdI04w-=DfoLS_%B}y z!0z^Y^V$HL*R$aHOWKeB`{q_tgY4lrK60UhCYgOA)GiEaTMG9q_Ip?AJi|~>@ac$+ zn%Z+)+?e*dQ)66{n9J9^SOerGEqU!-&m#>k3Ig9p*!n2(i?M}oO#H$NLgr$dRxa^A z`JWa7|7Q`16Dvt(H!zEZOk>Fn*@|mZt>F<~QK@N?$8z!&{lSiYe&9QW7C33jF2S8G zc2dmpk5e$L#8mNa|29%{(c5yvoSKap07X(Pzu&!g09!{wX$>NFA&}ue>8kCZZeF^d zCj8$jv^D;Og1igc6uA#<-~XhWsxGKgfw*~T%QFWF6711RT^lCiqv|%H8=98j|WY zOI9)m$MV9H{%Sh$_+S3weQI_kIQF@{_pg@PfBnNo#HWeS?8@(Vo&Re2_r?4@GJj9A z-)8-{&HTHo8CK~B6hdpzId|~>TSIi6<`ntV6u?ONCTh_Ap(ixiB$EaHv<0^*EXHj()C(E>@}(I$Ukc=B z(1j2#@u%tMH{8PQ<;*I)+PG0~^DRul^exVa=(k~~3z(;1?X^D}$4ciMnyn*ARt&M~ zOUrmHb%_r!Z_JE!Kjmk>TPqVWzHtINn?9COCsB6)v6Nw2#*q}F%g>VSfekwmWAn8# zI%n$N-~PMT{@<7_&)OKx1^AM}lo9@9_!R$Xd*-7hCN(dAp1XcDW< z4X(R4PK<_|Tsmc*#jT=XEtC`bZ*23Qq=|lGGM+a=ZaxtxFAh5^ZKyo=PM^-Y_>K7` z?)56J8}uR4JzPf|;{SjDu;RXo3fGZSy%opLtW9M6Q{Co&so{L`&wmob_>aI`|F5+D HbN0Uh?g< Date: Tue, 7 Apr 2020 15:34:48 +0800 Subject: [PATCH 226/523] =?UTF-8?q?=E9=87=8D=E6=9E=84=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 ++++---- docs/prometheus.md | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 docs/prometheus.md diff --git a/README.md b/README.md index 0a42e30ce..f0a2b7de2 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,14 @@ FlinkStreamSQL ============ [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) -##技术交流 +## 技术交流 - 招聘**大数据平台开发工程师**，想了解岗位详细信息可以添加本人微信号ysqwhiletrue，注明招聘，如有意者发送简历至[sishu@dtstack.com](mailto:sishu@dtstack.com) - 我们使用[钉钉](https://www.dingtalk.com/)沟通交流，可以搜索群号[**30537511**]或者扫描下面的二维码进入钉钉群
-##介绍 +## 介绍 * 基于开源的flink，对其实时sql进行扩展 * 自定义create table 语法（包括源表,输出表,维表） * 自定义create view 语法 @@ -18,9 +18,9 @@ FlinkStreamSQL * 支持原生FLinkSQL所有的语法 * 扩展了输入和输出的性能指标到promethus -##目录 +## 目录 -[ 1.1 demo]((docs/demo.md)) +[ 1.1 demo](docs/demo.md) [ 1.2 快速开始](docs/quickStart.md) [ 1.3 参数配置](docs/config.md) [ 1.4 支持的插件介绍和demo](docs/pluginsInfo.md) diff --git a/docs/prometheus.md b/docs/prometheus.md new file mode 100644 index 000000000..a36498b42 --- /dev/null +++ b/docs/prometheus.md @@ -0,0 +1,7 @@ +## 使用 prometheus pushgateway 需要设置的 confProp 参数 +* metrics.reporter.promgateway.class: org.apache.flink.metrics.prometheus.PrometheusPushGatewayReporter +* metrics.reporter.promgateway.host: prometheus pushgateway的地址 +* metrics.reporter.promgateway.port：prometheus pushgateway的端口 +* metrics.reporter.promgateway.jobName: 实例名称 +* metrics.reporter.promgateway.randomJobNameSuffix: 是否在实例名称后面添加随机字符串(默认:true) +* metrics.reporter.promgateway.deleteOnShutdown: 是否在停止的时候删除数据(默认false) \ No newline at end of file From bdfdc14f45a6355823d3f507b92754f1deceba91 Mon Sep 17 00:00:00 2001 From: xuchao Date: Tue, 7 Apr 2020 15:37:18 +0800 Subject: [PATCH 227/523] =?UTF-8?q?=E9=87=8D=E6=9E=84=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- docs/createView.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f0a2b7de2..53cff0cdd 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ FlinkStreamSQL [ 1.4 支持的插件介绍和demo](docs/pluginsInfo.md) [ 1.5 指标参数](docs/newMetric.md) [ 1.6 自定义函数](docs/function.md) -[ 1.7 视图定义](docs/createView.md) +[ 1.7 自定义视图](docs/createView.md) ## License diff --git a/docs/createView.md b/docs/createView.md index 6f81fa8cb..5b923f180 100644 --- a/docs/createView.md +++ b/docs/createView.md @@ -1,4 +1,4 @@ -##语法 +## 语法 ``` CREATE VIEW viewName [ (columnName[ , columnName]*) ] @@ -7,7 +7,7 @@ CREATE VIEW viewName [ (columnName[ , columnName]*) ]; INSERT INTO viewName queryStatement; ``` -##样例 +## 样例 ``` CREATE TABLE MyTable( name varchar, From d69987412b5a5258689dca7aca9391c10761fa57 Mon Sep 17 00:00:00 2001 From: xuchao Date: Tue, 7 Apr 2020 15:48:30 +0800 Subject: [PATCH 228/523] =?UTF-8?q?=E9=87=8D=E6=9E=84=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- docs/demo.md | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 53cff0cdd..e5d4622eb 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,9 @@ FlinkStreamSQL [ 1.6 自定义函数](docs/function.md) [ 1.7 自定义视图](docs/createView.md) +## 如何贡献FlinkStreamSQL ## License - FlinkStreamSQL is under the Apache 2.0 license. See the [LICENSE](http://www.apache.org/licenses/LICENSE-2.0) file for details. diff --git a/docs/demo.md b/docs/demo.md index fe4b5c720..421ad6617 100644 --- a/docs/demo.md +++ b/docs/demo.md @@ -80,6 +80,4 @@ into ``` ### 创建视图demo: - -``` -``` +[参考视图文档](docs/createView.md) From 2079ce409b665d21ca4ecf04efc0869f1f507ecf Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 8 Apr 2020 10:35:27 +0800 Subject: [PATCH 229/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=B0=8Dcountdown?= =?UTF-8?q?=E7=9A=84=E6=8D=95=E7=8D=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/rdb/async/RdbAsyncReqRow.java | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index da6cb3876..19d78eccd 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -119,19 +119,18 @@ private void connectWithRetry(Map inputParams, CRow input, Resul while(!finishFlag.get()){ CountDownLatch latch = new CountDownLatch(1); rdbSqlClient.getConnection(conn -> { - if(conn.failed()){ - if(failCounter.getAndIncrement() % 1000 == 0){ - logger.error("getConnection error", conn.cause()); - } - if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ - resultFuture.completeExceptionally(conn.cause()); - finishFlag.set(true); - } - latch.countDown(); - conn.result().close(); - return; - } try { + if(conn.failed()){ + if(failCounter.getAndIncrement() % 1000 == 0){ + logger.error("getConnection error", conn.cause()); + } + if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ + resultFuture.completeExceptionally(conn.cause()); + finishFlag.set(true); + } + conn.result().close(); + return; + } CONN_STATUS.set(true); ScheduledFuture timerFuture = registerTimer(input, resultFuture); cancelTimerWhenComplete(resultFuture, timerFuture); From 8e4196a3b5da27115ecfbf0b8ae9052851b83737 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 8 Apr 2020 10:45:18 +0800 Subject: [PATCH 230/523] fix conflict --- .../flink/sql/exec/ExecuteProcessHelper.java | 16 ++++------ .../dtstack/flink/sql/side/SideSQLParser.java | 1 + .../dtstack/flink/sql/side/SideSqlExec.java | 30 ++++++------------- .../flink/sql/table/AbstractTableParser.java | 4 +-- .../sql/side/oracle/OracleAsyncSideInfo.java | 4 +-- .../flink/sql/sink/oracle/OracleDialect.java | 8 ++--- .../flink/sql/sink/rdb/AbstractRdbSink.java | 9 +++--- 7 files changed, 27 insertions(+), 45 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 1c0c2a702..8b0c9ec10 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -20,6 +20,11 @@ import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; import com.dtstack.flink.sql.parser.CreateFuncParser; +import com.dtstack.flink.sql.parser.CreateTmpTableParser; +import com.dtstack.flink.sql.parser.FlinkPlanner; +import com.dtstack.flink.sql.parser.InsertSqlParser; +import com.dtstack.flink.sql.parser.SqlParser; +import com.dtstack.flink.sql.parser.SqlTree; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -63,17 +68,6 @@ import org.apache.calcite.sql.SqlNode; import org.apache.commons.io.Charsets; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.StreamQueryConfig; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index 61546bb60..2296f64c5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -48,6 +48,7 @@ import java.util.Set; import static org.apache.calcite.sql.SqlKind.IDENTIFIER; +import static org.apache.calcite.sql.SqlKind.LITERAL; /** * Parsing sql, obtain execution information dimension table diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 7a657263c..c41e2264a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -47,23 +47,13 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlInsert; -import org.apache.calcite.sql.SqlJoin; import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlWithItem; -import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -71,7 +61,6 @@ import java.sql.Timestamp; import java.util.Arrays; -import java.util.Collection; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -168,8 +157,7 @@ public void exec(String sql, }else if (pollObj instanceof JoinInfo){ LOG.info("----------exec join info----------\n{}", pollObj.toString()); - preIsSideJoin = true; - joinFun(pollObj, localTableCache, sideTableMap, tableEnv, replaceInfoList); + joinFun(pollObj, localTableCache, sideTableMap, tableEnv); } } @@ -295,7 +283,7 @@ private Table getTableFromCache(Map localTableCache, String table * * @return */ - private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, SideTableInfo sideTableInfo) { + private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, AbstractSideTableInfo sideTableInfo) { List conditionFields = getConditionFields(conditionNode, sideTableAlias, sideTableInfo); if(CollectionUtils.isEqualCollection(conditionFields, convertPrimaryAlias(sideTableInfo))){ return true; @@ -303,7 +291,7 @@ private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, return false; } - private List convertPrimaryAlias(SideTableInfo sideTableInfo) { + private List convertPrimaryAlias(AbstractSideTableInfo sideTableInfo) { List res = Lists.newArrayList(); sideTableInfo.getPrimaryKeys().forEach(field -> { res.add(sideTableInfo.getPhysicalFields().getOrDefault(field, field)); @@ -311,7 +299,7 @@ private List convertPrimaryAlias(SideTableInfo sideTableInfo) { return res; } - public List getConditionFields(SqlNode conditionNode, String specifyTableName, SideTableInfo sideTableInfo){ + public List getConditionFields(SqlNode conditionNode, String specifyTableName, AbstractSideTableInfo sideTableInfo){ List sqlNodeList = Lists.newArrayList(); ParseUtils.parseAnd(conditionNode, sqlNodeList); List conditionFields = Lists.newArrayList(); @@ -370,7 +358,7 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, private void joinFun(Object pollObj, Map localTableCache, - Map sideTableMap, + Map sideTableMap, StreamTableEnvironment tableEnv) throws Exception{ JoinInfo joinInfo = (JoinInfo) pollObj; @@ -386,7 +374,7 @@ private void joinFun(Object pollObj, JoinScope.ScopeChild rightScopeChild = new JoinScope.ScopeChild(); rightScopeChild.setAlias(joinInfo.getRightTableAlias()); rightScopeChild.setTableName(joinInfo.getRightTableName()); - SideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); + AbstractSideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); if(sideTableInfo == null){ sideTableInfo = sideTableMap.get(joinInfo.getRightTableAlias()); } @@ -395,9 +383,9 @@ private void joinFun(Object pollObj, throw new RuntimeException("can't not find side table:" + joinInfo.getRightTableName()); } -// if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ -// throw new RuntimeException("ON condition must contain all equal fields!!!"); -// } + if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ + throw new RuntimeException("ON condition must contain all equal fields!!!"); + } rightScopeChild.setRowTypeInfo(sideTableInfo.getRowTypeInfo()); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 136779cda..1f210cbf2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -112,12 +112,12 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ Class fieldClass = null; - TableInfo.FieldExtraInfo fieldExtraInfo = null; + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; Matcher matcher = charTypePattern.matcher(fieldType); if (matcher.find()) { fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); - fieldExtraInfo = new TableInfo.FieldExtraInfo(); + fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); } else { fieldClass = dbTypeConvertToJavaType(fieldType); diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java index d2c75c716..658020b1c 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -56,7 +56,7 @@ public String wrapperPlaceholder(String fieldName) { String rpadFormat = "rpad(?, %d, ' ')"; if (StringUtils.contains(type.toLowerCase(), "char")) { - TableInfo.FieldExtraInfo fieldExtraInfo = sideTableInfo.getFieldExtraInfoList().get(pos); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = sideTableInfo.getFieldExtraInfoList().get(pos); int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); if (charLength > 0) { return String.format(rpadFormat, charLength); diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java index a687aa012..3a320658c 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.oracle; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; @@ -41,7 +41,7 @@ public class OracleDialect implements JDBCDialect { private List fieldList; private List fieldTypeList; - private List fieldExtraInfoList; + private List fieldExtraInfoList; @Override public boolean canHandle(String url) { @@ -133,7 +133,7 @@ public String wrapperPlaceholder(String fieldName) { String type = fieldTypeList.get(pos); if (StringUtils.contains(type.toLowerCase(), DEAL_CHAR_KEY)) { - TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfoList.get(pos); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfoList.get(pos); int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); if (charLength > 0) { return String.format(RPAD_FORMAT, charLength); @@ -151,7 +151,7 @@ public void setFieldTypeList(List fieldTypeList) { this.fieldTypeList = fieldTypeList; } - public void setFieldExtraInfoList(List fieldExtraInfoList) { + public void setFieldExtraInfoList(List fieldExtraInfoList) { this.fieldExtraInfoList = fieldExtraInfoList; } } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java index e71d4f427..ee51c22d1 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java @@ -20,9 +20,8 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -84,20 +83,20 @@ public abstract class AbstractRdbSink implements RetractStreamTableSink, Se public List fieldList; public List fieldTypeList; - public List fieldExtraInfoList; + public List fieldExtraInfoList; public AbstractRdbSink(JDBCDialect jdbcDialect) { this.jdbcDialect = jdbcDialect; } @Override - public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { + public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { RdbTableInfo rdbTableInfo = (RdbTableInfo) targetTableInfo; this.batchNum = rdbTableInfo.getBatchSize() == null ? batchNum : rdbTableInfo.getBatchSize(); this.batchWaitInterval = rdbTableInfo.getBatchWaitInterval() == null ? batchWaitInterval : rdbTableInfo.getBatchWaitInterval(); this.parallelism = rdbTableInfo.getParallelism() == null ? parallelism : rdbTableInfo.getParallelism(); - this.dbURL = rdbTableInfo.getUrl(); + this.dbUrl = rdbTableInfo.getUrl(); this.userName = rdbTableInfo.getUserName(); this.password = rdbTableInfo.getPassword(); this.tableName = rdbTableInfo.getTableName(); From aa9845a1e93a37243dc9fa2abfb7ac1679a5c1fe Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 8 Apr 2020 11:27:18 +0800 Subject: [PATCH 231/523] =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=A0=B9=E6=8D=AE?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E6=AC=A1=E6=95=B0=E5=92=8CjoinType=E8=BE=93?= =?UTF-8?q?=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/AbstractSideTableInfo.java | 16 +++--------- .../flink/sql/side/BaseAsyncReqRow.java | 2 +- .../sql/table/AbstractSideTableParser.java | 2 +- .../sql/side/rdb/async/RdbAsyncReqRow.java | 26 ++++++++++++++++--- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index 0a6babc8a..71a309411 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -68,13 +68,11 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements private int asyncTimeout=10000; - private int asyncTimeoutNumLimit = Integer.MAX_VALUE; - private boolean partitionedJoin = false; private String cacheMode="ordered"; - private Integer asyncFailMaxNum; + private Long asyncFailMaxNum; private List predicateInfoes = Lists.newArrayList(); @@ -153,19 +151,11 @@ public List getPredicateInfoes() { return predicateInfoes; } - public int getAsyncTimeoutNumLimit() { - return asyncTimeoutNumLimit; - } - - public void setAsyncTimeoutNumLimit(int asyncTimeoutNumLimit) { - this.asyncTimeoutNumLimit = asyncTimeoutNumLimit; - } - - public Integer getAsyncFailMaxNum(Integer defaultValue) { + public Long getAsyncFailMaxNum(Long defaultValue) { return Objects.isNull(asyncFailMaxNum) ? defaultValue : asyncFailMaxNum; } - public void setAsyncFailMaxNum(Integer asyncFailMaxNum) { + public void setAsyncFailMaxNum(Long asyncFailMaxNum) { this.asyncFailMaxNum = asyncFailMaxNum; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 8713b541a..bfe14c3ea 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -149,7 +149,7 @@ public void timeout(CRow input, ResultFuture resultFuture) throws Exceptio resultFuture.complete(null); return; } - if(timeOutNum > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Integer.MAX_VALUE)){ + if(timeOutNum > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); return; } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 83a06d256..7df86b3cb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -116,7 +116,7 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0){ sideTableInfo.setAsyncFailMaxNum(asyncFailNum); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 19d78eccd..e115ed21c 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -30,6 +30,7 @@ import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import com.google.common.collect.Lists; +import org.apache.calcite.sql.JoinType; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -38,16 +39,19 @@ import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import scala.Int; import java.sql.Timestamp; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; /** @@ -81,6 +85,8 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private final static AtomicBoolean CONN_STATUS = new AtomicBoolean(true); + private final static AtomicLong TIMOUT_NUM = new AtomicLong(0); + private Logger logger = LoggerFactory.getLogger(getClass()); public RdbAsyncReqRow(BaseSideInfo sideInfo) { @@ -114,7 +120,7 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul } private void connectWithRetry(Map inputParams, CRow input, ResultFuture resultFuture, SQLClient rdbSqlClient) { - AtomicInteger failCounter = new AtomicInteger(0); + AtomicLong failCounter = new AtomicLong(0); AtomicBoolean finishFlag = new AtomicBoolean(false); while(!finishFlag.get()){ CountDownLatch latch = new CountDownLatch(1); @@ -124,8 +130,8 @@ private void connectWithRetry(Map inputParams, CRow input, Resul if(failCounter.getAndIncrement() % 1000 == 0){ logger.error("getConnection error", conn.cause()); } - if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ - resultFuture.completeExceptionally(conn.cause()); + if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3L)){ + outByJoinType(resultFuture, conn.cause()); finishFlag.set(true); } conn.result().close(); @@ -202,8 +208,12 @@ private void handleQuery(SQLConnection connection,Map inputParam JsonArray params = new JsonArray(Lists.newArrayList(inputParams.values())); connection.queryWithParams(sideInfo.getSqlCondition(), params, rs -> { if (rs.failed()) { + if(TIMOUT_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ + outByJoinType(resultFuture, rs.cause()); + return; + } LOG.error("Cannot retrieve the data from the database", rs.cause()); - resultFuture.completeExceptionally(rs.cause()); + resultFuture.complete(null); return; } @@ -242,4 +252,12 @@ private void handleQuery(SQLConnection connection,Map inputParam }); } + private void outByJoinType(ResultFuture resultFuture, Throwable e){ + if(sideInfo.getJoinType() == JoinType.LEFT){ + resultFuture.complete(null); + return; + } + resultFuture.completeExceptionally(e); + } + } From caa8824a6bcd8e14f13dbd2c3b6ffa284dea5c04 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 8 Apr 2020 11:29:11 +0800 Subject: [PATCH 232/523] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E7=9A=84import?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index e115ed21c..c86434528 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -21,17 +21,15 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.BaseAsyncReqRow; -import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; -import io.vertx.core.json.Json; +import com.google.common.collect.Lists; import io.vertx.core.json.JsonArray; import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; -import com.google.common.collect.Lists; import org.apache.calcite.sql.JoinType; -import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; @@ -39,20 +37,14 @@ import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.Int; import java.sql.Timestamp; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; /** * Date: 2018/11/26 From ac65b81d92c5ca181418e082a929a905acadac60 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 8 Apr 2020 11:40:37 +0800 Subject: [PATCH 233/523] =?UTF-8?q?rowkey=E7=9A=84=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E6=8B=BC=E6=8E=A5=E5=BD=A2=E5=BC=8F=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/hbase/HbaseOutputFormat.java | 22 ++++++++----------- .../flink/sql/sink/hbase/HbaseSink.java | 2 +- .../sql/sink/hbase/table/HbaseSinkParser.java | 2 +- .../sql/sink/hbase/table/HbaseTableInfo.java | 6 ++--- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 1769fbda7..a3189b0a6 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -55,7 +55,7 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private String host; private String zkParent; - private String[] rowkey; + private String rowkey; private String tableName; private String[] columnNames; private String updateMode; @@ -164,25 +164,21 @@ private Put getPutByRow(Row record) { } private String buildRowKey(Row record) { - List rowKeyValues = getRowKeyValues(record); + String rowKeyValues = getRowKeyValues(record); // all rowkey not null - if (rowKeyValues.size() != rowkey.length) { + if (StringUtils.isBlank(rowKeyValues)) { LOG.error("row key value must not null,record is ..{}", record); outDirtyRecords.inc(); return ""; } - return StringUtils.join(rowKeyValues, "-"); + return rowKeyValues; } - private List getRowKeyValues(Row record) { - List rowKeyValues = Lists.newArrayList(); + private String getRowKeyValues(Row record) { Map row = rowConvertMap(record); - for (String key : rowkey) { - RowKeyBuilder rowKeyBuilder = new RowKeyBuilder(); - rowKeyBuilder.init(key); - rowKeyValues.add(rowKeyBuilder.getRowKey(row)); - } - return rowKeyValues; + RowKeyBuilder rowKeyBuilder = new RowKeyBuilder(); + rowKeyBuilder.init(rowkey); + return rowKeyBuilder.getRowKey(row); } private Map rowConvertMap(Row record){ @@ -232,7 +228,7 @@ public HbaseOutputFormatBuilder setTable(String tableName) { return this; } - public HbaseOutputFormatBuilder setRowkey(String[] rowkey) { + public HbaseOutputFormatBuilder setRowkey(String rowkey) { format.rowkey = rowkey; return this; } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index 9308725a2..13bd98b70 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -51,7 +51,7 @@ public class HbaseSink implements RetractStreamTableSink, IStreamSinkGener< protected String parent; protected String tableName; protected String updateMode; - protected String[] rowkey; + protected String rowkey; public HbaseSink() { // TO DO NOTHING diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index 8d8996a23..bd1b7cac3 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -69,7 +69,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map columnNameFamily; @@ -90,11 +90,11 @@ public void setParent(String parent) { this.parent = parent; } - public String[] getRowkey() { + public String getRowkey() { return rowkey; } - public void setRowkey(String[] rowkey) { + public void setRowkey(String rowkey) { this.rowkey = rowkey; } From 5f30a74b9518d4d33239b8726ff4d027442ca097 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 8 Apr 2020 14:00:06 +0800 Subject: [PATCH 234/523] Revert "Merge branch 'feat_1.8release3.10.x_mergeTest' into '1.8_release_3.10.x'" This reverts merge request !18 --- .gitlab-ci.yml | 10 - README.md | 29 +- .../side/cassandra/CassandraAllReqRow.java | 10 +- .../side/cassandra/CassandraAllSideInfo.java | 10 +- .../side/cassandra/CassandraAsyncReqRow.java | 11 +- .../cassandra/CassandraAsyncSideInfo.java | 18 +- .../cassandra/table/CassandraSideParser.java | 17 +- .../table/CassandraSideTableInfo.java | 4 +- .../sink/cassandra/CassandraOutputFormat.java | 4 +- .../sql/sink/cassandra/CassandraSink.java | 4 +- .../cassandra/table/CassandraSinkParser.java | 10 +- .../cassandra/table/CassandraTableInfo.java | 4 +- .../test/java/com/dtstack/flinkx/AppTest.java | 43 +- ci/sonar_notify.sh | 14 - .../side/clickhouse/ClickhouseAllReqRow.java | 17 +- .../clickhouse/ClickhouseAllSideInfo.java | 4 +- .../clickhouse/ClickhouseAsyncReqRow.java | 12 +- .../clickhouse/ClickhouseAsyncSideInfo.java | 4 +- .../table/ClickhouseSideParser.java | 6 +- .../sql/sink/clickhouse/ClickhouseSink.java | 9 +- .../table/ClickhouseSinkParser.java | 6 +- .../sql/sink/console/ConsoleOutputFormat.java | 6 +- .../flink/sql/sink/console/ConsoleSink.java | 4 +- .../sink/console/table/ConsoleSinkParser.java | 10 +- .../sink/console/table/ConsoleTableInfo.java | 4 +- .../sink/console/table/TablePrintUtil.java | 23 +- core/pom.xml | 10 +- .../main/java/com/dtstack/flink/sql/Main.java | 5 +- .../sql/classloader/ClassLoaderManager.java | 4 +- .../flink/sql/constrant/ConfigConstrant.java | 13 +- .../dtstack/flink/sql/enums/ColumnType.java | 63 +-- .../flink/sql/enums/ECacheContentType.java | 9 - .../dtstack/flink/sql/enums/ECacheType.java | 13 +- .../flink/sql/enums/EDatabaseType.java | 9 - .../flink/sql/enums/EPluginLoadMode.java | 6 - .../flink/sql/enums/EStateBackend.java | 9 - .../environment/StreamEnvConfigManager.java | 59 +-- .../flink/sql/exec/ExecuteProcessHelper.java | 76 ++-- .../dtstack/flink/sql/exec/ParamsInfo.java | 1 + .../format/SerializationMetricWrapper.java | 11 +- .../DtNestRowDeserializationSchema.java | 11 +- .../flink/sql/option/OptionParser.java | 21 +- ...putFormat.java => DtRichOutputFormat.java} | 2 +- .../flink/sql/parser/CreateFuncParser.java | 8 +- .../dtstack/flink/sql/parser/SqlParser.java | 12 +- .../com/dtstack/flink/sql/parser/SqlTree.java | 8 +- .../{BaseAllReqRow.java => AllReqRow.java} | 18 +- ...{BaseAsyncReqRow.java => AsyncReqRow.java} | 18 +- .../flink/sql/side/JoinNodeDealer.java | 10 +- .../flink/sql/side/ParserJoinField.java | 2 - .../side/{BaseSideInfo.java => SideInfo.java} | 22 +- .../flink/sql/side/SidePredicatesParser.java | 8 +- .../dtstack/flink/sql/side/SideSQLParser.java | 7 +- .../dtstack/flink/sql/side/SideSqlExec.java | 41 +- ...tSideTableInfo.java => SideTableInfo.java} | 33 +- .../flink/sql/side/StreamSideFactory.java | 10 +- ...stractSideCache.java => AbsSideCache.java} | 8 +- .../flink/sql/side/cache/LRUSideCache.java | 6 +- .../sql/side/operator/SideAsyncOperator.java | 16 +- .../operator/SideWithAllCacheOperator.java | 18 +- .../flink/sql/sink/IStreamSinkGener.java | 4 +- .../flink/sql/sink/StreamSinkFactory.java | 12 +- .../flink/sql/source/IStreamSourceGener.java | 4 +- .../flink/sql/source/StreamSourceFactory.java | 12 +- ...bleParser.java => AbsSideTableParser.java} | 53 +-- ...SourceParser.java => AbsSourceParser.java} | 16 +- ...ctTableParser.java => AbsTableParser.java} | 20 +- .../sql/table/ITableFieldDealHandler.java | 2 +- ...rceTableInfo.java => SourceTableInfo.java} | 2 +- ...{AbstractTableInfo.java => TableInfo.java} | 2 +- ...leInfoParser.java => TableInfoParser.java} | 20 +- ...getTableInfo.java => TargetTableInfo.java} | 2 +- .../com/dtstack/flink/sql/util/ClassUtil.java | 3 +- .../com/dtstack/flink/sql/util/DateUtil.java | 408 ++++++++++-------- .../dtstack/flink/sql/util/DtStringUtil.java | 8 +- .../com/dtstack/flink/sql/util/JDBCUtils.java | 4 +- .../com/dtstack/flink/sql/util/MathUtil.java | 2 +- .../dtstack/flink/sql/util/PluginUtil.java | 10 +- ...arker.java => AbsCustomerWaterMarker.java} | 4 +- .../CustomerWaterMarkerForLong.java | 2 +- .../CustomerWaterMarkerForTimeStamp.java | 3 +- .../sql/watermarker/WaterMarkerAssigner.java | 8 +- .../sql/side/SidePredicatesParserTest.java | 4 +- .../flink/sql/side/db2/Db2AllReqRow.java | 15 +- .../flink/sql/side/db2/Db2AllSideInfo.java | 4 +- .../flink/sql/side/db2/Db2AsyncReqRow.java | 10 +- .../flink/sql/side/db2/Db2AsyncSideInfo.java | 4 +- .../sql/side/db2/table/Db2SideParser.java | 6 +- .../com/dtstack/flink/sql/sink/db/DbSink.java | 9 +- .../flink/sql/sink/db/table/DbSinkParser.java | 6 +- docs/kafkaSink.md | 223 ---------- docs/kafkaSource.md | 224 ++++++++-- docs/mongoSide.md | 5 +- docs/mongoSink.md | 7 +- .../sink/elasticsearch/ElasticsearchSink.java | 4 +- .../table/ElasticsearchSinkParser.java | 8 +- .../table/ElasticsearchTableInfo.java | 4 +- .../Elasticsearch6AllReqRow.java | 21 +- .../Elasticsearch6AllSideInfo.java | 11 +- .../Elasticsearch6AsyncReqRow.java | 11 +- .../Elasticsearch6AsyncSideInfo.java | 13 +- .../table/Elasticsearch6SideParser.java | 8 +- .../table/Elasticsearch6SideTableInfo.java | 4 +- .../sql/side/elasticsearch6/util/Es6Util.java | 10 +- .../sink/elasticsearch/ElasticsearchSink.java | 4 +- .../table/ElasticsearchSinkParser.java | 8 +- .../table/ElasticsearchTableInfo.java | 4 +- .../flink/sql/side/hbase/HbaseAllReqRow.java | 22 +- .../sql/side/hbase/HbaseAllSideInfo.java | 10 +- .../sql/side/hbase/HbaseAsyncReqRow.java | 14 +- .../sql/side/hbase/HbaseAsyncSideInfo.java | 10 +- ...deDealer.java => AbsRowKeyModeDealer.java} | 12 +- .../PreRowKeyModeDealerDealer.java | 12 +- .../rowkeydealer/RowKeyEqualModeDealer.java | 9 +- .../sql/side/hbase/Md5ReplaceOperator.java | 2 +- ...laceOperator.java => ReplaceOperator.java} | 4 +- .../sql/side/hbase/enums/EReplaceOpType.java | 9 +- .../sql/side/hbase/enums/EReplaceType.java | 9 - .../sql/side/hbase/table/HbaseSideParser.java | 12 +- .../side/hbase/table/HbaseSideTableInfo.java | 4 +- .../sql/side/hbase/utils/HbaseUtils.java | 2 - .../sql/sink/hbase/HbaseOutputFormat.java | 11 +- .../flink/sql/sink/hbase/HbaseSink.java | 5 +- .../sql/sink/hbase/table/HbaseSinkParser.java | 10 +- .../sql/sink/hbase/table/HbaseTableInfo.java | 4 +- .../sql/side/impala/ImpalaAllReqRow.java | 13 +- .../sql/side/impala/ImpalaAllSideInfo.java | 7 +- .../sql/side/impala/ImpalaAsyncReqRow.java | 29 +- .../sql/side/impala/ImpalaAsyncSideInfo.java | 6 +- .../side/impala/table/ImpalaSideParser.java | 6 +- .../flink/sql/sink/impala/ImpalaSink.java | 14 +- .../sink/impala/table/ImpalaSinkParser.java | 7 +- .../kafka/AbstractKafkaProducerFactory.java | 37 +- .../sql/sink/kafka/AbstractKafkaSink.java | 148 ------- .../sink/kafka/CustomerFlinkPartition.java | 8 +- .../CustomerKeyedSerializationSchema.java | 35 +- .../AvroCRowSerializationSchema.java | 366 ---------------- .../CsvCRowSerializationSchema.java | 374 ---------------- .../JsonCRowSerializationSchema.java | 234 ---------- .../sql/sink/kafka/table/KafkaSinkParser.java | 13 +- .../sink/kafka/table/KafkaSinkTableInfo.java | 44 +- .../sql/source/kafka/AbstractKafkaSource.java | 117 ----- .../KafkaDeserializationMetricWrapper.java | 10 +- .../source/kafka/table/KafkaSourceParser.java | 42 +- .../kafka/table/KafkaSourceTableInfo.java | 181 ++++---- .../flink/sql/sink/kafka/KafkaProducer.java | 6 +- .../sql/sink/kafka/KafkaProducerFactory.java | 4 +- .../flink/sql/sink/kafka/KafkaSink.java | 118 ++++- .../flink/sql/source/kafka/KafkaSource.java | 78 +++- .../flink/sql/sink/kafka/KafkaProducer09.java | 5 +- .../sink/kafka/KafkaProducer09Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 124 +++++- .../flink/sql/source/kafka/KafkaSource.java | 76 +++- .../sql/sink/kafka/KafkaProducer010.java | 6 +- .../sink/kafka/KafkaProducer010Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 122 +++++- .../flink/sql/source/kafka/KafkaSource.java | 89 +++- .../sql/sink/kafka/KafkaProducer011.java | 6 +- .../sink/kafka/KafkaProducer011Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 121 +++++- .../source/kafka/KafkaConsumer011Factory.java | 2 +- .../flink/sql/source/kafka/KafkaSource.java | 92 +++- .../flink/sql/side/kudu/KuduAllReqRow.java | 24 +- .../flink/sql/side/kudu/KuduAllSideInfo.java | 10 +- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 26 +- .../sql/side/kudu/KuduAsyncSideInfo.java | 10 +- .../sql/side/kudu/table/KuduSideParser.java | 11 +- .../side/kudu/table/KuduSideTableInfo.java | 4 +- .../flink/sql/side/kudu/utils/KuduUtil.java | 1 - .../flink/sql/sink/kudu/KuduOutputFormat.java | 8 +- .../dtstack/flink/sql/sink/kudu/KuduSink.java | 4 +- .../sql/sink/kudu/table/KuduSinkParser.java | 11 +- .../sql/sink/kudu/table/KuduTableInfo.java | 4 +- .../sql/launcher/ClusterClientFactory.java | 8 +- .../perjob/PerJobClusterClientBuilder.java | 3 +- .../sql/launcher/perjob/PerJobSubmitter.java | 1 + .../flink/sql/side/mongo/MongoAllReqRow.java | 62 ++- .../sql/side/mongo/MongoAllSideInfo.java | 10 +- .../sql/side/mongo/MongoAsyncReqRow.java | 83 ++-- .../sql/side/mongo/MongoAsyncSideInfo.java | 13 +- .../sql/side/mongo/table/MongoSideParser.java | 12 +- .../side/mongo/table/MongoSideTableInfo.java | 4 +- .../flink/sql/side/mongo/utils/MongoUtil.java | 1 - .../sql/sink/mongo/MongoOutputFormat.java | 44 +- .../flink/sql/sink/mongo/MongoSink.java | 4 +- .../sql/sink/mongo/table/MongoSinkParser.java | 10 +- .../sql/sink/mongo/table/MongoTableInfo.java | 4 +- .../test/java/com/dtstack/flinkx/AppTest.java | 58 +++ .../flink/sql/side/mysql/MysqlAllReqRow.java | 12 +- .../sql/side/mysql/MysqlAllSideInfo.java | 4 +- .../sql/side/mysql/MysqlAsyncReqRow.java | 15 +- .../sql/side/mysql/MysqlAsyncSideInfo.java | 4 +- .../sql/side/mysql/table/MysqlSideParser.java | 6 +- .../flink/sql/sink/mysql/MysqlSink.java | 6 +- .../sql/sink/mysql/table/MysqlSinkParser.java | 6 +- .../test/java/com/dtstack/flinkx/AppTest.java | 58 +++ .../sql/side/oracle/OracleAllReqRow.java | 12 +- .../sql/side/oracle/OracleAllSideInfo.java | 5 +- .../sql/side/oracle/OracleAsyncReqRow.java | 15 +- .../sql/side/oracle/OracleAsyncSideInfo.java | 9 +- .../side/oracle/table/OracleSideParser.java | 6 +- .../flink/sql/sink/oracle/OracleDialect.java | 8 +- .../flink/sql/sink/oracle/OracleSink.java | 6 +- .../sink/oracle/table/OracleSinkParser.java | 6 +- .../sql/side/polardb/PolardbAllReqRow.java | 34 +- .../sql/side/polardb/PolardbAllSideInfo.java | 26 +- .../sql/side/polardb/PolardbAsyncReqRow.java | 32 +- .../side/polardb/PolardbAsyncSideInfo.java | 26 +- .../side/polardb/table/PolardbSideParser.java | 29 +- .../flink/sql/sink/polardb/PolardbSink.java | 30 +- .../sink/polardb/table/PolardbSinkParser.java | 28 +- pom.xml | 16 +- .../side/postgresql/PostgresqlAllReqRow.java | 12 +- .../postgresql/PostgresqlAllSideInfo.java | 4 +- .../postgresql/PostgresqlAsyncReqRow.java | 10 +- .../postgresql/PostgresqlAsyncSideInfo.java | 4 +- .../table/PostgresqlSideParser.java | 6 +- .../sql/sink/postgresql/PostgresqlSink.java | 6 +- .../table/PostgresqlSinkParser.java | 6 +- ...actRdbAllReqRow.java => RdbAllReqRow.java} | 20 +- .../sql/side/rdb/all/RdbAllSideInfo.java | 15 +- .../sql/side/rdb/async/RdbAsyncReqRow.java | 45 +- .../sql/side/rdb/async/RdbAsyncSideInfo.java | 16 +- .../provider/DTC3P0DataSourceProvider.java | 10 +- .../sql/side/rdb/table/RdbSideParser.java | 8 +- .../sql/side/rdb/table/RdbSideTableInfo.java | 15 +- .../flink/sql/side/rdb/util/SwitchUtil.java | 1 - .../flink/sql/sink/rdb/JDBCOptions.java | 22 +- .../{AbstractRdbSink.java => RdbSink.java} | 19 +- .../rdb/format/AbstractJDBCOutputFormat.java | 14 +- .../rdb/format/JDBCUpsertOutputFormat.java | 14 +- .../rdb/format/RetractJDBCOutputFormat.java | 0 .../sql/sink/rdb/table/RdbSinkParser.java | 9 +- .../sql/sink/rdb/table/RdbTableInfo.java | 6 +- .../sql/sink/rdb/writer/AppendOnlyWriter.java | 13 +- ...actUpsertWriter.java => UpsertWriter.java} | 86 ++-- .../flink/sql/side/redis/RedisAllReqRow.java | 32 +- .../sql/side/redis/RedisAllSideInfo.java | 14 +- .../sql/side/redis/RedisAsyncReqRow.java | 26 +- .../sql/side/redis/RedisAsyncSideInfo.java | 14 +- .../sql/side/redis/table/RedisSideParser.java | 11 +- .../sql/side/redis/table/RedisSideReqRow.java | 8 +- .../side/redis/table/RedisSideTableInfo.java | 8 +- .../sql/sink/redis/RedisOutputFormat.java | 29 +- .../flink/sql/sink/redis/RedisSink.java | 8 +- .../sql/sink/redis/table/RedisSinkParser.java | 12 +- .../sql/sink/redis/table/RedisTableInfo.java | 8 +- .../CustomerSocketTextStreamFunction.java | 6 +- .../serversocket/ServersocketSource.java | 4 +- .../table/ServersocketSourceParser.java | 8 +- .../table/ServersocketSourceTableInfo.java | 6 +- .../side/sqlserver/SqlserverAllReqRow.java | 20 +- .../side/sqlserver/SqlserverAllSideInfo.java | 10 +- .../side/sqlserver/SqlserverAsyncReqRow.java | 14 +- .../sqlserver/SqlserverAsyncSideInfo.java | 11 +- .../sqlserver/table/SqlserverSideParser.java | 12 +- .../sql/sink/sqlserver/SqlserverSink.java | 6 +- .../sqlserver/table/SqlserverSinkParser.java | 6 +- 258 files changed, 2672 insertions(+), 3589 deletions(-) delete mode 100644 .gitlab-ci.yml rename kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java => cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java (56%) delete mode 100644 ci/sonar_notify.sh rename core/src/main/java/com/dtstack/flink/sql/outputformat/{AbstractDtRichOutputFormat.java => DtRichOutputFormat.java} (95%) rename core/src/main/java/com/dtstack/flink/sql/side/{BaseAllReqRow.java => AllReqRow.java} (79%) rename core/src/main/java/com/dtstack/flink/sql/side/{BaseAsyncReqRow.java => AsyncReqRow.java} (89%) rename core/src/main/java/com/dtstack/flink/sql/side/{BaseSideInfo.java => SideInfo.java} (92%) rename core/src/main/java/com/dtstack/flink/sql/side/{AbstractSideTableInfo.java => SideTableInfo.java} (79%) rename core/src/main/java/com/dtstack/flink/sql/side/cache/{AbstractSideCache.java => AbsSideCache.java} (84%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbstractSideTableParser.java => AbsSideTableParser.java} (62%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbstractSourceParser.java => AbsSourceParser.java} (81%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbstractTableParser.java => AbsTableParser.java} (88%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbstractSourceTableInfo.java => SourceTableInfo.java} (97%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbstractTableInfo.java => TableInfo.java} (98%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbstractTableInfoParser.java => TableInfoParser.java} (83%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbstractTargetTableInfo.java => TargetTableInfo.java} (94%) rename core/src/main/java/com/dtstack/flink/sql/watermarker/{AbstractCustomerWaterMarker.java => AbsCustomerWaterMarker.java} (94%) delete mode 100644 docs/kafkaSink.md rename hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/{AbstractRowKeyModeDealer.java => AbsRowKeyModeDealer.java} (87%) rename hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/{AbstractReplaceOperator.java => ReplaceOperator.java} (93%) delete mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java delete mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java delete mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java delete mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java delete mode 100644 kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java create mode 100644 mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java create mode 100644 mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java rename rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/{AbstractRdbAllReqRow.java => RdbAllReqRow.java} (92%) rename rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/{AbstractRdbSink.java => RdbSink.java} (90%) delete mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java rename rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/{AbstractUpsertWriter.java => UpsertWriter.java} (84%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 2952551ff..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,10 +0,0 @@ -build: - stage: test - script: - - mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q - - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.branch.name="v1.8.0_dev" -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar - - sh ci/sonar_notify.sh - only: - - v1.8.0_dev - tags: - - dt-insight-engine \ No newline at end of file diff --git a/README.md b/README.md index c84d77d5a..db25164b6 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,34 @@ > > * 支持原生FLinkSQL所有的语法 > > * 扩展了输入和输出的性能指标到promethus + ## 新特性: + * 1.kafka源表支持not null语法,支持字符串类型的时间转换。 + * 2.rdb维表与DB建立连接时，周期进行连接，防止连接断开。rdbsink写入时，对连接进行检查。 + * 3.异步维表支持非等值连接，比如：<>,<,>。 + * 4.增加kafka数组解析 + * 5.增加kafka1.0以上版本的支持 + * 6.增加postgresql、kudu、clickhouse维表、结果表的支持 + * 7.支持插件的依赖方式,参考pluginLoadMode参数 + * 8.支持cep处理 + * 9.支持udaf + * 10.支持谓词下移 + * 11.支持状态的ttl + + ## BUG修复： + * 1.修复不能解析sql中orderby,union语法。 + * 2.修复yarnPer模式提交失败的异常。 + * 3.一些bug的修复 + # 已支持 * 源表：kafka 0.9、0.10、0.11、1.x版本 * 维表：mysql, SQlServer,oracle, hbase, mongo, redis, cassandra, serversocket, kudu, postgresql, clickhouse, impala, db2, sqlserver * 结果表：mysql, SQlServer, oracle, hbase, elasticsearch5.x, mongo, redis, cassandra, console, kudu, postgresql, clickhouse, impala, db2, sqlserver +# 后续开发计划 + * 维表快照 + * kafka avro格式 + * topN + ## 1 快速起步 ### 1.1 运行模式 @@ -126,10 +149,7 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack * taskmanager.slots：per_job模式下指定每个taskmanager对应的slot数量(默认1) * savePointPath：任务恢复点的路径（默认无） * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） - * restore.enable：是否失败重启（默认是true） - * failure.interval：衡量失败率的时间段，单位分钟（默认6m） - * delay.interval：连续两次重启尝试间的间隔，单位是秒（默认10s） - * logLevel: 日志级别动态配置（默认info） + * logLevel: 日志级别动态配置（默认info） * [prometheus 相关参数](docs/prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 @@ -182,7 +202,6 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack * [impala 结果表插件](docs/impalaSink.md) * [db2 结果表插件](docs/db2Sink.md) * [sqlserver 结果表插件](docs/sqlserverSink.md) -* [kafka 结果表插件](docs/kafkaSink.md) ### 2.3 维表插件 * [hbase 维表插件](docs/hbaseSide.md) diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index 3ad8b934e..bc0a0f8cd 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -34,10 +34,10 @@ import com.datastax.driver.core.SocketOptions; import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; -import com.dtstack.flink.sql.side.BaseAllReqRow; +import com.dtstack.flink.sql.side.AllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -62,12 +62,14 @@ * * @author xuqianjin */ -public class CassandraAllReqRow extends BaseAllReqRow { +public class CassandraAllReqRow extends AllReqRow { private static final long serialVersionUID = 54015343561288219L; private static final Logger LOG = LoggerFactory.getLogger(CassandraAllReqRow.class); + private static final String cassandra_DRIVER = "com.cassandra.jdbc.Driver"; + private static final int CONN_RETRY_NUM = 3; private static final int FETCH_SIZE = 1000; @@ -77,7 +79,7 @@ public class CassandraAllReqRow extends BaseAllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public CassandraAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public CassandraAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new com.dtstack.flink.sql.side.cassandra.CassandraAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java index 5d95dc9bf..fa665f9a0 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; @@ -37,16 +37,16 @@ * * @author xuqianjin */ -public class CassandraAllSideInfo extends BaseSideInfo { +public class CassandraAllSideInfo extends SideInfo { private static final long serialVersionUID = -8690814317653033557L; - public CassandraAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public CassandraAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { CassandraSideTableInfo cassandraSideTableInfo = (CassandraSideTableInfo) sideTableInfo; sqlCondition = "select ${selectField} from ${tableName} "; diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index 7eee3cb5f..0173f2d6f 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -37,11 +37,11 @@ import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.AsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.google.common.base.Function; @@ -67,7 +67,7 @@ * * @author xuqianjin */ -public class CassandraAsyncReqRow extends BaseAsyncReqRow { +public class CassandraAsyncReqRow extends AsyncReqRow { private static final long serialVersionUID = 6631584128079864735L; @@ -83,7 +83,7 @@ public class CassandraAsyncReqRow extends BaseAsyncReqRow { private transient ListenableFuture session; private transient CassandraSideTableInfo cassandraSideTableInfo; - public CassandraAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public CassandraAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new com.dtstack.flink.sql.side.cassandra.CassandraAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -216,7 +216,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce connCassandraDB(cassandraSideTableInfo); String sqlCondition = sideInfo.getSqlCondition() + " " + sqlWhere + " ALLOW FILTERING "; - LOG.info("sqlCondition:{}" + sqlCondition); + System.out.println("sqlCondition:" + sqlCondition); ListenableFuture resultSet = Futures.transformAsync(session, new AsyncFunction() { @@ -265,6 +265,7 @@ public void onSuccess(List rows) { public void onFailure(Throwable t) { LOG.error("Failed to retrieve the data: %s%n", t.getMessage()); + System.out.println("Failed to retrieve the data: " + t.getMessage()); cluster.closeAsync(); resultFuture.completeExceptionally(t); } diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java index 4bee5648b..3557f0f73 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlBasicCall; @@ -30,8 +30,6 @@ import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; @@ -41,18 +39,16 @@ * * @author xuqianjin */ -public class CassandraAsyncSideInfo extends BaseSideInfo { +public class CassandraAsyncSideInfo extends SideInfo { private static final long serialVersionUID = -4403313049809013362L; - private static final Logger LOG = LoggerFactory.getLogger(CassandraAsyncSideInfo.class.getSimpleName()); - - public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { CassandraSideTableInfo cassandraSideTableInfo = (CassandraSideTableInfo) sideTableInfo; String sideTableName = joinInfo.getSideTableName(); @@ -67,9 +63,9 @@ public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInf } sqlCondition = "select ${selectField} from ${tableName}"; - sqlCondition = sqlCondition.replace("${tableName}", cassandraSideTableInfo.getDatabase()+"."+cassandraSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields); - LOG.info("---------side_exe_sql-----\n{}" + sqlCondition); + sqlCondition = sqlCondition.replace("${tableName}", cassandraSideTableInfo.getDatabase()+"."+cassandraSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields); + System.out.println("---------side_exe_sql-----\n" + sqlCondition); } diff --git a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java index e893e56b0..62dd753b9 100644 --- a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java +++ b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java @@ -19,16 +19,18 @@ package com.dtstack.flink.sql.side.cassandra.table; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; +import java.math.BigDecimal; +import java.sql.Date; import java.sql.Timestamp; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; /** * Reason: @@ -36,7 +38,7 @@ * * @author xuqianjin */ -public class CassandraSideParser extends AbstractSideTableParser { +public class CassandraSideParser extends AbsSideTableParser { private final static String SIDE_SIGN_KEY = "sideSignKey"; @@ -71,7 +73,7 @@ public CassandraSideParser() { } @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo cassandraSideTableInfo = new com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo(); cassandraSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, cassandraSideTableInfo); @@ -94,10 +96,9 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map { +public class CassandraOutputFormat extends DtRichOutputFormat { private static final long serialVersionUID = -7994311331389155692L; private static final Logger LOG = LoggerFactory.getLogger(CassandraOutputFormat.class); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java index 26152a7d3..eb7b23b53 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.cassandra.table.CassandraTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -63,7 +63,7 @@ public CassandraSink() { } @Override - public CassandraSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public CassandraSink genStreamSink(TargetTableInfo targetTableInfo) { CassandraTableInfo cassandraTableInfo = (CassandraTableInfo) targetTableInfo; this.address = cassandraTableInfo.getAddress(); this.tableName = cassandraTableInfo.getTableName(); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java index 9ef8639ba..4c68e71ae 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java @@ -19,13 +19,13 @@ package com.dtstack.flink.sql.sink.cassandra.table; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; /** * Reason: @@ -33,7 +33,7 @@ * * @author xuqianjin */ -public class CassandraSinkParser extends AbstractTableParser { +public class CassandraSinkParser extends AbsTableParser { public static final String ADDRESS_KEY = "address"; @@ -60,7 +60,7 @@ public class CassandraSinkParser extends AbstractTableParser { public static final String POOL_TIMEOUT_MILLIS_KEY = "poolTimeoutMillis"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { CassandraTableInfo cassandraTableInfo = new CassandraTableInfo(); cassandraTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, cassandraTableInfo); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java index ffb5fa876..c6626c42a 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.cassandra.table; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; /** @@ -28,7 +28,7 @@ * * @author xuqianjin */ -public class CassandraTableInfo extends AbstractTargetTableInfo { +public class CassandraTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "cassandra"; diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java b/cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java similarity index 56% rename from kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java rename to cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java index 476ccc8b7..33a0233ac 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java +++ b/cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java @@ -16,16 +16,43 @@ * limitations under the License. */ -package com.dtstack.flink.sql.source.kafka.enums; + + +package com.dtstack.flinkx; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; /** - * Date: 2020/3/20 - * Company: www.dtstack.com - * @author maqi + * Unit test for simple App. */ -public enum EKafkaOffset { +public class AppTest + extends TestCase +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public AppTest( String testName ) + { + super( testName ); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite( AppTest.class ); + } - LATEST, - EARLIEST, - NONE + /** + * Rigourous Test :-) + */ + public void testApp() + { + assertTrue( true ); + } } diff --git a/ci/sonar_notify.sh b/ci/sonar_notify.sh deleted file mode 100644 index 41f8a3c0e..000000000 --- a/ci/sonar_notify.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -#参考钉钉文档 https://open-doc.dingtalk.com/microapp/serverapi2/qf2nxq - sonarreport=$(curl -s http://172.16.100.198:8082/?projectname=dt-insight-engine/flinkStreamSQL) - curl -s "https://oapi.dingtalk.com/robot/send?access_token=71555061297a53d3ac922a6f4d94285d8e23bccdca0c00b4dc6df0a2d49da724" \ - -H "Content-Type: application/json" \ - -d "{ - \"msgtype\": \"markdown\", - \"markdown\": { - \"title\":\"sonar代码质量\", - \"text\": \"## sonar代码质量报告: \n -> [sonar地址](http://172.16.100.198:9000/dashboard?id=dt-insight-engine/flinkStreamSQL) \n -> ${sonarreport} \n\" - } - }" \ No newline at end of file diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java index c9a0c447b..68c0c7984 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java @@ -20,37 +20,40 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Connection; import java.sql.DriverManager; import java.util.List; +import java.util.Map; -public class ClickhouseAllReqRow extends AbstractRdbAllReqRow { +public class ClickhouseAllReqRow extends RdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(ClickhouseAllReqRow.class); private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - public ClickhouseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ClickhouseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new ClickhouseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbUrl, String userName, String passWord) { + public Connection getConn(String dbURL, String userName, String passWord) { try { Connection connection ; JDBCUtils.forName(CLICKHOUSE_DRIVER, getClass().getClassLoader()); // ClickHouseProperties contains all properties if (userName == null) { - connection = DriverManager.getConnection(dbUrl); + connection = DriverManager.getConnection(dbURL); } else { - connection = DriverManager.getConnection(dbUrl, userName, passWord); + connection = DriverManager.getConnection(dbURL, userName, passWord); } return connection; } catch (Exception e) { diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java index 43fbeaa56..973c069b9 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -28,7 +28,7 @@ public class ClickhouseAllSideInfo extends RdbAllSideInfo { - public ClickhouseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ClickhouseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index 9da7d8eaa..2b5223412 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -37,7 +37,7 @@ public class ClickhouseAsyncReqRow extends RdbAsyncReqRow { private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - public ClickhouseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ClickhouseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new ClickhouseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -48,7 +48,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); clickhouseClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", CLICKHOUSE_DRIVER) - .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -58,11 +58,11 @@ public void open(Configuration parameters) throws Exception { System.setProperty("vertx.disableFileCPResolving", "true"); VertxOptions vo = new VertxOptions(); - vo.setEventLoopPoolSize(rdbSideTableInfo.getAsyncPoolSize()); - vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); + setRdbSQLClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); } } diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java index eec5fbe74..254561de0 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -29,7 +29,7 @@ public class ClickhouseAsyncSideInfo extends RdbAsyncSideInfo { - public ClickhouseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ClickhouseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java b/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java index ab285c37f..7be387fd8 100644 --- a/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java +++ b/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.clickhouse.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import ru.yandex.clickhouse.domain.ClickHouseDataType; import java.util.Map; @@ -38,8 +38,8 @@ public class ClickhouseSideParser extends RdbSideParser { private static final String CURR_TYPE = "clickhouse"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); clickhouseTableInfo.setType(CURR_TYPE); return clickhouseTableInfo; } diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java index 5f7e41f7b..bbb6a6fe4 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java @@ -22,11 +22,14 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; +import java.util.List; +import java.util.Map; -public class ClickhouseSink extends AbstractRdbSink implements IStreamSinkGener { + +public class ClickhouseSink extends RdbSink implements IStreamSinkGener { public ClickhouseSink() { super(new ClickhouseDialect()); } @@ -34,7 +37,7 @@ public ClickhouseSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(dbUrl) + .setDBUrl(dbURL) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java index 5b0f2598f..8c3df93d7 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.clickhouse.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import ru.yandex.clickhouse.domain.ClickHouseDataType; import java.util.Map; @@ -30,8 +30,8 @@ public class ClickhouseSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "clickhouse"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); clickhouseTableInfo.setType(CURR_TYPE); return clickhouseTableInfo; } diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java index fc5a768c4..a397036ef 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.console; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import com.dtstack.flink.sql.sink.console.table.TablePrintUtil; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -37,7 +37,7 @@ * * @author xuqianjin */ -public class ConsoleOutputFormat extends AbstractDtRichOutputFormat { +public class ConsoleOutputFormat extends DtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(ConsoleOutputFormat.class); @@ -56,7 +56,7 @@ public void open(int taskNumber, int numTasks) throws IOException { @Override public void writeRecord(Tuple2 tuple2) throws IOException { - LOG.info("received oriainal data:{}" + tuple2); + System.out.println("received oriainal data:" + tuple2); Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); if (!retract) { diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java index 6cfdd2d3d..77a3efea2 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.console; import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -80,7 +80,7 @@ public void emitDataStream(DataStream> dataStream) { } @Override - public ConsoleSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public ConsoleSink genStreamSink(TargetTableInfo targetTableInfo) { return this; } } diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java index 93ed02420..e77444bfd 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java @@ -18,13 +18,13 @@ package com.dtstack.flink.sql.sink.console.table; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; /** * Reason: @@ -32,9 +32,9 @@ * * @author xuqianjin */ -public class ConsoleSinkParser extends AbstractTableParser { +public class ConsoleSinkParser extends AbsTableParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ConsoleTableInfo consoleTableInfo = new ConsoleTableInfo(); consoleTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, consoleTableInfo); diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java index 6fd7063c6..4b286c667 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.console.table; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; /** * Reason: @@ -26,7 +26,7 @@ * * @author xuqianjin */ -public class ConsoleTableInfo extends AbstractTargetTableInfo { +public class ConsoleTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "console"; diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java index 12bb2a5d4..8813da619 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java @@ -22,7 +22,6 @@ public class TablePrintUtil { public static final int ALIGN_LEFT = 1;//左对齐 public static final int ALIGN_RIGHT = 2;//右对齐 public static final int ALIGN_CENTER = 3;//居中对齐 - private static final Pattern PATTERN = Pattern.compile("[\u4e00-\u9fa5]"); private int align = ALIGN_CENTER;//默认居中对齐 private boolean equilong = false;//默认不等宽 @@ -59,9 +58,7 @@ public static TablePrintUtil build(String[][] data) { public static TablePrintUtil build(List data) { TablePrintUtil self = new TablePrintUtil(); self.data = new ArrayList<>(); - if (data.size() <= 0) { - throw new RuntimeException("数据源至少得有一行吧"); - } + if (data.size() <= 0) throw new RuntimeException("数据源至少得有一行吧"); Object obj = data.get(0); @@ -73,9 +70,7 @@ public static TablePrintUtil build(List data) { int length = ((List) obj).size(); for (Object item : data) { List col = (List) item; - if (col.size() != length) { - throw new RuntimeException("数据源每列长度必须一致"); - } + if (col.size() != length) throw new RuntimeException("数据源每列长度必须一致"); self.data.add(col.toArray(new String[length])); } } else { @@ -95,7 +90,7 @@ public static TablePrintUtil build(List data) { try { value = obj.getClass().getMethod(colList.get(j).getMethodName).invoke(data.get(i)).toString(); } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { - LOG.error("", e); + e.printStackTrace(); } item[j] = value == null ? "null" : value; } @@ -120,7 +115,7 @@ private static List
getColList(Object obj) { Method[] methods = obj.getClass().getMethods(); for (Method m : methods) { StringBuilder getMethodName = new StringBuilder(m.getName()); - if ("get".equals(getMethodName.substring(0, 3)) && !"getClass".equals(m.getName())) { + if (getMethodName.substring(0, 3).equals("get") && !m.getName().equals("getClass")) { Col col = new Col(); col.getMethodName = getMethodName.toString(); char first = Character.toLowerCase(getMethodName.delete(0, 3).charAt(0)); @@ -139,8 +134,8 @@ private static ListgetColList(Object obj) { * @return */ private int getStringCharLength(String str) { - //利用正则找到中文 - Matcher m = PATTERN.matcher(str); + Pattern p = Pattern.compile("[\u4e00-\u9fa5]");//利用正则找到中文 + Matcher m = p.matcher(str); int count = 0; while (m.find()) { count++; @@ -168,9 +163,7 @@ private int[] getColLengths() { if (equilong) {//如果等宽表格 int max = 0; for (int len : result) { - if (len > max) { - max = len; - } + if (len > max) max = len; } for (int i = 0; i < result.length; i++) { result[i] = max; @@ -220,8 +213,6 @@ public String getTableString() { sb.append(cell); for (int i = 0; i < right + padding; i++) {sb.append(s);} break; - default: - break; } sb.append(v); } diff --git a/core/pom.xml b/core/pom.xml index eb470a3c0..b17209612 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -20,7 +20,7 @@ 1.16.0 2.7.9 19.0 - 1.0.0-SNAPSHOT + 1.1.7 @@ -122,11 +122,13 @@ junit 4.12 + - com.aiweiergou - tools-logger - ${logger.tool.version} + ch.qos.logback + logback-classic + ${logback.version} + diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index fe925f9a6..b54c3feb2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -21,12 +21,14 @@ +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + import com.dtstack.flink.sql.exec.ExecuteProcessHelper; import com.dtstack.flink.sql.exec.ParamsInfo; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + /** * Date: 2018/6/26 * Company: www.dtstack.com @@ -38,7 +40,6 @@ public class Main { public static void main(String[] args) throws Exception { ParamsInfo paramsInfo = ExecuteProcessHelper.parseParams(args); - ExecuteProcessHelper.setLogLevel(paramsInfo); StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExecution(paramsInfo); env.execute(paramsInfo.getName()); LOG.info("program {} execution success", paramsInfo.getName()); diff --git a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java index 2e62e11ab..c03f60617 100644 --- a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java @@ -98,9 +98,9 @@ public static List getClassPath() { - public static URLClassLoader loadExtraJar(List jarUrlList, URLClassLoader classLoader) + public static URLClassLoader loadExtraJar(List jarURLList, URLClassLoader classLoader) throws IllegalAccessException, InvocationTargetException { - for(URL url : jarUrlList){ + for(URL url : jarURLList){ if(url.toString().endsWith(".jar")){ urlClassLoaderAddUrl(classLoader, url); } diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java index 13e1de8c8..00d1ea0a5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java @@ -63,21 +63,14 @@ public class ConfigConstrant { public static final String CHECKPOINTS_DIRECTORY_KEY = "state.checkpoints.dir"; public static final String STATE_BACKEND_INCREMENTAL_KEY = "state.backend.incremental"; - public static final String RESTOREENABLE = "restore.enable"; - public static final String LOG_LEVEL_KEY = "logLevel"; // restart plocy - public static final int FAILUEE_RATE = 3; - - public static final int FAILUEE_INTERVAL = 6; //min - - public static final int DELAY_INTERVAL = 10; //sec - - public static final String FAILUREINTERVAL = "failure.interval"; //min + public static final int failureRate = 3; - public static final String DELAYINTERVAL= "delay.interval"; //sec + public static final int failureInterval = 6; //min + public static final int delayInterval = 10; //sec } diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java index 7f3f0019c..749bbc907 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java @@ -46,66 +46,11 @@ * @author huyifan.zju@163.com */ public enum ColumnType { - /* - * string - */ - STRING, - /** - * varchar - */ - VARCHAR, - /** - * char - */ - CHAR, - /** - * int - */ - INT, - /** - * mediumint - */ - MEDIUMINT, - /** - * tinyint - */ - TINYINT, - /** - * datetime - */ - DATETIME, - /** - * smallint - */ - SMALLINT, - /** - * bigint - */ - BIGINT, - /** - * double - */ - DOUBLE, - /** - * float - */ - FLOAT, - /** - * boolean - */ + STRING, VARCHAR, CHAR, + INT, MEDIUMINT, TINYINT, DATETIME, SMALLINT, BIGINT, + DOUBLE, FLOAT, BOOLEAN, - /** - * date - */ - DATE, - /** - * timestamp - */ - TIMESTAMP, - /** - * decimal - */ - DECIMAL; + DATE, TIMESTAMP, DECIMAL; public static ColumnType fromString(String type) { if(type == null) { diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java index 49e352757..66160d820 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java @@ -30,17 +30,8 @@ public enum ECacheContentType { - /** - * 无 - */ MissVal(0), - /** - * 1行 - */ SingleLine(1), - /** - * 多行 - */ MultiLine(2); int type; diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java index 9d5bb5d11..582148c2c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java @@ -27,18 +27,7 @@ * @author xuchao */ public enum ECacheType { - /** - * none - */ - NONE, - /** - * lru - */ - LRU, - /** - * all - */ - ALL; + NONE, LRU, ALL; public static boolean isValid(String type){ for(ECacheType tmpType : ECacheType.values()){ diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java b/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java index 7d2235f2e..9b01bf052 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java @@ -26,17 +26,8 @@ */ public enum EDatabaseType { - /** - * mysql - */ MYSQL, - /** - * sqlserver - */ SQLSERVER, - /** - * oracle - */ ORACLE, } diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java b/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java index 439966dd2..6cb027ac3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java @@ -26,13 +26,7 @@ */ public enum EPluginLoadMode { - /** - * 0:classpath - */ CLASSPATH(0), - /** - * 1:shipfile - */ SHIPFILE(1); private int type; diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java b/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java index 098cb57fe..a8f926175 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java @@ -25,17 +25,8 @@ * @author maqi */ public enum EStateBackend { - /** - * memory - */ MEMORY, - /** - * rockdb - */ ROCKSDB, - /** - * filesystem - */ FILESYSTEM; public static EStateBackend convertFromString(String type) { diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index d7b772c6e..ce763ea99 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -18,12 +18,6 @@ package com.dtstack.flink.sql.environment; -import com.dtstack.flink.sql.constrant.ConfigConstrant; -import com.dtstack.flink.sql.enums.EStateBackend; -import com.dtstack.flink.sql.util.MathUtil; -import com.dtstack.flink.sql.util.PropertiesUtils; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.time.Time; @@ -40,6 +34,13 @@ import org.apache.flink.table.api.StreamQueryConfig; import org.apache.flink.table.api.java.StreamTableEnvironment; +import com.dtstack.flink.sql.constrant.ConfigConstrant; +import com.dtstack.flink.sql.enums.EStateBackend; +import com.dtstack.flink.sql.util.MathUtil; +import com.dtstack.flink.sql.util.PropertiesUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; + import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -101,15 +102,11 @@ public static void streamExecutionEnvironmentConfig(StreamExecutionEnvironment s } }); - if(isRestore(confProperties).get()){ - streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( - ConfigConstrant.FAILUEE_RATE, - Time.of(getFailureInterval(confProperties).get(), TimeUnit.MINUTES), - Time.of(getDelayInterval(confProperties).get(), TimeUnit.SECONDS) - )); - } else { - streamEnv.setRestartStrategy(RestartStrategies.noRestart()); - } + streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( + ConfigConstrant.failureRate, + Time.of(ConfigConstrant.failureInterval, TimeUnit.MINUTES), + Time.of(ConfigConstrant.delayInterval, TimeUnit.SECONDS) + )); // checkpoint config Optional checkpointingEnabled = isCheckpointingEnabled(confProperties); @@ -167,20 +164,6 @@ public static Optional getAutoWatermarkInterval(Properties properties) { return StringUtils.isNotBlank(autoWatermarkInterval) ? Optional.of(Long.valueOf(autoWatermarkInterval)) : Optional.empty(); } - public static Optional isRestore(Properties properties){ - String restoreEnable = properties.getProperty(ConfigConstrant.RESTOREENABLE, "true"); - return Optional.of(Boolean.valueOf(restoreEnable)); - } - - public static Optional getDelayInterval(Properties properties){ - String delayInterval = properties.getProperty(ConfigConstrant.DELAYINTERVAL, "10"); - return Optional.of(Integer.valueOf(delayInterval)); - } - public static Optional getFailureInterval(Properties properties){ - String failureInterval = properties.getProperty(ConfigConstrant.FAILUREINTERVAL, "6"); - return Optional.of(Integer.valueOf(failureInterval)); - } - /** * #ProcessingTime(默认), IngestionTime, EventTime * @param properties @@ -276,8 +259,6 @@ private static Optional createStateBackend(String backendType, Str checkpointDataUriEmptyCheck(checkpointDataUri, backendType); stateBackend = new RocksDBStateBackend(checkpointDataUri, BooleanUtils.toBoolean(backendIncremental)); break; - default: - break; } return stateBackend == null ? Optional.empty() : Optional.of(stateBackend); } @@ -337,14 +318,14 @@ private static void verityTtl(String ttlMintimeStr, String ttlMaxtimeStr) { * @return */ private static Long getTtlTime(Integer timeNumber, String timeUnit) { - if ("d".equalsIgnoreCase(timeUnit)) { - return timeNumber * 1000L * 60 * 60 * 24; - } else if ("h".equalsIgnoreCase(timeUnit)) { - return timeNumber * 1000L * 60 * 60; - } else if ("m".equalsIgnoreCase(timeUnit)) { - return timeNumber * 1000L * 60; - } else if ("s".equalsIgnoreCase(timeUnit)) { - return timeNumber * 1000L; + if (timeUnit.equalsIgnoreCase("d")) { + return timeNumber * 1000l * 60 * 60 * 24; + } else if (timeUnit.equalsIgnoreCase("h")) { + return timeNumber * 1000l * 60 * 60; + } else if (timeUnit.equalsIgnoreCase("m")) { + return timeNumber * 1000l * 60; + } else if (timeUnit.equalsIgnoreCase("s")) { + return timeNumber * 1000l; } else { throw new RuntimeException("not support " + timeNumber + timeUnit); } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 8b0c9ec10..823cd5143 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -18,13 +18,7 @@ package com.dtstack.flink.sql.exec; -import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; -import com.dtstack.flink.sql.parser.CreateFuncParser; -import com.dtstack.flink.sql.parser.CreateTmpTableParser; -import com.dtstack.flink.sql.parser.FlinkPlanner; -import com.dtstack.flink.sql.parser.InsertSqlParser; -import com.dtstack.flink.sql.parser.SqlParser; -import com.dtstack.flink.sql.parser.SqlTree; +import com.dtstack.flink.sql.parser.*; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -39,7 +33,6 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.constrant.ConfigConstrant; import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.enums.EPluginLoadMode; @@ -49,12 +42,12 @@ import com.dtstack.flink.sql.option.OptionParser; import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.side.SideSqlExec; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.sink.StreamSinkFactory; import com.dtstack.flink.sql.source.StreamSourceFactory; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; -import com.dtstack.flink.sql.table.AbstractTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import com.dtstack.flink.sql.watermarker.WaterMarkerAssigner; @@ -97,8 +90,11 @@ public class ExecuteProcessHelper { public static ParamsInfo parseParams(String[] args) throws Exception { LOG.info("------------program params-------------------------"); + System.out.println("------------program params-------------------------"); Arrays.stream(args).forEach(arg -> LOG.info("{}", arg)); + Arrays.stream(args).forEach(System.out::println); LOG.info("-------------------------------------------"); + System.out.println("----------------------------------------"); OptionParser optionParser = new OptionParser(args); Options options = optionParser.getOptions(); @@ -109,14 +105,13 @@ public static ParamsInfo parseParams(String[] args) throws Exception { String remoteSqlPluginPath = options.getRemoteSqlPluginPath(); String pluginLoadMode = options.getPluginLoadMode(); String deployMode = options.getMode(); - String logLevel = options.getLogLevel(); Preconditions.checkArgument(checkRemoteSqlPluginPath(remoteSqlPluginPath, deployMode, pluginLoadMode), "Non-local mode or shipfile deployment mode, remoteSqlPluginPath is required"); String confProp = URLDecoder.decode(options.getConfProp(), Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - List jarUrlList = getExternalJarUrls(options.getAddjar()); + List jarURList = getExternalJarUrls(options.getAddjar()); return ParamsInfo.builder() .setSql(sql) @@ -126,7 +121,7 @@ public static ParamsInfo parseParams(String[] args) throws Exception { .setPluginLoadMode(pluginLoadMode) .setDeployMode(deployMode) .setConfProp(confProperties) - .setJarUrlList(jarUrlList) + .setJarUrlList(jarURList) .build(); } @@ -155,7 +150,7 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf SqlParser.setLocalSqlPluginRoot(paramsInfo.getLocalSqlPluginPath()); SqlTree sqlTree = SqlParser.parseSql(paramsInfo.getSql()); - Map sideTableMap = Maps.newHashMap(); + Map sideTableMap = Maps.newHashMap(); Map registerTableCache = Maps.newHashMap(); //register udf @@ -188,14 +183,14 @@ public static List getExternalJarUrls(String addJarListStr) throws java.io. } return jarUrlList; } - + private static void sqlTranslation(String localSqlPluginPath, - StreamTableEnvironment tableEnv, - SqlTree sqlTree,Map sideTableMap, - Map registerTableCache, - StreamQueryConfig queryConfig) throws Exception { + StreamTableEnvironment tableEnv, + SqlTree sqlTree,Map sideTableMap, + Map registerTableCache, + StreamQueryConfig queryConfig) throws Exception { - SideSqlExec sideSqlExec = new SideSqlExec(); + SideSqlExec sideSqlExec = new SideSqlExec(); sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, result); @@ -227,8 +222,9 @@ private static void sqlTranslation(String localSqlPluginPath, //sql-dimensional table contains the dimension table of execution sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null); } else { - LOG.info("----------exec sql without dimension join-----------"); - LOG.info("----------real sql exec is--------------------------\n{}", result.getExecSql()); + System.out.println("----------exec sql without dimension join-----------"); + System.out.println("----------real sql exec is--------------------------"); + System.out.println(result.getExecSql()); FlinkSQLExec.sqlUpdate(tableEnv, result.getExecSql(), queryConfig); if (LOG.isInfoEnabled()) { LOG.info("exec sql: " + result.getExecSql()); @@ -268,14 +264,14 @@ public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrl * @throws Exception */ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { + String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { Set pluginClassPatshSets = Sets.newHashSet(); WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); - for (AbstractTableInfo tableInfo : sqlTree.getTableInfoMap().values()) { + for (TableInfo tableInfo : sqlTree.getTableInfoMap().values()) { - if (tableInfo instanceof AbstractSourceTableInfo) { + if (tableInfo instanceof SourceTableInfo) { - AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; + SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; Table table = StreamSourceFactory.getStreamSource(sourceTableInfo, env, tableEnv, localSqlPluginPath); tableEnv.registerTable(sourceTableInfo.getAdaptName(), table); //Note --- parameter conversion function can not be used inside a function of the type of polymerization @@ -306,21 +302,21 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment } registerTableCache.put(tableInfo.getName(), regTable); - URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractSourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), SourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sourceTablePathUrl); - } else if (tableInfo instanceof AbstractTargetTableInfo) { + } else if (tableInfo instanceof TargetTableInfo) { - TableSink tableSink = StreamSinkFactory.getTableSink((AbstractTargetTableInfo) tableInfo, localSqlPluginPath); + TableSink tableSink = StreamSinkFactory.getTableSink((TargetTableInfo) tableInfo, localSqlPluginPath); TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); - URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), TargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sinkTablePathUrl); - } else if (tableInfo instanceof AbstractSideTableInfo) { - String sideOperator = ECacheType.ALL.name().equals(((AbstractSideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; - sideTableMap.put(tableInfo.getName(), (AbstractSideTableInfo) tableInfo); + } else if (tableInfo instanceof SideTableInfo) { + String sideOperator = ECacheType.ALL.name().equals(((SideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; + sideTableMap.put(tableInfo.getName(), (SideTableInfo) tableInfo); - URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, AbstractSideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, SideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sideTablePathUrl); } else { throw new RuntimeException("not support table type:" + tableInfo.getType()); @@ -353,12 +349,4 @@ public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperti } - public static void setLogLevel(ParamsInfo paramsInfo){ - String logLevel = paramsInfo.getConfProp().getProperty(ConfigConstrant.LOG_LEVEL_KEY); - if(org.apache.commons.lang3.StringUtils.isBlank(logLevel)){ - return; - } - ChangeLogLevelProcess logLevelProcess = new ChangeLogLevelProcess(); - logLevelProcess.process(logLevel); - } } \ No newline at end of file diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java index 27cc7702d..b36dc31eb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java @@ -84,6 +84,7 @@ public Properties getConfProp() { return confProp; } + @Override public String toString() { return "ParamsInfo{" + diff --git a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java index 8802198a0..3a5af18b1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java @@ -24,7 +24,6 @@ import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.Meter; import org.apache.flink.metrics.MeterView; -import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; @@ -35,11 +34,11 @@ * author: toutian * create: 2019/12/24 */ -public class SerializationMetricWrapper implements SerializationSchema { +public class SerializationMetricWrapper implements SerializationSchema { private static final long serialVersionUID = 1L; - private SerializationSchema serializationSchema; + private SerializationSchema serializationSchema; private transient RuntimeContext runtimeContext; @@ -48,7 +47,7 @@ public class SerializationMetricWrapper implements SerializationSchema { protected transient Meter dtNumRecordsOutRate; - public SerializationMetricWrapper(SerializationSchema serializationSchema) { + public SerializationMetricWrapper(SerializationSchema serializationSchema) { this.serializationSchema = serializationSchema; } @@ -58,7 +57,7 @@ public void initMetric() { } @Override - public byte[] serialize(CRow element) { + public byte[] serialize(Row element) { beforeSerialize(); byte[] row = serializationSchema.serialize(element); afterSerialize(); @@ -80,7 +79,7 @@ public void setRuntimeContext(RuntimeContext runtimeContext) { this.runtimeContext = runtimeContext; } - public SerializationSchema getSerializationSchema() { + public SerializationSchema getSerializationSchema() { return serializationSchema; } diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 5f1b1c6f3..78b579305 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.format.dtnest; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.google.common.base.Strings; import com.google.common.collect.Maps; import org.apache.flink.api.common.serialization.AbstractDeserializationSchema; @@ -28,9 +28,8 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.*; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.NullNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.types.Row; import java.io.IOException; @@ -58,9 +57,9 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final String[] fieldNames; private final TypeInformation[] fieldTypes; - private List fieldExtraInfos; + private List fieldExtraInfos; - public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { + public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); this.rowAndFieldMapping = rowAndFieldMapping; @@ -76,7 +75,7 @@ public Row deserialize(byte[] message) throws IOException { try { for (int i = 0; i < fieldNames.length; i++) { JsonNode node = getIgnoreCase(fieldNames[i]); - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); + TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); if (node == null) { if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java index 0fd057c5a..43f599d14 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java @@ -31,10 +31,7 @@ import java.io.File; import java.io.FileInputStream; import java.net.URLEncoder; -import java.util.stream.Stream; - import org.apache.commons.codec.Charsets; -import org.apache.flink.util.FileUtils; /** @@ -95,16 +92,19 @@ public Options getOptions(){ } public List getProgramExeArgList() throws Exception { - Map mapConf = PluginUtil.objectToMap(properties); + Map mapConf = PluginUtil.ObjectToMap(properties); List args = Lists.newArrayList(); - for (Map.Entry one : mapConf.entrySet()) { + for(Map.Entry one : mapConf.entrySet()){ String key = one.getKey(); Object value = one.getValue(); - if (value == null) { + if(value == null){ continue; - } else if (OPTION_SQL.equalsIgnoreCase(key)) { + }else if(OPTION_SQL.equalsIgnoreCase(key)){ File file = new File(value.toString()); - String content = FileUtils.readFile(file, "UTF-8"); + FileInputStream in = new FileInputStream(file); + byte[] filecontent = new byte[(int) file.length()]; + in.read(filecontent); + String content = new String(filecontent, Charsets.UTF_8.name()); value = URLEncoder.encode(content, Charsets.UTF_8.name()); } args.add("-" + key); @@ -112,4 +112,9 @@ public List getProgramExeArgList() throws Exception { } return args; } + + public static void main(String[] args) throws Exception { + OptionParser OptionParser = new OptionParser(args); + System.out.println(OptionParser.getOptions()); + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java b/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java similarity index 95% rename from core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java rename to core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java index fbcc86bbd..1fc40c13b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java +++ b/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java @@ -27,7 +27,7 @@ * extend RichOutputFormat with metric 'dtNumRecordsOut', 'dtNumDirtyRecordsOut', 'dtNumRecordsOutRate' * Created by sishu.yss on 2018/11/28. */ -public abstract class AbstractDtRichOutputFormat extends RichOutputFormat{ +public abstract class DtRichOutputFormat extends RichOutputFormat{ public transient Counter outRecords; public transient Counter outDirtyRecords; diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java index fc6a7e16c..670d98a7e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java @@ -32,18 +32,18 @@ public class CreateFuncParser implements IParser { - private static final String FUNC_PATTERN_STR = "(?i)\\s*create\\s+(scala|table|aggregate)\\s+function\\s+(\\S+)\\s+WITH\\s+(\\S+)"; + private static final String funcPatternStr = "(?i)\\s*create\\s+(scala|table|aggregate)\\s+function\\s+(\\S+)\\s+WITH\\s+(\\S+)"; - private static final Pattern FUNC_PATTERN = Pattern.compile(FUNC_PATTERN_STR); + private static final Pattern funcPattern = Pattern.compile(funcPatternStr); @Override public boolean verify(String sql) { - return FUNC_PATTERN.matcher(sql).find(); + return funcPattern.matcher(sql).find(); } @Override public void parseSql(String sql, SqlTree sqlTree) { - Matcher matcher = FUNC_PATTERN.matcher(sql); + Matcher matcher = funcPattern.matcher(sql); if(matcher.find()){ String type = matcher.group(1); String funcName = matcher.group(2); diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index 2afc76c48..a76c1b31a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.parser; import com.dtstack.flink.sql.enums.ETableType; -import com.dtstack.flink.sql.table.AbstractTableInfo; -import com.dtstack.flink.sql.table.AbstractTableInfoParser; +import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.TableInfoParser; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import com.google.common.collect.Lists; @@ -75,7 +75,7 @@ public static SqlTree parseSql(String sql) throws Exception { List sqlArr = DtStringUtil.splitIgnoreQuota(sql, SQL_DELIMITER); SqlTree sqlTree = new SqlTree(); - AbstractTableInfoParser tableInfoParser = new AbstractTableInfoParser(); + TableInfoParser tableInfoParser = new TableInfoParser(); for(String childSql : sqlArr){ if(Strings.isNullOrEmpty(childSql)){ continue; @@ -113,7 +113,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } - AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), + TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } @@ -126,7 +126,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } - AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SINK.getType(), + TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SINK.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } @@ -144,7 +144,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } } else { - AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), + TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java index 5252ee022..1b64b7c68 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.parser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.google.common.collect.Maps; import com.google.common.collect.Lists; @@ -41,7 +41,7 @@ public class SqlTree { private Map preDealTableMap = Maps.newHashMap(); - private Map tableInfoMap = Maps.newLinkedHashMap(); + private Map tableInfoMap = Maps.newLinkedHashMap(); private List execSqlList = Lists.newArrayList(); @@ -89,11 +89,11 @@ public List getTmpSqlList(){ return tmpSqlList; } - public Map getTableInfoMap() { + public Map getTableInfoMap() { return tableInfoMap; } - public void addTableInfo(String tableName, AbstractTableInfo tableInfo){ + public void addTableInfo(String tableName, TableInfo tableInfo){ tableInfoMap.put(tableName, tableInfo); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java similarity index 79% rename from core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java rename to core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java index 3e3d0a8fb..64fd844b1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java @@ -25,15 +25,13 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.dtstack.flink.sql.factory.DTThreadFactory; import org.apache.calcite.sql.JoinType; import java.sql.SQLException; +import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; /** @@ -43,15 +41,13 @@ * @author xuchao */ -public abstract class BaseAllReqRow extends RichFlatMapFunction implements ISideReqRow { +public abstract class AllReqRow extends RichFlatMapFunction implements ISideReqRow { - private static final Logger LOG = LoggerFactory.getLogger(BaseAllReqRow.class); - - protected BaseSideInfo sideInfo; + protected SideInfo sideInfo; private ScheduledExecutorService es; - public BaseAllReqRow(BaseSideInfo sideInfo){ + public AllReqRow(SideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -64,11 +60,11 @@ public BaseAllReqRow(BaseSideInfo sideInfo){ public void open(Configuration parameters) throws Exception { super.open(parameters); initCache(); - LOG.info("----- all cacheRef init end-----"); + System.out.println("----- all cacheRef init end-----"); //start reload cache thread - AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); - es = new ScheduledThreadPoolExecutor(1,new DTThreadFactory("cache-all-reload")); + SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + es = Executors.newSingleThreadScheduledExecutor(new DTThreadFactory("cache-all-reload")); es.scheduleAtFixedRate(() -> reloadCache(), sideTableInfo.getCacheTimeout(), sideTableInfo.getCacheTimeout(), TimeUnit.MILLISECONDS); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java similarity index 89% rename from core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java rename to core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index e5bce4437..fae0ba9aa 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.metric.MetricConstant; -import com.dtstack.flink.sql.side.cache.AbstractSideCache; +import com.dtstack.flink.sql.side.cache.AbsSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cache.LRUSideCache; import org.apache.calcite.sql.JoinType; @@ -30,12 +30,14 @@ import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; +import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collections; +import java.util.concurrent.TimeoutException; /** * All interfaces inherit naming rules: type + "AsyncReqRow" such as == "MysqlAsyncReqRow @@ -45,17 +47,17 @@ * @author xuchao */ -public abstract class BaseAsyncReqRow extends RichAsyncFunction implements ISideReqRow { - private static final Logger LOG = LoggerFactory.getLogger(BaseAsyncReqRow.class); +public abstract class AsyncReqRow extends RichAsyncFunction implements ISideReqRow { + private static final Logger LOG = LoggerFactory.getLogger(AsyncReqRow.class); private static final long serialVersionUID = 2098635244857937717L; private static int TIMEOUT_LOG_FLUSH_NUM = 10; private int timeOutNum = 0; - protected BaseSideInfo sideInfo; + protected SideInfo sideInfo; protected transient Counter parseErrorRecords; - public BaseAsyncReqRow(BaseSideInfo sideInfo){ + public AsyncReqRow(SideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -67,12 +69,12 @@ public void open(Configuration parameters) throws Exception { } private void initCache(){ - AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); if(sideTableInfo.getCacheType() == null || ECacheType.NONE.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ return; } - AbstractSideCache sideCache; + AbsSideCache sideCache; if(ECacheType.LRU.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ sideCache = new LRUSideCache(sideTableInfo); sideInfo.setSideCache(sideCache); @@ -121,10 +123,10 @@ protected void dealCacheData(String key, CacheObj missKeyObj) { @Override public void timeout(CRow input, ResultFuture resultFuture) throws Exception { + //TODO 需要添加数据指标 if(timeOutNum % TIMEOUT_LOG_FLUSH_NUM == 0){ LOG.info("Async function call has timed out. input:{}, timeOutNum:{}",input.toString(), timeOutNum); } - timeOutNum ++; if(timeOutNum > sideInfo.getSideTableInfo().getAsyncTimeoutNumLimit()){ resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index e38c263e2..f072e2591 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -22,7 +22,6 @@ import com.dtstack.flink.sql.parser.FlinkPlanner; import com.dtstack.flink.sql.util.ParseUtils; import com.dtstack.flink.sql.util.TableUtils; -import com.esotericsoftware.minlog.Log; import com.google.common.base.Preconditions; import com.google.common.collect.*; import org.apache.calcite.sql.JoinType; @@ -440,18 +439,18 @@ private void extractTemporaryQuery(SqlNode node, String tableAlias, tableRef.put(tbTmp, tableAlias); } - Log.info("-------build temporary query-----------\n{}", tmpSelectSql); - Log.info("---------------------------------------"); + System.out.println("-------build temporary query-----------"); + System.out.println(tmpSelectSql); + System.out.println("---------------------------------------"); }catch (Exception e){ - Log.error("", e); + e.printStackTrace(); throw new RuntimeException(e); } } /** * 抽取上层需用使用到的字段 - * 由于where字段已经抽取到上一层了所以不用查询出来 * @param parentSelectList * @param fromTableNameSet * @return @@ -593,6 +592,7 @@ private void extractSelectField(SqlNode selectNode, } }else if(selectNode.getKind() == CASE){ + System.out.println("selectNode"); SqlCase sqlCase = (SqlCase) selectNode; SqlNodeList whenOperands = sqlCase.getWhenOperands(); SqlNodeList thenOperands = sqlCase.getThenOperands(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java b/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java index d9d79f34e..738e0f84e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java @@ -80,7 +80,6 @@ public static List getRowTypeInfo(SqlNode sqlNode, JoinScope scope, b switch(identifierSize) { case 1: fieldInfoList.addAll(getAllField(scope)); - break; default: SqlIdentifier tableIdentify = identifier.skipLast(1); JoinScope.ScopeChild scopeChild = scope.getScope(tableIdentify.getSimple()); @@ -100,7 +99,6 @@ public static List getRowTypeInfo(SqlNode sqlNode, JoinScope scope, b fieldInfo.setTypeInformation(type); fieldInfoList.add(fieldInfo); } - break; } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java similarity index 92% rename from core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java rename to core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java index 3b2faa713..029c86e25 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.side.cache.AbstractSideCache; +import com.dtstack.flink.sql.side.cache.AbsSideCache; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; @@ -41,7 +41,7 @@ * @author xuchao */ -public abstract class BaseSideInfo implements Serializable{ +public abstract class SideInfo implements Serializable{ protected RowTypeInfo rowTypeInfo; @@ -68,12 +68,12 @@ public abstract class BaseSideInfo implements Serializable{ //key:Returns the value of the position, value: the ref field name​in the side table protected Map sideFieldNameIndex = Maps.newHashMap(); - protected AbstractSideTableInfo sideTableInfo; + protected SideTableInfo sideTableInfo; - protected AbstractSideCache sideCache; + protected AbsSideCache sideCache; - public BaseSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, - AbstractSideTableInfo sideTableInfo){ + public SideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, + SideTableInfo sideTableInfo){ this.rowTypeInfo = rowTypeInfo; this.outFieldInfoList = outFieldInfoList; this.joinType = joinInfo.getJoinType(); @@ -172,7 +172,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName){ } } - public abstract void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo); + public abstract void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo); public RowTypeInfo getRowTypeInfo() { return rowTypeInfo; @@ -246,19 +246,19 @@ public void setSideFieldIndex(Map sideFieldIndex) { this.sideFieldIndex = sideFieldIndex; } - public AbstractSideTableInfo getSideTableInfo() { + public SideTableInfo getSideTableInfo() { return sideTableInfo; } - public void setSideTableInfo(AbstractSideTableInfo sideTableInfo) { + public void setSideTableInfo(SideTableInfo sideTableInfo) { this.sideTableInfo = sideTableInfo; } - public AbstractSideCache getSideCache() { + public AbsSideCache getSideCache() { return sideCache; } - public void setSideCache(AbstractSideCache sideCache) { + public void setSideCache(AbsSideCache sideCache) { this.sideCache = sideCache; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java index 4524145db..0902bf39f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java @@ -45,7 +45,7 @@ * @author maqi */ public class SidePredicatesParser { - public void fillPredicatesForSideTable(String exeSql, Map sideTableMap) throws SqlParseException { + public void fillPredicatesForSideTable(String exeSql, Map sideTableMap) throws SqlParseException { FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); SqlNode sqlNode = flinkPlanner.parse(exeSql); parseSql(sqlNode, sideTableMap, Maps.newHashMap()); @@ -57,7 +57,7 @@ public void fillPredicatesForSideTable(String exeSql, Map sideTableMap, Map tabMapping) { + private void parseSql(SqlNode sqlNode, Map sideTableMap, Map tabMapping) { SqlKind sqlKind = sqlNode.getKind(); switch (sqlKind) { case INSERT: @@ -100,12 +100,10 @@ private void parseSql(SqlNode sqlNode, Map sideTa parseSql(unionLeft, sideTableMap, tabMapping); parseSql(unionRight, sideTableMap, tabMapping); break; - default: - break; } } - private void fillToSideTableInfo(Map sideTableMap, Map tabMapping, List predicateInfoList) { + private void fillToSideTableInfo(Map sideTableMap, Map tabMapping, List predicateInfoList) { predicateInfoList.stream().filter(info -> sideTableMap.containsKey(tabMapping.getOrDefault(info.getOwnerTable(), info.getOwnerTable()))) .map(info -> sideTableMap.get(tabMapping.getOrDefault(info.getOwnerTable(), info.getOwnerTable())).getPredicateInfoes().add(info)) .count(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index 2296f64c5..f37c3f78c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -47,8 +47,7 @@ import java.util.Queue; import java.util.Set; -import static org.apache.calcite.sql.SqlKind.IDENTIFIER; -import static org.apache.calcite.sql.SqlKind.LITERAL; +import static org.apache.calcite.sql.SqlKind.*; /** * Parsing sql, obtain execution information dimension table @@ -63,6 +62,8 @@ public class SideSQLParser { private Map localTableCache = Maps.newHashMap(); public Queue getExeQueue(String exeSql, Set sideTableSet) throws SqlParseException { + System.out.println("----------exec original Sql----------"); + System.out.println(exeSql); LOG.info("----------exec original Sql----------"); LOG.info(exeSql); @@ -167,8 +168,6 @@ public Object parseSql(SqlNode sqlNode, case LITERAL: return LITERAL.toString(); - default: - break; } return ""; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index c41e2264a..f90138b2a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -43,17 +43,24 @@ import com.dtstack.flink.sql.util.ParseUtils; import com.dtstack.flink.sql.util.TableUtils; import com.google.common.base.Preconditions; -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import com.google.common.collect.*; +import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlInsert; +import org.apache.calcite.sql.SqlJoin; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlWithItem; +import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -61,6 +68,7 @@ import java.sql.Timestamp; import java.util.Arrays; +import java.util.Collection; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -89,7 +97,7 @@ public class SideSqlExec { private Map localTableCache = Maps.newHashMap(); public void exec(String sql, - Map sideTableMap, + Map sideTableMap, StreamTableEnvironment tableEnv, Map tableCache, StreamQueryConfig queryConfig, @@ -123,9 +131,11 @@ public void exec(String sql, if(pollSqlNode.getKind() == INSERT){ + System.out.println("----------real exec sql-----------" ); + System.out.println(pollSqlNode.toString()); FlinkSQLExec.sqlUpdate(tableEnv, pollSqlNode.toString(), queryConfig); if(LOG.isInfoEnabled()){ - LOG.info("----------real exec sql-----------\n{}", pollSqlNode.toString()); + LOG.info("exec sql: " + pollSqlNode.toString()); } }else if(pollSqlNode.getKind() == AS){ @@ -156,7 +166,8 @@ public void exec(String sql, } }else if (pollObj instanceof JoinInfo){ - LOG.info("----------exec join info----------\n{}", pollObj.toString()); + System.out.println("----------exec join info----------"); + System.out.println(pollObj.toString()); joinFun(pollObj, localTableCache, sideTableMap, tableEnv); } } @@ -283,7 +294,7 @@ private Table getTableFromCache(Map localTableCache, String table * * @return */ - private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, AbstractSideTableInfo sideTableInfo) { + private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, SideTableInfo sideTableInfo) { List conditionFields = getConditionFields(conditionNode, sideTableAlias, sideTableInfo); if(CollectionUtils.isEqualCollection(conditionFields, convertPrimaryAlias(sideTableInfo))){ return true; @@ -291,7 +302,7 @@ private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, return false; } - private List convertPrimaryAlias(AbstractSideTableInfo sideTableInfo) { + private List convertPrimaryAlias(SideTableInfo sideTableInfo) { List res = Lists.newArrayList(); sideTableInfo.getPrimaryKeys().forEach(field -> { res.add(sideTableInfo.getPhysicalFields().getOrDefault(field, field)); @@ -299,7 +310,7 @@ private List convertPrimaryAlias(AbstractSideTableInfo sideTableInfo) { return res; } - public List getConditionFields(SqlNode conditionNode, String specifyTableName, AbstractSideTableInfo sideTableInfo){ + public List getConditionFields(SqlNode conditionNode, String specifyTableName, SideTableInfo sideTableInfo){ List sqlNodeList = Lists.newArrayList(); ParseUtils.parseAnd(conditionNode, sqlNodeList); List conditionFields = Lists.newArrayList(); @@ -358,7 +369,7 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, private void joinFun(Object pollObj, Map localTableCache, - Map sideTableMap, + Map sideTableMap, StreamTableEnvironment tableEnv) throws Exception{ JoinInfo joinInfo = (JoinInfo) pollObj; @@ -374,7 +385,7 @@ private void joinFun(Object pollObj, JoinScope.ScopeChild rightScopeChild = new JoinScope.ScopeChild(); rightScopeChild.setAlias(joinInfo.getRightTableAlias()); rightScopeChild.setTableName(joinInfo.getRightTableName()); - AbstractSideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); + SideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); if(sideTableInfo == null){ sideTableInfo = sideTableMap.get(joinInfo.getRightTableAlias()); } @@ -383,9 +394,9 @@ private void joinFun(Object pollObj, throw new RuntimeException("can't not find side table:" + joinInfo.getRightTableName()); } - if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ - throw new RuntimeException("ON condition must contain all equal fields!!!"); - } +// if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ +// throw new RuntimeException("ON condition must contain all equal fields!!!"); +// } rightScopeChild.setRowTypeInfo(sideTableInfo.getRowTypeInfo()); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java similarity index 79% rename from core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java index 7a832d0a1..8c78578fb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.google.common.collect.Lists; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ * @author xuchao */ -public abstract class AbstractSideTableInfo extends AbstractTableInfo implements Serializable { +public abstract class SideTableInfo extends TableInfo implements Serializable { public static final String TARGET_SUFFIX = "Side"; @@ -55,8 +55,6 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements public static final String ASYNC_TIMEOUT_NUM_KEY = "asyncTimeoutNum"; - public static final String ASYNC_REQ_POOL_KEY = "asyncPoolSize"; - private String cacheType = "none";//None or LRU or ALL private int cacheSize = 10000; @@ -67,11 +65,6 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements private int asyncTimeout=10000; - /** - * async operator req outside conn pool size, egg rdb conn pool size - */ - private int asyncPoolSize = 0; - private int asyncTimeoutNumLimit = Integer.MAX_VALUE; private boolean partitionedJoin = false; @@ -163,26 +156,4 @@ public void setAsyncTimeoutNumLimit(int asyncTimeoutNumLimit) { this.asyncTimeoutNumLimit = asyncTimeoutNumLimit; } - public int getAsyncPoolSize() { - return asyncPoolSize; - } - - public void setAsyncPoolSize(int asyncPoolSize) { - this.asyncPoolSize = asyncPoolSize; - } - - @Override - public String toString() { - return "Cache Info{" + - "cacheType='" + cacheType + '\'' + - ", cacheSize=" + cacheSize + - ", cacheTimeout=" + cacheTimeout + - ", asyncCapacity=" + asyncCapacity + - ", asyncTimeout=" + asyncTimeout + - ", asyncPoolSize=" + asyncPoolSize + - ", asyncTimeoutNumLimit=" + asyncTimeoutNumLimit + - ", partitionedJoin=" + partitionedJoin + - ", cacheMode='" + cacheMode + '\'' + - '}'; - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java b/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java index 583e4597b..8417c4519 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java @@ -21,8 +21,8 @@ import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.enums.ECacheType; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.AbsTableParser; import com.dtstack.flink.sql.util.PluginUtil; /** @@ -37,7 +37,7 @@ public class StreamSideFactory { private static final String CURR_TYPE = "side"; - public static AbstractTableParser getSqlParser(String pluginType, String sqlRootDir, String cacheType) throws Exception { + public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir, String cacheType) throws Exception { String sideOperator = ECacheType.ALL.name().equalsIgnoreCase(cacheType) ? "all" : "async"; String pluginJarPath = PluginUtil.getSideJarFileDirPath(pluginType, sideOperator, "side", sqlRootDir); @@ -45,10 +45,10 @@ public static AbstractTableParser getSqlParser(String pluginType, String sqlRoot return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class sideParser = cl.loadClass(className); - if (!AbstractSideTableParser.class.isAssignableFrom(sideParser)) { + if (!AbsSideTableParser.class.isAssignableFrom(sideParser)) { throw new RuntimeException("class " + sideParser.getName() + " not subClass of AbsSideTableParser"); } - return sideParser.asSubclass(AbstractTableParser.class).newInstance(); + return sideParser.asSubclass(AbsTableParser.class).newInstance(); }); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java b/core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java similarity index 84% rename from core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java rename to core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java index 58832c583..757f91600 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.cache; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; /** * Reason: @@ -30,11 +30,11 @@ * @author xuchao */ -public abstract class AbstractSideCache { +public abstract class AbsSideCache { - protected AbstractSideTableInfo sideTableInfo; + protected SideTableInfo sideTableInfo; - public AbstractSideCache(AbstractSideTableInfo sideTableInfo){ + public AbsSideCache(SideTableInfo sideTableInfo){ this.sideTableInfo = sideTableInfo; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java b/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java index 2664efd34..700e13bb2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.cache; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -33,11 +33,11 @@ * @author xuchao */ -public class LRUSideCache extends AbstractSideCache { +public class LRUSideCache extends AbsSideCache{ protected transient Cache cache; - public LRUSideCache(AbstractSideTableInfo sideTableInfo) { + public LRUSideCache(SideTableInfo sideTableInfo) { super(sideTableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java index 3c2010907..290804200 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java @@ -20,10 +20,10 @@ package com.dtstack.flink.sql.side.operator; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.AsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.AsyncDataStream; @@ -49,20 +49,20 @@ public class SideAsyncOperator { private static final String ORDERED = "ordered"; - private static BaseAsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, - JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { + private static AsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { String pathOfType = String.format(PATH_FORMAT, sideType); String pluginJarPath = PluginUtil.getJarFileDirPath(pathOfType, sqlRootDir); String className = PluginUtil.getSqlSideClassName(sideType, "side", OPERATOR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> - cl.loadClass(className).asSubclass(BaseAsyncReqRow.class) - .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, AbstractSideTableInfo.class) + cl.loadClass(className).asSubclass(AsyncReqRow.class) + .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class) .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, - List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { - BaseAsyncReqRow asyncDbReq = loadAsyncReq(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { + AsyncReqRow asyncDbReq = loadAsyncReq(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); //TODO How much should be set for the degree of parallelism? Timeout? capacity settings? if (ORDERED.equals(sideTableInfo.getCacheMode())){ diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java index 6b6f9fe1b..5aa810b0f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java @@ -20,10 +20,10 @@ package com.dtstack.flink.sql.side.operator; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.side.BaseAllReqRow; +import com.dtstack.flink.sql.side.AllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; @@ -44,22 +44,22 @@ public class SideWithAllCacheOperator { private static final String OPERATOR_TYPE = "All"; - private static BaseAllReqRow loadFlatMap(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, - JoinInfo joinInfo, List outFieldInfoList, - AbstractSideTableInfo sideTableInfo) throws Exception { + private static AllReqRow loadFlatMap(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, List outFieldInfoList, + SideTableInfo sideTableInfo) throws Exception { String pathOfType = String.format(PATH_FORMAT, sideType); String pluginJarPath = PluginUtil.getJarFileDirPath(pathOfType, sqlRootDir); String className = PluginUtil.getSqlSideClassName(sideType, "side", OPERATOR_TYPE); - return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> cl.loadClass(className).asSubclass(BaseAllReqRow.class) - .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, AbstractSideTableInfo.class) + return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> cl.loadClass(className).asSubclass(AllReqRow.class) + .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class) .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, - List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { - BaseAllReqRow allReqRow = loadFlatMap(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { + AllReqRow allReqRow = loadFlatMap(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); return inputStream.flatMap(allReqRow); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java b/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java index e29421369..3cfd48f1b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; /** * Reason: @@ -30,5 +30,5 @@ */ public interface IStreamSinkGener { - T genStreamSink(AbstractTargetTableInfo targetTableInfo); + T genStreamSink(TargetTableInfo targetTableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java b/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java index 89061db3a..53460081d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.sink; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.table.sinks.TableSink; @@ -40,21 +40,21 @@ public class StreamSinkFactory { private static final String DIR_NAME_FORMAT = "%ssink"; - public static AbstractTableParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { + public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { String pluginJarPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), sqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); String className = PluginUtil.getSqlParserClassName(typeNoVersion, CURR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class targetParser = cl.loadClass(className); - if(!AbstractTableParser.class.isAssignableFrom(targetParser)){ + if(!AbsTableParser.class.isAssignableFrom(targetParser)){ throw new RuntimeException("class " + targetParser.getName() + " not subClass of AbsTableParser"); } - return targetParser.asSubclass(AbstractTableParser.class).newInstance(); + return targetParser.asSubclass(AbsTableParser.class).newInstance(); }); } - public static TableSink getTableSink(AbstractTargetTableInfo targetTableInfo, String localSqlRootDir) throws Exception { + public static TableSink getTableSink(TargetTableInfo targetTableInfo, String localSqlRootDir) throws Exception { String pluginType = targetTableInfo.getType(); String pluginJarDirPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), localSqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); diff --git a/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java b/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java index fde5a235d..b8b8f6edc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java +++ b/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.source; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.java.StreamTableEnvironment; @@ -38,6 +38,6 @@ public interface IStreamSourceGener { * @param tableEnv * @return */ - T genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv); + T genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv); } diff --git a/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java b/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java index e0cec1415..1057fb0ed 100644 --- a/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.table.AbstractSourceParser; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.AbsSourceParser; +import com.dtstack.flink.sql.table.SourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -43,17 +43,17 @@ public class StreamSourceFactory { private static final String DIR_NAME_FORMAT = "%ssource"; - public static AbstractSourceParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { + public static AbsSourceParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { String pluginJarPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), sqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); String className = PluginUtil.getSqlParserClassName(typeNoVersion, CURR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class sourceParser = cl.loadClass(className); - if(!AbstractSourceParser.class.isAssignableFrom(sourceParser)){ + if(!AbsSourceParser.class.isAssignableFrom(sourceParser)){ throw new RuntimeException("class " + sourceParser.getName() + " not subClass of AbsSourceParser"); } - return sourceParser.asSubclass(AbstractSourceParser.class).newInstance(); + return sourceParser.asSubclass(AbsSourceParser.class).newInstance(); }); } @@ -62,7 +62,7 @@ public static AbstractSourceParser getSqlParser(String pluginType, String sqlRoo * @param sourceTableInfo * @return */ - public static Table getStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, + public static Table getStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String sqlRootDir) throws Exception { String sourceTypeStr = sourceTableInfo.getType(); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java similarity index 62% rename from core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java index 4973a891e..bf0b8a7a4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java @@ -21,10 +21,8 @@ package com.dtstack.flink.sql.table; import com.dtstack.flink.sql.enums.ECacheType; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.MathUtil; -import org.apache.flink.util.Preconditions; - import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -36,24 +34,24 @@ * @author xuchao */ -public abstract class AbstractSideTableParser extends AbstractTableParser { +public abstract class AbsSideTableParser extends AbsTableParser { private final static String SIDE_SIGN_KEY = "sideSignKey"; private final static Pattern SIDE_TABLE_SIGN = Pattern.compile("(?i)^PERIOD\\s+FOR\\s+SYSTEM_TIME$"); - public AbstractSideTableParser() { + public AbsSideTableParser() { addParserHandler(SIDE_SIGN_KEY, SIDE_TABLE_SIGN, this::dealSideSign); } - private void dealSideSign(Matcher matcher, AbstractTableInfo tableInfo){ + private void dealSideSign(Matcher matcher, TableInfo tableInfo){ //FIXME SIDE_TABLE_SIGN current just used as a sign for side table; and do nothing } //Analytical create table attributes ==> Get information cache - protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map props){ - if(props.containsKey(AbstractSideTableInfo.CACHE_KEY.toLowerCase())){ - String cacheType = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_KEY.toLowerCase())); + protected void parseCacheProp(SideTableInfo sideTableInfo, Map props){ + if(props.containsKey(SideTableInfo.CACHE_KEY.toLowerCase())){ + String cacheType = MathUtil.getString(props.get(SideTableInfo.CACHE_KEY.toLowerCase())); if(cacheType == null){ return; } @@ -63,66 +61,59 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0."); } sideTableInfo.setCacheSize(cacheSize); } - if(props.containsKey(AbstractSideTableInfo.CACHE_TTLMS_KEY.toLowerCase())){ - Long cacheTTLMS = MathUtil.getLongVal(props.get(AbstractSideTableInfo.CACHE_TTLMS_KEY.toLowerCase())); + if(props.containsKey(SideTableInfo.CACHE_TTLMS_KEY.toLowerCase())){ + Long cacheTTLMS = MathUtil.getLongVal(props.get(SideTableInfo.CACHE_TTLMS_KEY.toLowerCase())); if(cacheTTLMS < 1000){ throw new RuntimeException("cache time out need > 1000 ms."); } sideTableInfo.setCacheTimeout(cacheTTLMS); } - if(props.containsKey(AbstractSideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())){ - Boolean partitionedJoinKey = MathUtil.getBoolean(props.get(AbstractSideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())); + if(props.containsKey(SideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())){ + Boolean partitionedJoinKey = MathUtil.getBoolean(props.get(SideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())); if(partitionedJoinKey){ sideTableInfo.setPartitionedJoin(true); } } - if(props.containsKey(AbstractSideTableInfo.CACHE_MODE_KEY.toLowerCase())){ - String cachemode = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_MODE_KEY.toLowerCase())); + if(props.containsKey(SideTableInfo.CACHE_MODE_KEY.toLowerCase())){ + String cachemode = MathUtil.getString(props.get(SideTableInfo.CACHE_MODE_KEY.toLowerCase())); - if(!"ordered".equalsIgnoreCase(cachemode) && !"unordered".equalsIgnoreCase(cachemode)){ + if(!cachemode.equalsIgnoreCase("ordered") && !cachemode.equalsIgnoreCase("unordered")){ throw new RuntimeException("cachemode must ordered or unordered!"); } sideTableInfo.setCacheMode(cachemode.toLowerCase()); } - if(props.containsKey(AbstractSideTableInfo.ASYNC_CAP_KEY.toLowerCase())){ - Integer asyncCap = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_CAP_KEY.toLowerCase())); + if(props.containsKey(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())){ + Integer asyncCap = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())); if(asyncCap < 0){ throw new RuntimeException("asyncCapacity size need > 0."); } sideTableInfo.setAsyncCapacity(asyncCap); } - if(props.containsKey(AbstractSideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())){ - Integer asyncTimeout = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())); + if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())){ + Integer asyncTimeout = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())); if (asyncTimeout<0){ throw new RuntimeException("asyncTimeout size need > 0."); } sideTableInfo.setAsyncTimeout(asyncTimeout); } - if(props.containsKey(AbstractSideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())){ - Integer asyncTimeoutNum = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())); + if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())){ + Integer asyncTimeoutNum = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())); if (asyncTimeoutNum > 0){ sideTableInfo.setAsyncTimeoutNumLimit(asyncTimeoutNum); } } - - if (props.containsKey(AbstractSideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())) { - Integer asyncPoolSize = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())); - Preconditions.checkArgument(asyncPoolSize > 0 && asyncPoolSize <= 20, "asyncPoolSize size limit (0,20]"); - sideTableInfo.setAsyncPoolSize(asyncPoolSize); - } - } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java similarity index 81% rename from core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java index 308f5859f..745357162 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java @@ -33,7 +33,7 @@ * @author xuchao */ -public abstract class AbstractSourceParser extends AbstractTableParser { +public abstract class AbsSourceParser extends AbsTableParser { private static final String VIRTUAL_KEY = "virtualFieldKey"; private static final String WATERMARK_KEY = "waterMarkKey"; @@ -43,21 +43,21 @@ public abstract class AbstractSourceParser extends AbstractTableParser { private static Pattern waterMarkKeyPattern = Pattern.compile("(?i)^\\s*WATERMARK\\s+FOR\\s+(\\S+)\\s+AS\\s+withOffset\\(\\s*(\\S+)\\s*,\\s*(\\d+)\\s*\\)$"); private static Pattern notNullKeyPattern = Pattern.compile("(?i)^(\\w+)\\s+(\\w+)\\s+NOT\\s+NULL?$"); - public AbstractSourceParser() { + public AbsSourceParser() { addParserHandler(VIRTUAL_KEY, virtualFieldKeyPattern, this::dealVirtualField); addParserHandler(WATERMARK_KEY, waterMarkKeyPattern, this::dealWaterMark); addParserHandler(NOTNULL_KEY, notNullKeyPattern, this::dealNotNull); } - protected void dealVirtualField(Matcher matcher, AbstractTableInfo tableInfo){ - AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; + protected void dealVirtualField(Matcher matcher, TableInfo tableInfo){ + SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; String fieldName = matcher.group(2); String expression = matcher.group(1); sourceTableInfo.addVirtualField(fieldName, expression); } - protected void dealWaterMark(Matcher matcher, AbstractTableInfo tableInfo){ - AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; + protected void dealWaterMark(Matcher matcher, TableInfo tableInfo){ + SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; String eventTimeField = matcher.group(1); //FIXME Temporarily resolve the second parameter row_time_field Integer offset = MathUtil.getIntegerVal(matcher.group(3)); @@ -65,11 +65,11 @@ protected void dealWaterMark(Matcher matcher, AbstractTableInfo tableInfo){ sourceTableInfo.setMaxOutOrderness(offset); } - protected void dealNotNull(Matcher matcher, AbstractTableInfo tableInfo) { + protected void dealNotNull(Matcher matcher, TableInfo tableInfo) { String fieldName = matcher.group(1); String fieldType = matcher.group(2); Class fieldClass= dbTypeConvertToJavaType(fieldType); - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); + TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); fieldExtraInfo.setNotNull(true); tableInfo.addPhysicalMappings(fieldName, fieldName); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java similarity index 88% rename from core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java index 1f210cbf2..3e4027b2c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java @@ -39,7 +39,7 @@ * @author xuchao */ -public abstract class AbstractTableParser { +public abstract class AbsTableParser { private static final String PRIMARY_KEY = "primaryKey"; private static final String NEST_JSON_FIELD_KEY = "nestFieldKey"; @@ -54,7 +54,7 @@ public abstract class AbstractTableParser { private Map handlerMap = Maps.newHashMap(); - public AbstractTableParser() { + public AbsTableParser() { addParserHandler(PRIMARY_KEY, primaryKeyPattern, this::dealPrimaryKey); addParserHandler(NEST_JSON_FIELD_KEY, nestJsonFieldKeyPattern, this::dealNestField); } @@ -63,9 +63,9 @@ protected boolean fieldNameNeedsUpperCase() { return true; } - public abstract AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception; + public abstract TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception; - public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ + public boolean dealKeyPattern(String fieldRow, TableInfo tableInfo){ for(Map.Entry keyPattern : patternMap.entrySet()){ Pattern pattern = keyPattern.getValue(); String key = keyPattern.getKey(); @@ -84,7 +84,7 @@ public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ return false; } - public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ + public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); for(String fieldRow : fieldRows){ @@ -112,12 +112,12 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ Class fieldClass = null; - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; + TableInfo.FieldExtraInfo fieldExtraInfo = null; Matcher matcher = charTypePattern.matcher(fieldType); if (matcher.find()) { fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); - fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); + fieldExtraInfo = new TableInfo.FieldExtraInfo(); fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); } else { fieldClass = dbTypeConvertToJavaType(fieldType); @@ -133,7 +133,7 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ tableInfo.finish(); } - public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo){ + public void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ String primaryFields = matcher.group(1).trim(); String[] splitArry = primaryFields.split(","); List primaryKes = Lists.newArrayList(splitArry); @@ -145,7 +145,7 @@ public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo){ * @param matcher * @param tableInfo */ - protected void dealNestField(Matcher matcher, AbstractTableInfo tableInfo) { + protected void dealNestField(Matcher matcher, TableInfo tableInfo) { String physicalField = matcher.group(1); Preconditions.checkArgument(!physicalFieldFunPattern.matcher(physicalField).find(), "No need to add data types when using functions, The correct way is : strLen(name) as nameSize, "); @@ -154,7 +154,7 @@ protected void dealNestField(Matcher matcher, AbstractTableInfo tableInfo) { String mappingField = matcher.group(4); Class fieldClass = dbTypeConvertToJavaType(fieldType); boolean notNull = matcher.group(5) != null; - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); + TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); fieldExtraInfo.setNotNull(notNull); tableInfo.addPhysicalMappings(mappingField, physicalField); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java b/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java index fab634cdc..db804ea34 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java @@ -30,5 +30,5 @@ */ public interface ITableFieldDealHandler { - void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo); + void dealPrimaryKey(Matcher matcher, TableInfo tableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java similarity index 97% rename from core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java index 0fdc0e911..9a41fa0a1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java @@ -36,7 +36,7 @@ * @author xuchao */ -public abstract class AbstractSourceTableInfo extends AbstractTableInfo { +public abstract class SourceTableInfo extends TableInfo { public static final String SOURCE_SUFFIX = "Source"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java similarity index 98% rename from core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java index 3b514dd48..2fdc297a2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java @@ -34,7 +34,7 @@ * @author xuchao */ -public abstract class AbstractTableInfo implements Serializable { +public abstract class TableInfo implements Serializable { public static final String PARALLELISM_KEY = "parallelism"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java similarity index 83% rename from core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java index d50e1f23c..3c1be25c6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.enums.ETableType; import com.dtstack.flink.sql.parser.CreateTableParser; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.StreamSideFactory; import com.dtstack.flink.sql.sink.StreamSinkFactory; import com.dtstack.flink.sql.source.StreamSourceFactory; @@ -42,7 +42,7 @@ * @author xuchao */ -public class AbstractTableInfoParser { +public class TableInfoParser { private final static String TYPE_KEY = "type"; @@ -50,16 +50,16 @@ public class AbstractTableInfoParser { private final static Pattern SIDE_PATTERN = Pattern.compile(SIDE_TABLE_SIGN); - private Map sourceTableInfoMap = Maps.newConcurrentMap(); + private Map sourceTableInfoMap = Maps.newConcurrentMap(); - private Map targetTableInfoMap = Maps.newConcurrentMap(); + private Map targetTableInfoMap = Maps.newConcurrentMap(); - private Map sideTableInfoMap = Maps.newConcurrentMap(); + private Map sideTableInfoMap = Maps.newConcurrentMap(); //Parsing loaded plugin - public AbstractTableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserResult parserResult, - String localPluginRoot) throws Exception { - AbstractTableParser absTableParser = null; + public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserResult parserResult, + String localPluginRoot) throws Exception { + AbsTableParser absTableParser = null; Map props = parserResult.getPropMap(); String type = MathUtil.getString(props.get(TYPE_KEY)); @@ -77,9 +77,9 @@ public AbstractTableInfo parseWithTableType(int tableType, CreateTableParser.Sql sourceTableInfoMap.put(type, absTableParser); } }else{ - absTableParser = sideTableInfoMap.get(type); + String cacheType = MathUtil.getString(props.get(SideTableInfo.CACHE_KEY)); + absTableParser = sideTableInfoMap.get(type + cacheType); if(absTableParser == null){ - String cacheType = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_KEY)); absTableParser = StreamSideFactory.getSqlParser(type, localPluginRoot, cacheType); sideTableInfoMap.put(type + cacheType, absTableParser); } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java similarity index 94% rename from core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java index dbd99f993..0dc9ca77b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java @@ -29,7 +29,7 @@ * @author xuchao */ -public abstract class AbstractTargetTableInfo extends AbstractTableInfo { +public abstract class TargetTableInfo extends TableInfo { public static final String TARGET_SUFFIX = "Sink"; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index 4bb4ff0d8..feebbe3f1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -86,8 +86,7 @@ public static Class stringConvertClass(String str) { case "decimal": case "decimalunsigned": return BigDecimal.class; - default: - break; + } throw new RuntimeException("不支持 " + str + " 类型"); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index 409602baa..a0f5291e7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -16,17 +16,22 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.util; import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.*; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.util.*; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.util.Calendar; +import java.util.Date; +import java.util.Locale; +import java.util.SimpleTimeZone; +import java.util.TimeZone; import java.util.regex.Pattern; import static java.time.format.DateTimeFormatter.ISO_INSTANT; @@ -34,7 +39,7 @@ /** * - * 日期工具 + * Reason: TODO ADD REASON(可选) * Date: 2017年03月10日 下午1:16:37 * Company: www.dtstack.com * @author sishu.yss @@ -42,14 +47,18 @@ */ public class DateUtil { - static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss"); + static final String timeZone = "GMT+8"; + static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss"; + static final String dateFormat = "yyyy-MM-dd"; + static final String timeFormat = "HH:mm:ss"; + static final SimpleDateFormat datetimeFormatter = new SimpleDateFormat(datetimeFormat); + static final SimpleDateFormat dateFormatter = new SimpleDateFormat(dateFormat); + static final SimpleDateFormat timeFormatter = new SimpleDateFormat(timeFormat); private static final Pattern DATETIME = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); private static final Pattern DATE = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}$"); - private static final int MILLIS_PER_SECOND = 1000; + private static final int MILLIS_PER_SECOND = 1000; public static java.sql.Date columnToDate(Object column) { @@ -75,32 +84,23 @@ public static Date stringToDate(String strDate) { return null; } try { - ; - return localDateTimetoDate(LocalDateTime.parse(strDate, DATE_TIME_FORMATTER)); - } catch (DateTimeParseException ignored) { + return datetimeFormatter.parse(strDate); + } catch (ParseException ignored) { } try { - return localDateTimetoDate(LocalDate.parse(strDate, DATE_FORMATTER).atStartOfDay()); - } catch (DateTimeParseException ignored) { + return dateFormatter.parse(strDate); + } catch (ParseException ignored) { } try { - return localDateTimetoDate(LocalDateTime.of(LocalDate.now(), LocalTime.parse(strDate, TIME_FORMATTER))); - } catch (DateTimeParseException ignored) { + return timeFormatter.parse(strDate); + } catch (ParseException ignored) { } throw new RuntimeException("can't parse date"); } - public static Date localDateTimetoDate(LocalDateTime localDateTime){ - return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant()); - } - - public static LocalDateTime dateToLocalDateTime(Date date){ - return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); - } - /** * * @@ -130,13 +130,13 @@ public static long getTodayStart(long day) { * @return */ public static long getTodayStart(long day,String scope) { - if("MS".equals(scope)){ - return getTodayStart(day)*1000; - }else if("S".equals(scope)){ - return getTodayStart(day); - }else{ - return getTodayStart(day); - } + if(scope.equals("MS")){ + return getTodayStart(day)*1000; + }else if(scope.equals("S")){ + return getTodayStart(day); + }else{ + return getTodayStart(day); + } } /** @@ -168,15 +168,15 @@ public static long getNextDayStart(long day) { * @return */ public static long getNextDayStart(long day,String scope) { - if("MS".equals(scope)){ - return getNextDayStart(day)*1000; - }else if("S".equals(scope)){ - return getNextDayStart(day); - }else{ - return getNextDayStart(day); - } + if(scope.equals("MS")){ + return getNextDayStart(day)*1000; + }else if(scope.equals("S")){ + return getNextDayStart(day); + }else{ + return getNextDayStart(day); + } } - + /** * @@ -237,7 +237,7 @@ public static long getWeekFirst(long day) { /** * 根据某个日期时间戳秒值，获取所在周在一年中是第几周. - * + * * @param day * @return */ @@ -257,17 +257,17 @@ public static int getWeekOfYear(long day) { */ public static String getYesterdayByString(String day, String inFormat, String outFormat){ try { - SimpleDateFormat sdf = new SimpleDateFormat(inFormat); - Date date = sdf.parse(day); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - int calendarDay = calendar.get(Calendar.DATE); - calendar.set(Calendar.DATE, calendarDay - 1); - String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); - return dayBefore; - } catch (ParseException e) { - return null; - } + SimpleDateFormat sdf = new SimpleDateFormat(inFormat); + Date date = sdf.parse(day); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + int calendarDay = calendar.get(Calendar.DATE); + calendar.set(Calendar.DATE, calendarDay - 1); + String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); + return dayBefore; + } catch (ParseException e) { + return null; + } } /** @@ -288,7 +288,7 @@ public static String getTomorrowByString(String day, String inFormat, String out String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); return dayBefore; } - + /** * * @param date @@ -320,7 +320,7 @@ public static String get30DaysBeforeByString(String day, String inFormat, String calendar.set(Calendar.DATE, calendarDay - 30); return new SimpleDateFormat(outFormat).format(calendar.getTime()); } - + /** * * @param day @@ -330,14 +330,14 @@ public static String get30DaysBeforeByString(String day, String inFormat, String * @throws ParseException */ public static String get30DaysLaterByString(String day, String inFormat, String outFormat) throws ParseException { - SimpleDateFormat sdf = new SimpleDateFormat(inFormat); - Date date = sdf.parse(day); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - int calendarDay = calendar.get(Calendar.DATE); - calendar.set(Calendar.DATE, calendarDay + 30); - String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); - return dayBefore; + SimpleDateFormat sdf = new SimpleDateFormat(inFormat); + Date date = sdf.parse(day); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + int calendarDay = calendar.get(Calendar.DATE); + calendar.set(Calendar.DATE, calendarDay + 30); + String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); + return dayBefore; } @@ -349,7 +349,7 @@ public static String get30DaysLaterByString(String day, String inFormat, String * @return String * @throws ParseException */ - public static String getDateStrToFormat(String day, String inFormat, String outFormat) throws ParseException { + public static String getDateStrTOFormat(String day, String inFormat, String outFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -357,8 +357,8 @@ public static String getDateStrToFormat(String day, String inFormat, String outF String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); return dayBefore; } - - public static long getDateMillToFormat(String day, String inFormat) throws ParseException { + + public static long getDateMillTOFormat(String day, String inFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -474,79 +474,79 @@ public static long getMillByLastWeekDay() { cal.set(Calendar.MILLISECOND, 0); return cal.getTimeInMillis() / 1000; } - - /** - * @return long - */ + + /** + * @return long + */ public static long getMillByDay(int severalDays,String condition) { - int dateT=0; + int dateT=0; Calendar cal = Calendar.getInstance(); - if(condition==null){ - return getMillToDay(cal,dateT); - } - if("-".equals(condition)){ - dateT = (cal.get(Calendar.DATE) - severalDays); - return getMillToDay(cal,dateT); - } - if("+".equals(condition)){ - dateT = (cal.get(Calendar.DATE) + severalDays); - return getMillToDay(cal,dateT); - } - return getMillToDay(cal,dateT); - } - + if(condition==null){ + return getMillToDay(cal,dateT); + } + if(condition.equals("-")){ + dateT = (cal.get(Calendar.DATE) - severalDays); + return getMillToDay(cal,dateT); + } + if(condition.equals("+")){ + dateT = (cal.get(Calendar.DATE) + severalDays); + return getMillToDay(cal,dateT); + } + return getMillToDay(cal,dateT); + } + /** * @return long */ public static long getStampByDay(int severalDays,String condition) { - int dateT=0; - Calendar cal = Calendar.getInstance(); - if(condition==null){ - return getStampToDay(cal,dateT); - } - if("-".equals(condition)){ - dateT = (cal.get(Calendar.DATE) - severalDays); - return getStampToDay(cal,dateT); - } - if("+".equals(condition)){ - dateT = (cal.get(Calendar.DATE) + severalDays); - return getStampToDay(cal,dateT); - } - return getStampToDay(cal,dateT); + int dateT=0; + Calendar cal = Calendar.getInstance(); + if(condition==null){ + return getStampToDay(cal,dateT); + } + if(condition.equals("-")){ + dateT = (cal.get(Calendar.DATE) - severalDays); + return getStampToDay(cal,dateT); + } + if(condition.equals("+")){ + dateT = (cal.get(Calendar.DATE) + severalDays); + return getStampToDay(cal,dateT); + } + return getStampToDay(cal,dateT); } /** * @return long */ public static long getMillByDay(){ - return getMillByDay(0,null); + return getMillByDay(0,null); } - + /** * @param cal Calendar - * @param dateT Integer + * @param dateT Integer * @return long */ public static long getMillToDay(Calendar cal,int dateT){ - if(dateT!=0){ - cal.set(Calendar.DATE, dateT); - } - cal.set(Calendar.HOUR_OF_DAY, 0); - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - return cal.getTimeInMillis()/1000; - } - + if(dateT!=0){ + cal.set(Calendar.DATE, dateT); + } + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + return cal.getTimeInMillis()/1000; + } + /** * @param cal Calendar - * @param dateT Integer + * @param dateT Integer * @return long */ public static long getStampToDay(Calendar cal,int dateT){ - if(dateT!=0){ - cal.set(Calendar.DATE, dateT); - } - return cal.getTimeInMillis(); + if(dateT!=0){ + cal.set(Calendar.DATE, dateT); + } + return cal.getTimeInMillis(); } public static String getToday() { @@ -568,7 +568,7 @@ public static String getDate(long day, String format) { SimpleDateFormat sf = new SimpleDateFormat(format); return sf.format(cal.getTime()); } - + /** * * @param date @@ -578,21 +578,21 @@ public static String getDate(Date date, String format) { SimpleDateFormat sf = new SimpleDateFormat(format); return sf.format(date); } - - + + /** * * @param day * @param format * @return long - * @throws ParseException + * @throws ParseException */ public static long stringToLong(String day, String format) throws ParseException { - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - long date = dateFormat.parse(day).getTime(); - return date; + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + long Date = dateFormat.parse(day).getTime(); + return Date; } - + /** * @param day * @param format @@ -600,30 +600,30 @@ public static long stringToLong(String day, String format) throws ParseException * @throws ParseException */ public static Date stringToDate(String day, String format) { - try { - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - Date date = dateFormat.parse(day); - return date; - } catch (ParseException e) { - return new Date(); - } + try { + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + Date Date = dateFormat.parse(day); + return Date; + } catch (ParseException e) { + return new Date(); + } } - - + + /** * long型时间戳转为String型 - * + * * @param day 秒 * @return 格式化后的日期 - * @throws ParseException + * @throws ParseException */ public static String longToString(long day, String format) throws ParseException { - if (("" + day).length() <= 10){ + if (("" + day).length() <= 10){ day=day*1000; } - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - String date = dateFormat.format(day); - return date; + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + String Date = dateFormat.format(day); + return Date; } /** @@ -653,39 +653,39 @@ public static long getMillByNow() { return cal.getTimeInMillis(); } - public static int getWeeksBetweenTwoDates(long startDay, long endDay) { - int week = getWeekOfYear(endDay) - getWeekOfYear(startDay) + 1; - if(week<1){ - week = getWeekOfYear(endDay) + getMaxWeekOfYear(startDay) - getWeekOfYear(startDay) + 1; - } - return week; - } + public static int getWeeksBetweenTwoDates(long startDay, long endDay) { + int week = getWeekOfYear(endDay) - getWeekOfYear(startDay) + 1; + if(week<1){ + week = getWeekOfYear(endDay) + getMaxWeekOfYear(startDay) - getWeekOfYear(startDay) + 1; + } + return week; + } - public static int getMaxWeekOfYear(long startDay) { - Calendar cal = Calendar.getInstance(); + public static int getMaxWeekOfYear(long startDay) { + Calendar cal = Calendar.getInstance(); cal.setTime(new Date(startDay * 1000)); return cal.getMaximum(Calendar.WEEK_OF_YEAR); - } - - public static int getMonthsBetweenTwoDates(long startDay, long endDay) { - int month = DateUtil.getMonth(endDay) - DateUtil.getMonth(startDay) + 1; - if(month<1){ - month = getMonth(endDay) + 12 - getMonth(startDay) +1; - } - return month; - } - - public static Date parseDate(String dateStr, String pattern){ - SimpleDateFormat sdf = new SimpleDateFormat(); - sdf.applyPattern(pattern); - try { - return sdf.parse(dateStr); - } catch (ParseException e) { - return null; - } - } - - /** + } + + public static int getMonthsBetweenTwoDates(long startDay, long endDay) { + int month = DateUtil.getMonth(endDay) - DateUtil.getMonth(startDay) + 1; + if(month<1){ + month = getMonth(endDay) + 12 - getMonth(startDay) +1; + } + return month; + } + + public static Date parseDate(String dateStr, String pattern){ + SimpleDateFormat sdf = new SimpleDateFormat(); + sdf.applyPattern(pattern); + try { + return sdf.parse(dateStr); + } catch (ParseException e) { + return null; + } + } + + /** * * @param time Long 时间 * @return long @@ -703,7 +703,7 @@ public static long getMinuteStart(long time) { firstDay = cal.getTimeInMillis() / 1000; return firstDay; } - + /** * @param time Long * @return long @@ -732,24 +732,24 @@ public static Date getDateByLong(long time){ date.setTime(time); return date; } - + public static Date parseDate(String dateStr, String pattern, Locale locale){ - SimpleDateFormat df = new SimpleDateFormat( - pattern, locale); - - df.setTimeZone(new SimpleTimeZone(0, "GMT")); - try { - return df.parse(dateStr); - } catch (ParseException e) { - return null; - } - } - + SimpleDateFormat df = new SimpleDateFormat( + pattern, locale); + + df.setTimeZone(new SimpleTimeZone(0, "GMT")); + try { + return df.parse(dateStr); + } catch (ParseException e) { + return null; + } + } + public static String getDate(Date date, String format, Locale locale) { - SimpleDateFormat df = new SimpleDateFormat( - format, locale); - df.setTimeZone(new SimpleTimeZone(0, "GMT")); + SimpleDateFormat df = new SimpleDateFormat( + format, locale); + df.setTimeZone(new SimpleTimeZone(0, "GMT")); return df.format(date); } @@ -777,33 +777,59 @@ public static java.sql.Timestamp columnToTimestamp(Object column) { } public static String dateToString(Date date) { - LocalDateTime localDateTime = dateToLocalDateTime(date); - return localDateTime.format(DATE_FORMATTER); + return dateFormatter.format(date); } public static String timestampToString(Date date) { - LocalDateTime localDateTime = dateToLocalDateTime(date); - return localDateTime.format(DATE_TIME_FORMATTER); + return datetimeFormatter.format(date); } + public static Timestamp getTimestampFromStr(String timeStr) { if (DATETIME.matcher(timeStr).matches()) { Instant instant = Instant.from(ISO_INSTANT.parse(timeStr)); return new Timestamp(instant.getEpochSecond() * MILLIS_PER_SECOND); + } else { + java.sql.Date date = null; + try { + date = new java.sql.Date(datetimeFormatter.parse(timeStr).getTime()); + } catch (ParseException e) { + throw new RuntimeException("getTimestampFromStr error data is " + timeStr); + } + return new Timestamp(date.getTime()); } - return new Timestamp(stringToDate(timeStr).getTime()); } public static java.sql.Date getDateFromStr(String dateStr) { + // 2020-01-01 format if (DATE.matcher(dateStr).matches()) { + // convert from local date to instant Instant instant = LocalDate.parse(dateStr).atTime(LocalTime.of(0, 0, 0, 0)).toInstant(ZoneOffset.UTC); + // calculate the timezone offset in millis int offset = TimeZone.getDefault().getOffset(instant.toEpochMilli()); + // need to remove the offset since time has no TZ component return new java.sql.Date(instant.toEpochMilli() - offset); } else if (DATETIME.matcher(dateStr).matches()) { + // 2020-01-01T12:12:12Z format Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); return new java.sql.Date(instant.toEpochMilli()); + } else { + try { + // 2020-01-01 12:12:12.0 format + return new java.sql.Date(datetimeFormatter.parse(dateStr).getTime()); + } catch (ParseException e) { + throw new RuntimeException("String convert to Date fail."); + } } - return new java.sql.Date(stringToDate(dateStr).getTime()); + } + + + public static String getStringFromTimestamp(Timestamp timestamp) { + return datetimeFormatter.format(timestamp); + } + + public static String getStringFromDate(java.sql.Date date) { + return dateFormatter.format(date); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index 10a6b4f63..b2486b1ab 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -219,14 +219,14 @@ public static String addJdbcParam(String dbUrl, Map addParams, b return preStr + "?" + sb.toString(); } - public static boolean isJson(String str) { + public static boolean isJosn(String str){ boolean flag = false; - if (StringUtils.isNotBlank(str)) { + if(StringUtils.isNotBlank(str)){ try { - objectMapper.readValue(str, Map.class); + objectMapper.readValue(str,Map.class); flag = true; } catch (Throwable e) { - flag = false; + flag=false; } } return flag; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java index 3d66d8a6d..fde2f166e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java @@ -28,10 +28,10 @@ public class JDBCUtils { private static final Logger LOG = LoggerFactory.getLogger(ClassUtil.class); - public final static String LOCK_STR = "jdbc_lock_str"; + public final static String lock_str = "jdbc_lock_str"; public static void forName(String clazz, ClassLoader classLoader) { - synchronized (LOCK_STR){ + synchronized (lock_str){ try { Class.forName(clazz, true, classLoader); DriverManager.setLoginTimeout(10); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index 5aaa40a15..23513a912 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -227,7 +227,7 @@ public static BigDecimal getBigDecimal(Object obj) { } else if (obj instanceof BigInteger) { return new BigDecimal((BigInteger) obj); } else if (obj instanceof Number) { - return BigDecimal.valueOf(((Number) obj).doubleValue()); + return new BigDecimal(((Number) obj).doubleValue()); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to BigDecimal."); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java index 8eb743cf0..a63e5a663 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java @@ -109,7 +109,7 @@ public static String getSqlSideClassName(String pluginTypeName, String type, Str return CLASS_PRE_STR + "." + type.toLowerCase() + "." + pluginTypeName + "." + pluginClassName; } - public static Map objectToMap(Object obj) throws Exception{ + public static Map ObjectToMap(Object obj) throws Exception{ return objectMapper.readValue(objectMapper.writeValueAsBytes(obj), Map.class); } @@ -173,8 +173,8 @@ public static void addPluginJar(String pluginDir, DtClassLoader classLoader) thr } for(File file : files){ - URL pluginJarUrl = file.toURI().toURL(); - classLoader.addURL(pluginJarUrl); + URL pluginJarURL = file.toURI().toURL(); + classLoader.addURL(pluginJarURL); } } @@ -191,8 +191,8 @@ public static URL[] getPluginJarUrls(String pluginDir) throws MalformedURLExcept } for(File file : files){ - URL pluginJarUrl = file.toURI().toURL(); - urlList.add(pluginJarUrl); + URL pluginJarURL = file.toURI().toURL(); + urlList.add(pluginJarURL); } return urlList.toArray(new URL[urlList.size()]); } diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java similarity index 94% rename from core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java rename to core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java index d75d26a61..ed8f64288 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java @@ -38,7 +38,7 @@ * @author xuchao */ -public abstract class AbstractCustomerWaterMarker extends BoundedOutOfOrdernessTimestampExtractor implements RichFunction { +public abstract class AbsCustomerWaterMarker extends BoundedOutOfOrdernessTimestampExtractor implements RichFunction { private static final long serialVersionUID = 1L; @@ -55,7 +55,7 @@ public abstract class AbstractCustomerWaterMarker extends BoundedOutOfOrderne protected TimeZone timezone; - public AbstractCustomerWaterMarker(Time maxOutOfOrderness) { + public AbsCustomerWaterMarker(Time maxOutOfOrderness) { super(maxOutOfOrderness); } diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java index 979b4c3d6..99415ba61 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java @@ -35,7 +35,7 @@ * @author xuchao */ -public class CustomerWaterMarkerForLong extends AbstractCustomerWaterMarker { +public class CustomerWaterMarkerForLong extends AbsCustomerWaterMarker { private static final Logger logger = LoggerFactory.getLogger(CustomerWaterMarkerForLong.class); diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java index 05ee3e46e..b7961e410 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.watermarker; +import com.dtstack.flink.sql.util.MathUtil; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -35,7 +36,7 @@ * @author xuchao */ -public class CustomerWaterMarkerForTimeStamp extends AbstractCustomerWaterMarker { +public class CustomerWaterMarkerForTimeStamp extends AbsCustomerWaterMarker { private static final Logger logger = LoggerFactory.getLogger(CustomerWaterMarkerForTimeStamp.class); diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java index b0ca0335f..f3208db03 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.watermarker; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.base.Strings; @@ -40,7 +40,7 @@ public class WaterMarkerAssigner { - public boolean checkNeedAssignWaterMarker(AbstractSourceTableInfo tableInfo){ + public boolean checkNeedAssignWaterMarker(SourceTableInfo tableInfo){ if(Strings.isNullOrEmpty(tableInfo.getEventTimeField())){ return false; } @@ -48,7 +48,7 @@ public boolean checkNeedAssignWaterMarker(AbstractSourceTableInfo tableInfo){ return true; } - public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo typeInfo, AbstractSourceTableInfo sourceTableInfo){ + public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo typeInfo, SourceTableInfo sourceTableInfo){ String eventTimeFieldName = sourceTableInfo.getEventTimeField(); @@ -75,7 +75,7 @@ public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo type TypeInformation fieldType = fieldTypes[pos]; - AbstractCustomerWaterMarker waterMarker = null; + AbsCustomerWaterMarker waterMarker = null; if(fieldType.getTypeClass().isAssignableFrom(Timestamp.class)){ waterMarker = new CustomerWaterMarkerForTimeStamp(Time.milliseconds(maxOutOrderness), pos,timeZone); }else if(fieldType.getTypeClass().isAssignableFrom(Long.class)){ diff --git a/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java b/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java index feec8e547..546fcbbac 100644 --- a/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java +++ b/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java @@ -54,7 +54,7 @@ public void testfillPredicatesForSideTable() throws SqlParseException { " MyTable.a='1' and s.d='1' and s.d <> '3' and s.c LIKE '%xx%' and s.c in ('1','2') and s.c between '10' and '23' and s.d is not null\n"; - AbstractSideTableInfo sideTableInfo = new AbstractSideTableInfo(){ + SideTableInfo sideTableInfo = new SideTableInfo(){ @Override public boolean check() { return false; @@ -63,7 +63,7 @@ public boolean check() { sideTableInfo.setName("sideTable"); - Map sideTableMap = new HashMap<>(); + Map sideTableMap = new HashMap<>(); sideTableMap.put("sideTable", sideTableInfo); SidePredicatesParser sidePredicatesParser = new SidePredicatesParser(); diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java index b354a6269..a77201b80 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java @@ -20,8 +20,10 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.google.common.collect.Maps; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,6 +31,7 @@ import java.sql.Connection; import java.sql.DriverManager; import java.util.List; +import java.util.Map; /** * Date: 2019/11/20 @@ -37,21 +40,21 @@ * @author xiuzhu */ -public class Db2AllReqRow extends AbstractRdbAllReqRow { +public class Db2AllReqRow extends RdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(Db2AllReqRow.class); private static final String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; - public Db2AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Db2AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new Db2AllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbUrl, String userName, String password) { + public Connection getConn(String dbURL, String userName, String password) { try { Class.forName(DB2_DRIVER); - Connection conn = DriverManager.getConnection(dbUrl, userName, password); + Connection conn = DriverManager.getConnection(dbURL, userName, password); return conn; } catch (Exception e) { LOG.error("", e); diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java index 0120c58a0..282ee440c 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class Db2AllSideInfo extends RdbAllSideInfo { - public Db2AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Db2AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index 8e7275977..3d85c6fbd 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -48,7 +48,7 @@ public class Db2AsyncReqRow extends RdbAsyncReqRow { private final static String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; - public Db2AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Db2AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new Db2AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -59,7 +59,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); db2lientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", DB2_DRIVER) - .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -71,10 +71,10 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, db2lientConfig)); + setRdbSQLClient(JDBCClient.createNonShared(vertx, db2lientConfig)); } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java index 9cbdc997e..e53dd7f27 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class Db2AsyncSideInfo extends RdbAsyncSideInfo { - public Db2AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Db2AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java b/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java index 96be4ec15..db8a07dc6 100644 --- a/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java +++ b/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.db2.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -36,8 +36,8 @@ public class Db2SideParser extends RdbSideParser { private static final String CURR_TYPE = "db2"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURR_TYPE); return tableInfo; diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java index da1434ace..9942d4438 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java @@ -1,10 +1,13 @@ package com.dtstack.flink.sql.sink.db; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -public class DbSink extends AbstractRdbSink { +import java.util.List; +import java.util.Map; + +public class DbSink extends RdbSink { public DbSink() { super(new DbDialect()); @@ -12,7 +15,7 @@ public DbSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(dbUrl) + .setDBUrl(dbURL) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java index 3e823ab20..ba11aabf3 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java @@ -1,7 +1,7 @@ package com.dtstack.flink.sql.sink.db.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -10,8 +10,8 @@ public class DbSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "db2"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURR_TYPE); return tableInfo; } diff --git a/docs/kafkaSink.md b/docs/kafkaSink.md deleted file mode 100644 index 5d7c7b2a7..000000000 --- a/docs/kafkaSink.md +++ /dev/null @@ -1,223 +0,0 @@ -## 1.格式： -``` -CREATE TABLE tableName( - colName colType, - ... - function(colNameX) AS aliasName, - WATERMARK FOR colName AS withOffset( colName , delayTime ) - )WITH( - type ='kafka11', - bootstrapServers ='ip:port,ip:port...', - zookeeperQuorum ='ip:port,ip:port/zkparent', - offsetReset ='latest', - topic ='topicName', - groupId='test', - parallelism ='parllNum', - ); -``` - -## 2.支持的版本 - kafka09,kafka10,kafka11及以上版本 - **kafka读取和写入的版本必须一致，否则会有兼容性错误。** - -## 3.表结构定义 - -|参数名称|含义| -|----|---| -| tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称| -| colName | 列名称| -| colType | 列类型 [colType支持的类型](colType.md)| - -## 4.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|kafka09、kafka10、kafka11、kafka(对应kafka1.0及以上版本)| -|groupId | 需要读取的 groupId 名称|否|| -|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|topic | 需要读取的 topic 名称|是|| -|parallelism | 并行度设置|否|1| -|partitionKeys | 用来分区的字段|否|| -|updateMode | 回溯流数据下发模式，append,upsert.upsert模式下会将是否为回溯信息以字段形式进行下发。|否|append| -|sinkdatatype | 写入kafka数据格式，json,avro,csv|否|json| -|fieldDelimiter | csv数据分隔符|否| , | - - -**kafka相关参数可以自定义，使用kafka.开头即可。** -``` -kafka.consumer.id -kafka.socket.timeout.ms -kafka.fetch.message.max.bytes -kafka.num.consumer.fetchers -kafka.auto.commit.enable -kafka.auto.commit.interval.ms -kafka.queued.max.message.chunks -kafka.rebalance.max.retries -kafka.fetch.min.bytes -kafka.fetch.wait.max.ms -kafka.rebalance.backoff.ms -kafka.refresh.leader.backoff.ms -kafka.consumer.timeout.ms -kafka.exclude.internal.topics -kafka.partition.assignment.strategy -kafka.client.id -kafka.zookeeper.session.timeout.ms -kafka.zookeeper.connection.timeout.ms -kafka.zookeeper.sync.time.ms -kafka.offsets.storage -kafka.offsets.channel.backoff.ms -kafka.offsets.channel.socket.timeout.ms -kafka.offsets.commit.max.retries -kafka.dual.commit.enabled -kafka.partition.assignment.strategy -kafka.socket.receive.buffer.bytes -kafka.fetch.min.bytes -``` - -## 5.样例： - -### json格式： -``` -CREATE TABLE MyResult( - channel varchar, - pv varchar - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - topic='mqTest02', - parallelism ='2', - partitionKeys = 'channel,pv', - updateMode='upsert' - ); - -upsert模式下发的数据格式：{"channel":"zs","pv":"330",retract:true} -append模式下发的数据格式：{"channel":"zs","pv":"330"} - -``` - -### avro格式： - -如果updateMode='upsert',schemaInfo需要包含retract属性信息。 - -``` -CREATE TABLE MyTable( - channel varchar, - pv varchar - --xctime bigint - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - groupId='mqTest01', - offsetReset='latest', - topic='mqTest01', - parallelism ='1', - topicIsPattern ='false' - ); - -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), - PERIOD FOR SYSTEM_TIME - )WITH( - type='mysql', - url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='sidetest', - cache = 'LRU', - cacheTTLMs='10000', - parallelism ='1' - - ); - - -CREATE TABLE MyResult( - channel varchar, - pv varchar - )WITH( - --type='console' - type='kafka', - bootstrapServers='172.16.8.107:9092', - topic='mqTest02', - parallelism ='1', - updateMode='upsert', - sinkdatatype = 'avro', - schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"} - ,{"name":"pv","type":"string"},{"name":"channel","type":"string"}, - {"name":"retract","type":"boolean"}]}' - - ); - - -insert -into - MyResult - select - a.channel as channel, - a.pv as pv - from - MyTable a -``` -### csv格式： - -``` -CREATE TABLE MyTable( - channel varchar, - pv varchar - --xctime bigint - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - groupId='mqTest01', - offsetReset='latest', - topic='mqTest01', - parallelism ='2', - topicIsPattern ='false' - ); - -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), - PERIOD FOR SYSTEM_TIME - )WITH( - type='mysql', - url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='sidetest', - cache = 'LRU', - cacheTTLMs='10000', - parallelism ='1' - - ); - - -CREATE TABLE MyResult( - channel varchar, - pv varchar - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - topic='mqTest02', - parallelism ='2', - updateMode='upsert', - sinkdatatype = 'csv', - fieldDelimiter='*' - - - - ); - - -insert -into - MyResult - select - a.channel as channel, - a.pv as pv - from - MyTable a -``` diff --git a/docs/kafkaSource.md b/docs/kafkaSource.md index 00c252c99..76096aba6 100644 --- a/docs/kafkaSource.md +++ b/docs/kafkaSource.md @@ -1,5 +1,6 @@ ## 1.格式： ``` +数据现在支持json格式{"xx":"bb","cc":"dd"} CREATE TABLE tableName( colName colType, @@ -14,8 +15,9 @@ CREATE TABLE tableName( topic ='topicName', groupId='test', parallelism ='parllNum', + --timezone='America/Los_Angeles', timezone='Asia/Shanghai', - sourcedatatype ='dt_nest' #可不设置 + sourcedatatype ='json' #可不设置 ); ``` @@ -45,9 +47,7 @@ CREATE TABLE tableName( |topicIsPattern | topic是否是正则表达式格式(true|false) |否| false |offsetReset | 读取的topic 的offset初始位置[latest|earliest|指定offset值({"0":12312,"1":12321,"2":12312},{"partition_no":offset_value})]|否|latest| |parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型,avro,csv,json,dt_nest。dt_nest为默认JSON解析器，能够解析嵌套JSON数据类型，其他仅支持非嵌套格式|否|dt_nest| -|schemaInfo | avro类型使用的schema信息|否|| -|fieldDelimiter |csv类型使用的数据分隔符|否| | | +|sourcedatatype | 数据类型|否|json| |timezone|时区设置[timezone支持的参数](timeZone.md)|否|'Asia/Shanghai' **kafka相关参数可以自定义，使用kafka.开头即可。** ``` @@ -169,10 +169,24 @@ CREATE TABLE MyTable( parallelism ='1' ); ``` +# 二、csv格式数据源 +根据字段分隔符进行数据分隔，按顺序匹配sql中配置的列。如数据分隔列数和sql中配置的列数相等直接匹配；如不同参照lengthcheckpolicy策略处理。 +## 1.参数： + +|参数名称|含义|是否必填|默认值| +|----|---|---|---| +|type | kafka09 | 是|| +|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| +|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| +|topic | 需要读取的 topic 名称|是|| +|offsetReset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| +|parallelism | 并行度设置 |否|1| +|sourcedatatype | 数据类型|是 |csv| +|fielddelimiter | 字段分隔符|是 || +|lengthcheckpolicy | 单行字段条数检查策略 |否|可选，默认为SKIP,其它可选值为EXCEPTION、PAD。SKIP：字段数目不符合时跳过 。EXCEPTION:字段数目不符合时抛出异常。PAD:按顺序填充，不存在的置为null。| +**kafka相关参数可以自定义，使用kafka.开头即可。** -## 7.csv格式数据源 - - +## 2.样例： ``` CREATE TABLE MyTable( name varchar, @@ -189,28 +203,186 @@ CREATE TABLE MyTable( --topic ='mqTest.*', --topicIsPattern='true' parallelism ='1', - sourceDatatype ='csv' + sourcedatatype ='csv', + fielddelimiter ='\|', + lengthcheckpolicy = 'PAD' ); ``` -## 8.avro格式数据源 +# 三、text格式数据源UDF自定义拆分 +Kafka源表数据解析流程：Kafka Source Table -> UDTF ->Realtime Compute -> SINK。从Kakfa读入的数据，都是VARBINARY（二进制）格式，对读入的每条数据，都需要用UDTF将其解析成格式化数据。 + 与其他格式不同，本格式定义DDL必须与以下SQL一摸一样，表中的五个字段顺序务必保持一致： + +## 1. 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 +``` +create table kafka_stream( + _topic STRING, + _messageKey STRING, + _message STRING, + _partition INT, + _offset BIGINT, +) with ( + type ='kafka09', + bootstrapServers ='172.16.8.198:9092', + zookeeperQuorum ='172.16.8.198:2181/kafka', + offsetReset ='latest', + topic ='nbTest1', + parallelism ='1', + sourcedatatype='text' + ） +``` +## 2.参数： + +|参数名称|含义|是否必填|默认值| +|----|---|---|---| +|type | kafka09 | 是|| +|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| +|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| +|topic | 需要读取的 topic 名称|是|| +|offsetReset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| +|parallelism | 并行度设置|否|1| +|sourcedatatype | 数据类型|否|text| +**kafka相关参数可以自定义，使用kafka.开头即可。** +## 2.自定义： +从kafka读出的数据，需要进行窗口计算。 按照实时计算目前的设计，滚窗/滑窗等窗口操作，需要（且必须）在源表DDL上定义Watermark。Kafka源表比较特殊。如果要以kafka中message字段中的的Event Time进行窗口操作， +需要先从message字段，使用UDX解析出event time，才能定义watermark。 在kafka源表场景中，需要使用计算列。 假设，kafka中写入的数据如下： +2018-11-11 00:00:00|1|Anna|female整个计算流程为：Kafka SOURCE->UDTF->Realtime Compute->RDS SINK（单一分隔符可直接使用类csv格式模板，自定义适用于更复杂的数据类型，本说明只做参考） + +**SQL** ``` -CREATE TABLE MyTable( - channel varchar, - pv varchar - --xctime bigint - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - groupId='mqTest01', - offsetReset='latest', - topic='mqTest01', - parallelism ='1', - topicIsPattern ='false', - kafka.group.id='mqTest', - sourceDataType ='avro', - schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"},{"name":"pv","type":"string"}]}' +-- 定义解析Kakfa message的UDTF + CREATE FUNCTION kafkapaser AS 'com.XXXX.kafkaUDTF'; + CREATE FUNCTION kafkaUDF AS 'com.XXXX.kafkaUDF'; + -- 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 + create table kafka_src ( + _topic STRING, + _messageKey STRING, + _message STRING, + _partition INT, + _offset BIGINT, + ctime AS TO_TIMESTAMP(kafkaUDF(_message)), -- 定义计算列，计算列可理解为占位符，源表中并没有这一列，其中的数据可经过下游计算得出。注意计算里的类型必须为timestamp才能在做watermark。 + watermark for ctime as withoffset(ctime,0) -- 在计算列上定义watermark + ) WITH ( + type = 'kafka010', -- Kafka Source类型，与Kafka版本强相关，目前支持的Kafka版本请参考本文档 + topic = 'test_kafka_topic', + ... ); - + create table rds_sink ( + name VARCHAR, + age INT, + grade VARCHAR, + updateTime TIMESTAMP + ) WITH( + type='mysql', + url='jdbc:mysql://localhost:3306/test', + tableName='test4', + userName='test', + password='XXXXXX' + ); + -- 使用UDTF，将二进制数据解析成格式化数据 + CREATE VIEW input_view ( + name, + age, + grade, + updateTime + ) AS + SELECT + COUNT(*) as cnt, + T.ctime, + T.order, + T.name, + T.sex + from + kafka_src as S, + LATERAL TABLE (kafkapaser _message)) as T ( + ctime, + order, + name, + sex + ) + Group BY T.sex, + TUMBLE(ctime, INTERVAL '1' MINUTE); + -- 对input_view中输出的数据做计算 + CREATE VIEW view2 ( + cnt, + sex + ) AS + SELECT + COUNT(*) as cnt, + T.sex + from + input_view + Group BY sex, TUMBLE(ctime, INTERVAL '1' MINUTE); + -- 使用解析出的格式化数据进行计算，并将结果输出到RDS中 + insert into rds_sink + SELECT + cnt,sex + from view2; + ``` +**UDF&UDTF** ``` - +package com.XXXX; + import com.XXXX.fastjson.JSONObject; + import org.apache.flink.table.functions.TableFunction; + import org.apache.flink.table.types.DataType; + import org.apache.flink.table.types.DataTypes; + import org.apache.flink.types.Row; + import java.io.UnsupportedEncodingException; + /** + 以下例子解析输入Kafka中的JSON字符串，并将其格式化输出 + **/ + public class kafkaUDTF extends TableFunction { + public void eval(byte[] message) { + try { + // 读入一个二进制数据，并将其转换为String格式 + String msg = new String(message, "UTF-8"); + // 提取JSON Object中各字段 + String ctime = Timestamp.valueOf(data.split('\\|')[0]); + String order = data.split('\\|')[1]; + String name = data.split('\\|')[2]; + String sex = data.split('\\|')[3]; + // 将解析出的字段放到要输出的Row()对象 + Row row = new Row(4); + row.setField(0, ctime); + row.setField(1, age); + row.setField(2, grade); + row.setField(3, updateTime); + System.out.println("Kafka message str ==>" + row.toString()); + // 输出一行 + collect(row); + } catch (ClassCastException e) { + System.out.println("Input data format error. Input data " + msg + "is not json string"); + } + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + } + @Override + // 如果返回值是Row，就必须重载实现这个方法，显式地告诉系统返回的字段类型 + // 定义输出Row()对象的字段类型 + public DataType getResultType(Object[] arguments, Class[] argTypes) { + return DataTypes.createRowType(DataTypes.TIMESTAMP,DataTypes.STRING, DataTypes.Integer, DataTypes.STRING,DataTypes.STRING); + } + } + + package com.dp58; + package com.dp58.sql.udx; + import org.apache.flink.table.functions.FunctionContext; + import org.apache.flink.table.functions.ScalarFunction; + public class KafkaUDF extends ScalarFunction { + // 可选，open方法可以不写 + // 需要import org.apache.flink.table.functions.FunctionContext; + public String eval(byte[] message) { + // 读入一个二进制数据，并将其转换为String格式 + String msg = new String(message, "UTF-8"); + return msg.split('\\|')[0]; + } + public long eval(String b, String c) { + return eval(b) + eval(c); + } + //可选，close方法可以不写 + @Override + public void close() { + } + } + ``` diff --git a/docs/mongoSide.md b/docs/mongoSide.md index 73ce9644f..db557f441 100644 --- a/docs/mongoSide.md +++ b/docs/mongoSide.md @@ -40,6 +40,8 @@ |----|---|---|----| | type |表明 输出表类型 mongo|是|| | address | 连接mongo数据库 jdbcUrl |是|| + | userName | mongo连接用户名|否|| + | password | mongo连接密码|否|| | tableName | mongo表名称|是|| | database | mongo表名称|是|| | cache | 维表缓存策略(NONE/LRU)|否|NONE| @@ -62,8 +64,7 @@ create table sideTable( PERIOD FOR SYSTEM_TIME )WITH( type ='mongo', - //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] - address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', + address ='172.21.32.1:27017,172.21.32.1:27017', database ='test', tableName ='sidetest', cache ='LRU', diff --git a/docs/mongoSink.md b/docs/mongoSink.md index b5bb6df0d..b0f916aab 100644 --- a/docs/mongoSink.md +++ b/docs/mongoSink.md @@ -33,6 +33,8 @@ CREATE TABLE tableName( |----|----|----|----| |type |表明 输出表类型 mongo|是|| |address | 连接mongo数据库 jdbcUrl |是|| +|userName | mongo连接用户名|否|| +|password | mongo连接密码|否|| |tableName | mongo表名称|是|| |database | mongo表名称|是|| |parallelism | 并行度设置|否|1| @@ -44,8 +46,9 @@ CREATE TABLE MyResult( pv VARCHAR )WITH( type ='mongo', - //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] - address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', + address ='172.21.32.1:27017,172.21.32.1:27017', + userName ='dtstack', + password ='abc123', database ='test', tableName ='pv', parallelism ='1' diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 5ca81c5ed..491d1af27 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -20,7 +20,6 @@ package com.dtstack.flink.sql.sink.elasticsearch; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -34,6 +33,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -164,7 +164,7 @@ public void setBulkFlushMaxActions(int bulkFlushMaxActions) { } @Override - public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { ElasticsearchTableInfo elasticsearchTableInfo = (ElasticsearchTableInfo) targetTableInfo; esTableInfo = elasticsearchTableInfo; clusterName = elasticsearchTableInfo.getClusterName(); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 7988e597c..93682de46 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -32,7 +32,7 @@ * @author sishu.yss * @Company: www.dtstack.com */ -public class ElasticsearchSinkParser extends AbstractTableParser { +public class ElasticsearchSinkParser extends AbsTableParser { private static final String KEY_ES_ADDRESS = "address"; @@ -56,7 +56,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ElasticsearchTableInfo elasticsearchTableInfo = new ElasticsearchTableInfo(); elasticsearchTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearchTableInfo); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index f254d5b4e..9681742c2 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -33,7 +33,7 @@ * @author sishu.yss * @Company: www.dtstack.com */ -public class ElasticsearchTableInfo extends AbstractTargetTableInfo { +public class ElasticsearchTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "elasticsearch"; diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index b8cf2c46a..648f09fba 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -18,22 +18,24 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.BaseAllReqRow; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; + +import com.dtstack.flink.sql.side.AllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; import com.dtstack.flink.sql.side.elasticsearch6.util.SwitchUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -47,6 +49,7 @@ import java.io.IOException; import java.io.Serializable; +import java.sql.SQLException; import java.sql.Timestamp; import java.util.Calendar; import java.util.List; @@ -57,7 +60,7 @@ * @author yinxi * @date 2020/1/13 - 1:00 */ -public class Elasticsearch6AllReqRow extends BaseAllReqRow implements Serializable { +public class Elasticsearch6AllReqRow extends AllReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AllReqRow.class); @@ -67,7 +70,7 @@ public class Elasticsearch6AllReqRow extends BaseAllReqRow implements Serializab private SearchRequest searchRequest; private BoolQueryBuilder boolQueryBuilder; - public Elasticsearch6AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Elasticsearch6AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new Elasticsearch6AllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java index d353a583d..bf21b3ca8 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java @@ -18,13 +18,12 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; - +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; @@ -36,15 +35,15 @@ * @author yinxi * @date 2020/1/13 - 1:01 */ -public class Elasticsearch6AllSideInfo extends BaseSideInfo { +public class Elasticsearch6AllSideInfo extends SideInfo { - public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index 1f4bd1bf1..f67177fe4 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -18,12 +18,6 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; -import com.dtstack.flink.sql.side.CacheMissVal; -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.PredicateInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -32,6 +26,7 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.enums.ECacheContentType; +import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; @@ -62,14 +57,14 @@ * @author yinxi * @date 2020/2/13 - 13:10 */ -public class Elasticsearch6AsyncReqRow extends BaseAsyncReqRow implements Serializable { +public class Elasticsearch6AsyncReqRow extends AsyncReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AsyncReqRow.class); private transient RestHighLevelClient rhlClient; private SearchRequest searchRequest; private List sqlJoinCompareOperate = Lists.newArrayList(); - public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new Elasticsearch6AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); SqlNode conditionNode = joinInfo.getCondition(); ParseUtils.parseJoinCompareOperate(conditionNode, sqlJoinCompareOperate); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java index 4118988ab..7b3a2f9e7 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java @@ -18,17 +18,18 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; + import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -36,15 +37,15 @@ * @author yinxi * @date 2020/2/13 - 13:09 */ -public class Elasticsearch6AsyncSideInfo extends BaseSideInfo { +public class Elasticsearch6AsyncSideInfo extends SideInfo { - public Elasticsearch6AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Elasticsearch6AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java index 1b39bbf0f..13dfe2995 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.side.elasticsearch6.table; import com.dtstack.flink.sql.side.elasticsearch6.util.ClassUtil; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -30,7 +30,7 @@ * @author yinxi * @date 2020/1/13 - 1:07 */ -public class Elasticsearch6SideParser extends AbstractSideTableParser { +public class Elasticsearch6SideParser extends AbsSideTableParser { private static final String KEY_ES6_ADDRESS = "address"; @@ -55,7 +55,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { Elasticsearch6SideTableInfo elasticsearch6SideTableInfo = new Elasticsearch6SideTableInfo(); elasticsearch6SideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearch6SideTableInfo); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java index a01c90a53..0afe2d59e 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side.elasticsearch6.table; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.google.common.base.Preconditions; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -26,7 +26,7 @@ * @author yinxi * @date 2020/1/13 - 15:00 */ -public class Elasticsearch6SideTableInfo extends AbstractSideTableInfo { +public class Elasticsearch6SideTableInfo extends SideTableInfo { private static final String CURR_TYPE = "elasticsearch6"; diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java index 3d9e4d62a..cc591789e 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.side.elasticsearch6.util; -import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.PredicateInfo; +import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHost; @@ -98,7 +98,7 @@ public static RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh } // add index and type to search request - public static SearchRequest setSearchRequest(BaseSideInfo sideInfo) { + public static SearchRequest setSearchRequest(SideInfo sideInfo) { SearchRequest searchRequest = new SearchRequest(); Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); // determine existence of index @@ -129,7 +129,7 @@ public static SearchRequest setSearchRequest(BaseSideInfo sideInfo) { } // build where cause - public static BoolQueryBuilder setPredicateclause(BaseSideInfo sideInfo) { + public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { BoolQueryBuilder boolQueryBuilder = null; List predicateInfoes = sideInfo.getSideTableInfo().getPredicateInfoes(); @@ -144,7 +144,7 @@ public static BoolQueryBuilder setPredicateclause(BaseSideInfo sideInfo) { } // build filter condition - public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, BaseSideInfo sideInfo) { + public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, SideInfo sideInfo) { switch (info.getOperatorKind()) { case "IN": return boolQueryBuilder.must(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName(), sideInfo), removeSpaceAndApostrophe(info.getCondition()))); @@ -202,7 +202,7 @@ public static String[] removeSpaceAndApostrophe(String str) { } // prevent word segmentation - public static String textConvertToKeyword(String fieldName, BaseSideInfo sideInfo) { + public static String textConvertToKeyword(String fieldName, SideInfo sideInfo) { String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); String fieldType = sideFieldTypes[fieldIndex]; diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index b7d9de6fc..2646c50e9 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -18,7 +18,6 @@ package com.dtstack.flink.sql.sink.elasticsearch; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -32,6 +31,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.collect.Maps; import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; @@ -128,7 +128,7 @@ public void emitDataStream(DataStream> dataStream) { } @Override - public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { esTableInfo = (ElasticsearchTableInfo) targetTableInfo; clusterName = esTableInfo.getClusterName(); index = esTableInfo.getIndex(); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 22c2b72bc..8902b953c 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbstractTableInfo; -import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -29,7 +29,7 @@ * @author yinxi * @date 2020/1/9 - 15:06 */ -public class ElasticsearchSinkParser extends AbstractTableParser { +public class ElasticsearchSinkParser extends AbsTableParser { private static final String KEY_ES6_ADDRESS = "address"; @@ -55,7 +55,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ElasticsearchTableInfo elasticsearchTableInfo = new ElasticsearchTableInfo(); elasticsearchTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearchTableInfo); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index d0eef18f0..3cc3dd9ff 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -29,7 +29,7 @@ * @author yinxi * @date 2020/1/9 - 15:06 */ -public class ElasticsearchTableInfo extends AbstractTargetTableInfo { +public class ElasticsearchTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "elasticsearch6"; diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index ed2931fff..11380eea6 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -45,7 +45,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicReference; -public class HbaseAllReqRow extends BaseAllReqRow { +public class HbaseAllReqRow extends AllReqRow { private static final Logger LOG = LoggerFactory.getLogger(HbaseAllReqRow.class); @@ -55,7 +55,7 @@ public class HbaseAllReqRow extends BaseAllReqRow { private AtomicReference>> cacheRef = new AtomicReference<>(); - public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new HbaseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); tableName = ((HbaseSideTableInfo)sideTableInfo).getTableName(); @@ -134,7 +134,7 @@ public void flatMap(CRow input, Collector out) throws Exception { Map cacheList = null; - AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; if (hbaseSideTableInfo.isPreRowKey()) { for (Map.Entry> entry : cacheRef.get().entrySet()) { @@ -153,7 +153,7 @@ public void flatMap(CRow input, Collector out) throws Exception { } private void loadData(Map> tmpCache) throws SQLException { - AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; Configuration conf = new Configuration(); conf.set("hbase.zookeeper.quorum", hbaseSideTableInfo.getHost()); @@ -182,17 +182,9 @@ private void loadData(Map> tmpCache) throws SQLExcep LOG.error("", e); } finally { try { - if (null != conn && !conn.isClosed()) { - conn.close(); - } - - if (null != table) { - table.close(); - } - - if (null != resultScanner) { - resultScanner.close(); - } + conn.close(); + table.close(); + resultScanner.close(); } catch (IOException e) { LOG.error("", e); } diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java index 663c2927b..ea51f46e4 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,16 +31,16 @@ import java.util.List; -public class HbaseAllSideInfo extends BaseSideInfo { +public class HbaseAllSideInfo extends SideInfo { private RowKeyBuilder rowKeyBuilder; - public HbaseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public HbaseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { rowKeyBuilder = new RowKeyBuilder(); if(sideTableInfo.getPrimaryKeys().size() < 1){ throw new RuntimeException("Primary key dimension table must be filled"); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 24ee01786..251b88034 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -21,12 +21,12 @@ package com.dtstack.flink.sql.side.hbase; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.AsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; -import com.dtstack.flink.sql.side.hbase.rowkeydealer.AbstractRowKeyModeDealer; +import com.dtstack.flink.sql.side.hbase.rowkeydealer.AbsRowKeyModeDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.PreRowKeyModeDealerDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.RowKeyEqualModeDealer; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; @@ -58,7 +58,7 @@ * @author xuchao */ -public class HbaseAsyncReqRow extends BaseAsyncReqRow { +public class HbaseAsyncReqRow extends AsyncReqRow { private static final long serialVersionUID = 2098635104857937717L; @@ -73,13 +73,13 @@ public class HbaseAsyncReqRow extends BaseAsyncReqRow { private transient HBaseClient hBaseClient; - private transient AbstractRowKeyModeDealer rowKeyMode; + private transient AbsRowKeyModeDealer rowKeyMode; private String tableName; private String[] colNames; - public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new HbaseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); tableName = ((HbaseSideTableInfo)sideTableInfo).getTableName(); @@ -89,7 +89,7 @@ public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List colRefType; - public HbaseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public HbaseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { rowKeyBuilder = new RowKeyBuilder(); if(sideTableInfo.getPrimaryKeys().size() < 1){ throw new RuntimeException("Primary key dimension table must be filled"); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java similarity index 87% rename from hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java rename to hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java index 90ee289bd..1506bc440 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.side.hbase.rowkeydealer; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbstractSideCache; +import com.dtstack.flink.sql.side.cache.AbsSideCache; import org.apache.calcite.sql.JoinType; import com.google.common.collect.Maps; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -41,7 +41,7 @@ * @author xuchao */ -public abstract class AbstractRowKeyModeDealer { +public abstract class AbsRowKeyModeDealer { protected Map colRefType; @@ -60,9 +60,9 @@ public abstract class AbstractRowKeyModeDealer { protected Map sideFieldIndex = Maps.newHashMap(); - public AbstractRowKeyModeDealer(Map colRefType, String[] colNames, HBaseClient hBaseClient, - boolean openCache, JoinType joinType, List outFieldInfoList, - Map inFieldIndex, Map sideFieldIndex){ + public AbsRowKeyModeDealer(Map colRefType, String[] colNames, HBaseClient hBaseClient, + boolean openCache, JoinType joinType, List outFieldInfoList, + Map inFieldIndex, Map sideFieldIndex){ this.colRefType = colRefType; this.colNames = colNames; this.hBaseClient = hBaseClient; @@ -111,5 +111,5 @@ protected Row fillData(Row input, Object sideInput){ } public abstract void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbstractSideCache sideCache); + AbsSideCache sideCache); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java index dcd50131d..a41ffe916 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbstractSideCache; +import com.dtstack.flink.sql.side.cache.AbsSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.google.common.collect.Maps; @@ -44,6 +44,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -54,7 +55,7 @@ * @author xuchao */ -public class PreRowKeyModeDealerDealer extends AbstractRowKeyModeDealer { +public class PreRowKeyModeDealerDealer extends AbsRowKeyModeDealer { private static final Logger LOG = LoggerFactory.getLogger(PreRowKeyModeDealerDealer.class); @@ -66,7 +67,7 @@ public PreRowKeyModeDealerDealer(Map colRefType, String[] colNam @Override public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbstractSideCache sideCache) { + AbsSideCache sideCache) { Scanner prefixScanner = hBaseClient.newScanner(tableName); ScanFilter scanFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(Bytes.UTF8(rowKeyStr))); prefixScanner.setFilter(scanFilter); @@ -79,7 +80,7 @@ public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultF } - private String dealOneRow(ArrayList> args, String rowKeyStr, CRow input, ResultFuture resultFuture, AbstractSideCache sideCache) { + private String dealOneRow(ArrayList> args, String rowKeyStr, CRow input, ResultFuture resultFuture, AbsSideCache sideCache) { if(args == null || args.size() == 0){ dealMissKey(input, resultFuture); if (openCache) { @@ -110,7 +111,8 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, for (String key : colNames) { Object val = sideMap.get(key); if (val == null) { - LOG.error("can't get data with column {}", key); + System.out.println("can't get data with column " + key); + LOG.error("can't get data with column " + key); } sideVal.add(val); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index 41208c7f3..b20c316db 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbstractSideCache; +import com.dtstack.flink.sql.side.cache.AbsSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.google.common.collect.Maps; @@ -49,7 +49,7 @@ * @author xuchao */ -public class RowKeyEqualModeDealer extends AbstractRowKeyModeDealer { +public class RowKeyEqualModeDealer extends AbsRowKeyModeDealer { private static final Logger LOG = LoggerFactory.getLogger(RowKeyEqualModeDealer.class); @@ -62,7 +62,7 @@ public RowKeyEqualModeDealer(Map colRefType, String[] colNames, @Override public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbstractSideCache sideCache){ + AbsSideCache sideCache){ //TODO 是否有查询多个col family 和多个col的方法 GetRequest getRequest = new GetRequest(tableName, rowKeyStr); hBaseClient.get(getRequest).addCallbacks(arg -> { @@ -86,7 +86,8 @@ public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultF for(String key : colNames){ Object val = sideMap.get(key); if(val == null){ - LOG.error("can't get data with column {}", key); + System.out.println("can't get data with column " + key); + LOG.error("can't get data with column " + key); } sideVal.add(val); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java index ab826c98f..af5e24dd6 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java @@ -30,7 +30,7 @@ * @author xuchao */ -public class Md5ReplaceOperator extends AbstractReplaceOperator { +public class Md5ReplaceOperator extends ReplaceOperator{ public Md5ReplaceOperator(EReplaceOpType opType) { super(opType); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java similarity index 93% rename from hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java rename to hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java index c75fa43c1..c3c9b5cfe 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java @@ -31,11 +31,11 @@ * @author xuchao */ -public abstract class AbstractReplaceOperator { +public abstract class ReplaceOperator { private EReplaceOpType opType; - public AbstractReplaceOperator(EReplaceOpType opType){ + public ReplaceOperator(EReplaceOpType opType){ this.opType = opType; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java index ac9a3b87c..834edf9ae 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java @@ -28,12 +28,5 @@ */ public enum EReplaceOpType { - /** - * 没有func - */ - NO_FUNC, - /** - * md5 func - */ - MD5_FUNC; + NO_FUNC, MD5_FUNC; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java index bb4b1556b..f4b039ef8 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java @@ -28,16 +28,7 @@ * @author xuchao */ public enum EReplaceType { - /** - * 参数 - */ PARAM, - /** - * 函数 - */ FUNC, - /** - * 常量 - */ CONSTANT; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index cf44c181f..7b627da4d 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -20,15 +20,15 @@ package com.dtstack.flink.sql.side.hbase.table; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; /** * hbase field information must include the definition of an alias -> sql which does not allow ":" @@ -37,7 +37,7 @@ * @author xuchao */ -public class HbaseSideParser extends AbstractSideTableParser { +public class HbaseSideParser extends AbsSideTableParser { private final static String FIELD_KEY = "fieldKey"; @@ -58,7 +58,7 @@ public HbaseSideParser() { } @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseSideTableInfo hbaseTableInfo = new HbaseSideTableInfo(); hbaseTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, hbaseTableInfo); @@ -77,7 +77,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map { +public class HbaseOutputFormat extends DtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(HbaseOutputFormat.class); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index 9308725a2..447b02921 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.hbase.table.HbaseTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -30,6 +30,7 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; +import org.apache.flink.table.sinks.AppendStreamTableSink; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; @@ -58,7 +59,7 @@ public HbaseSink() { } @Override - public HbaseSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public HbaseSink genStreamSink(TargetTableInfo targetTableInfo) { HbaseTableInfo hbaseTableInfo = (HbaseTableInfo) targetTableInfo; this.zookeeperQuorum = hbaseTableInfo.getHost(); this.port = hbaseTableInfo.getPort(); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index 8d8996a23..064b13e69 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.enums.EUpdateMode; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -32,14 +32,14 @@ import java.util.List; import java.util.Map; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; /** * Date: 2018/09/14 * Company: www.dtstack.com * @author sishu.yss */ -public class HbaseSinkParser extends AbstractTableParser { +public class HbaseSinkParser extends AbsTableParser { public static final String HBASE_ZOOKEEPER_QUORUM = "zookeeperQuorum"; @@ -60,7 +60,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseTableInfo hbaseTableInfo = new HbaseTableInfo(); hbaseTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, hbaseTableInfo); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java index 62f7c9936..610eb5c30 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.sink.hbase.table; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; import java.util.Map; @@ -30,7 +30,7 @@ * Company: www.dtstack.com * @author sishu.yss */ -public class HbaseTableInfo extends AbstractTargetTableInfo { +public class HbaseTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "hbase"; diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java index 10938308a..9cd8c9194 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java @@ -20,10 +20,12 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.JDBCUtils; +import com.google.common.collect.Maps; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -34,6 +36,7 @@ import java.sql.Connection; import java.sql.DriverManager; import java.util.List; +import java.util.Map; /** * side operator with cache for all(period reload) @@ -43,7 +46,7 @@ * @author xiuzhu */ -public class ImpalaAllReqRow extends AbstractRdbAllReqRow { +public class ImpalaAllReqRow extends RdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -53,13 +56,13 @@ public class ImpalaAllReqRow extends AbstractRdbAllReqRow { private ImpalaSideTableInfo impalaSideTableInfo; - public ImpalaAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ImpalaAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new ImpalaAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); this.impalaSideTableInfo = (ImpalaSideTableInfo) sideTableInfo; } @Override - public Connection getConn(String dbUrl, String userName, String password) { + public Connection getConn(String dbURL, String userName, String password) { try { Connection connection ; String url = getUrl(); diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java index a5e643967..d40c5f48c 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java @@ -20,9 +20,10 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -31,7 +32,7 @@ public class ImpalaAllSideInfo extends RdbAllSideInfo { - public ImpalaAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ImpalaAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -60,7 +61,7 @@ private String buildPartitionCondition(ImpalaSideTableInfo impalaSideTableInfo) private String getPartitionVaule(String fieldType, List values) { String partitionVaule = values.stream().map(val -> { - return ("string".equals(fieldType.toLowerCase()) || "varchar".equals(fieldType.toLowerCase())) ? "'" + val + "'" : val.toString(); + return (fieldType.toLowerCase().equals("string") || fieldType.toLowerCase().equals("varchar")) ? "'" + val + "'" : val.toString(); }).collect(Collectors.joining(" , ")).toString(); return partitionVaule; diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index b5e0ed5fb..725e7c3ee 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import io.vertx.core.Vertx; @@ -50,19 +50,30 @@ public class ImpalaAsyncReqRow extends RdbAsyncReqRow { private final static String IMPALA_DRIVER = "com.cloudera.impala.jdbc41.Driver"; - public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new ImpalaAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override public void open(Configuration parameters) throws Exception { super.open(parameters); - ImpalaSideTableInfo impalaSideTableInfo = (ImpalaSideTableInfo) sideInfo.getSideTableInfo(); + JsonObject impalaClientConfig = getClientConfig(); + + System.setProperty("vertx.disableFileCPResolving", "true"); + + VertxOptions vo = new VertxOptions(); + vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setFileResolverCachingEnabled(false); + Vertx vertx = Vertx.vertx(vo); + setRdbSQLClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); + } + public JsonObject getClientConfig() { JsonObject impalaClientConfig = new JsonObject(); impalaClientConfig.put("url", getUrl()) .put("driver_class", IMPALA_DRIVER) - .put("max_pool_size", impalaSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("provider_class", DT_PROVIDER_CLASS) .put("idle_connection_test_period", 300) .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN) @@ -71,17 +82,9 @@ public void open(Configuration parameters) throws Exception { .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); - System.setProperty("vertx.disableFileCPResolving", "true"); - - VertxOptions vo = new VertxOptions(); - vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(impalaSideTableInfo.getAsyncPoolSize()); - vo.setFileResolverCachingEnabled(false); - Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); + return impalaClientConfig; } - public String getUrl() { ImpalaSideTableInfo impalaSideTableInfo = (ImpalaSideTableInfo) sideInfo.getSideTableInfo(); diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java index 14fbf0ed3..53559b9a7 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -38,7 +38,7 @@ public class ImpalaAsyncSideInfo extends RdbAsyncSideInfo { - public ImpalaAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ImpalaAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -69,7 +69,7 @@ private String buildPartitionCondition(ImpalaSideTableInfo impalaSideTableInfo) private String getPartitionVaule(String fieldType, List values) { String partitionVaule = values.stream().map(val -> { - return ("string".equals(fieldType.toLowerCase()) || "varchar".equals(fieldType.toLowerCase())) ? "'" + val + "'" : val.toString(); + return (fieldType.toLowerCase().equals("string") || fieldType.toLowerCase().equals("varchar")) ? "'" + val + "'" : val.toString(); }).collect(Collectors.joining(" , ")).toString(); return partitionVaule; diff --git a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java index 2f1d73be7..b6a54ca3f 100644 --- a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java +++ b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.impala.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang3.StringUtils; @@ -48,7 +48,7 @@ public class ImpalaSideParser extends RdbSideParser { private static final String CURR_TYPE = "impala"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ImpalaSideTableInfo impalaSideTableInfo = new ImpalaSideTableInfo(); impalaSideTableInfo.setType(CURR_TYPE); impalaSideTableInfo.setName(tableName); @@ -147,8 +147,6 @@ public Class dbTypeConvertToJavaType(String fieldType) { return String.class; case "timestamp": return Timestamp.class; - default: - break; } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index cd1e1e945..41fa00926 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -21,9 +21,9 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.impala.table.ImpalaTableInfo; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -36,7 +36,7 @@ * @author xiuzhu */ -public class ImpalaSink extends AbstractRdbSink implements IStreamSinkGener { +public class ImpalaSink extends RdbSink implements IStreamSinkGener { private ImpalaTableInfo impalaTableInfo; @@ -47,7 +47,7 @@ public ImpalaSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(getImpalaJdbcUrl()) + .setDBUrl(getImpalaJdbcUrl()) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) @@ -70,8 +70,8 @@ public JDBCUpsertOutputFormat getOutputFormat() { public String getImpalaJdbcUrl() { Integer authMech = impalaTableInfo.getAuthMech(); - String newUrl = dbUrl; - StringBuffer urlBuffer = new StringBuffer(dbUrl); + String newUrl = dbURL; + StringBuffer urlBuffer = new StringBuffer(dbURL); if (authMech == EAuthMech.NoAuthentication.getType()) { return newUrl; } else if (authMech == EAuthMech.Kerberos.getType()) { @@ -121,7 +121,7 @@ public String getImpalaJdbcUrl() { } @Override - public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { super.genStreamSink(targetTableInfo); this.impalaTableInfo = (ImpalaTableInfo) targetTableInfo; return this; diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java index 5b6fdeafe..4921f5e51 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java @@ -19,11 +19,12 @@ package com.dtstack.flink.sql.sink.impala.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; import java.sql.Timestamp; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -41,7 +42,7 @@ public class ImpalaSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "impala"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ImpalaTableInfo impalaTableInfo = new ImpalaTableInfo(); impalaTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, impalaTableInfo); @@ -116,8 +117,6 @@ public Class dbTypeConvertToJavaType(String fieldType) { return String.class; case "timestamp": return Timestamp.class; - default: - break; } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java index 9958a2544..88c2ca939 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java @@ -19,16 +19,16 @@ import com.dtstack.flink.sql.format.FormatType; import com.dtstack.flink.sql.format.SerializationMetricWrapper; -import com.dtstack.flink.sql.sink.kafka.serialization.AvroCRowSerializationSchema; -import com.dtstack.flink.sql.sink.kafka.serialization.CsvCRowSerializationSchema; -import com.dtstack.flink.sql.sink.kafka.serialization.JsonCRowSerializationSchema; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.formats.avro.AvroRowSerializationSchema; +import org.apache.flink.formats.csv.CsvRowSerializationSchema; +import org.apache.flink.formats.json.JsonRowSerializationSchema; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -51,37 +51,42 @@ public abstract class AbstractKafkaProducerFactory { * @param partitioner * @return */ - public abstract RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys); + public abstract RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys); - protected SerializationMetricWrapper createSerializationMetricWrapper(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { - SerializationSchema serializationSchema = createSerializationSchema(kafkaSinkTableInfo, typeInformation); - return new SerializationMetricWrapper(serializationSchema); + protected SerializationMetricWrapper createSerializationMetricWrapper(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { + return new SerializationMetricWrapper(createSerializationSchema(kafkaSinkTableInfo, typeInformation)); } - private SerializationSchema createSerializationSchema(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { - SerializationSchema serializationSchema = null; + private SerializationSchema createSerializationSchema(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { + SerializationSchema serializationSchema = null; if (FormatType.JSON.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { + if (StringUtils.isNotBlank(kafkaSinkTableInfo.getSchemaString())) { - serializationSchema = new JsonCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(), kafkaSinkTableInfo.getUpdateMode()); + serializationSchema = new JsonRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); } else if (typeInformation != null && typeInformation.getArity() != 0) { - serializationSchema = new JsonCRowSerializationSchema(typeInformation, kafkaSinkTableInfo.getUpdateMode()); + serializationSchema = new JsonRowSerializationSchema(typeInformation); } else { throw new IllegalArgumentException("sinkDataType:" + FormatType.JSON.name() + " must set schemaString（JSON Schema）or TypeInformation"); } + } else if (FormatType.CSV.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { + if (StringUtils.isBlank(kafkaSinkTableInfo.getFieldDelimiter())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.CSV.name() + " must set fieldDelimiter"); } - final CsvCRowSerializationSchema.Builder serSchemaBuilder = new CsvCRowSerializationSchema.Builder(typeInformation); - serSchemaBuilder.setFieldDelimiter(kafkaSinkTableInfo.getFieldDelimiter().toCharArray()[0]); - serSchemaBuilder.setUpdateMode(kafkaSinkTableInfo.getUpdateMode()); + final CsvRowSerializationSchema.Builder serSchemaBuilder = new CsvRowSerializationSchema.Builder(typeInformation); + serSchemaBuilder.setFieldDelimiter(kafkaSinkTableInfo.getFieldDelimiter().toCharArray()[0]); serializationSchema = serSchemaBuilder.build(); + } else if (FormatType.AVRO.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { + if (StringUtils.isBlank(kafkaSinkTableInfo.getSchemaString())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.AVRO.name() + " must set schemaString"); } - serializationSchema = new AvroCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(), kafkaSinkTableInfo.getUpdateMode()); + + serializationSchema = new AvroRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); + } if (null == serializationSchema) { diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java deleted file mode 100644 index 7234216a7..000000000 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka; - -import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.functions.sink.SinkFunction; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.runtime.types.CRowTypeInfo; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; -import org.apache.kafka.clients.consumer.ConsumerConfig; - -import java.util.Optional; -import java.util.Properties; -import java.util.stream.IntStream; - -/** - * Date: 2020/4/1 - * Company: www.dtstack.com - * @author maqi - */ -public abstract class AbstractKafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - public static final String SINK_OPERATOR_NAME_TPL = "${topic}_${table}"; - - protected String[] fieldNames; - protected TypeInformation[] fieldTypes; - - protected String[] partitionKeys; - protected String sinkOperatorName; - protected Properties properties; - protected int parallelism; - protected String topic; - protected String tableName; - - protected TableSchema schema; - protected SinkFunction kafkaProducer; - - - protected Optional> partitioner; - - protected Properties getKafkaProperties(KafkaSinkTableInfo KafkaSinkTableInfo) { - Properties props = new Properties(); - props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KafkaSinkTableInfo.getBootstrapServers()); - - for (String key : KafkaSinkTableInfo.getKafkaParamKeys()) { - props.setProperty(key, KafkaSinkTableInfo.getKafkaParam(key)); - } - return props; - } - - protected TypeInformation[] getTypeInformations(KafkaSinkTableInfo kafka11SinkTableInfo) { - Class[] fieldClasses = kafka11SinkTableInfo.getFieldClasses(); - TypeInformation[] types = IntStream.range(0, fieldClasses.length) - .mapToObj(i -> TypeInformation.of(fieldClasses[i])) - .toArray(TypeInformation[]::new); - return types; - } - - - protected TableSchema buildTableSchema(String[] fieldNames, TypeInformation[] fieldTypes) { - Preconditions.checkArgument(fieldNames.length == fieldTypes.length, "fieldNames length must equals fieldTypes length !"); - - TableSchema.Builder builder = TableSchema.builder(); - IntStream.range(0, fieldTypes.length) - .forEach(i -> builder.field(fieldNames[i], fieldTypes[i])); - - return builder.build(); - } - - - @Override - public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream - .map((Tuple2 record) -> new CRow(record.f1, record.f0)) - .returns(getRowTypeInfo()) - .setParallelism(parallelism); - - mapDataStream.addSink(kafkaProducer).name(sinkOperatorName); - } - - public CRowTypeInfo getRowTypeInfo() { - return new CRowTypeInfo(new RowTypeInfo(fieldTypes, fieldNames)); - } - - protected String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo) { - if (StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())) { - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - -} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index 90dfe996b..e212d1f57 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -11,13 +11,13 @@ public class CustomerFlinkPartition extends FlinkKafkaPartitioner { public CustomerFlinkPartition() { } - @Override + public void open(int parallelInstanceId, int parallelInstances) { Preconditions.checkArgument(parallelInstanceId >= 0, "Id of this subtask cannot be negative."); Preconditions.checkArgument(parallelInstances > 0, "Number of subtasks must be larger than 0."); this.parallelInstanceId = parallelInstanceId; } - @Override + public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); if(key == null){ @@ -25,11 +25,11 @@ public int partition(T record, byte[] key, byte[] value, String targetTopic, int } return partitions[Math.abs(new String(key).hashCode()) % partitions.length]; } - @Override + public boolean equals(Object o) { return this == o || o instanceof CustomerFlinkPartition; } - @Override + public int hashCode() { return CustomerFlinkPartition.class.hashCode(); } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index cde8d1b1d..498766564 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -2,20 +2,18 @@ import com.dtstack.flink.sql.format.SerializationMetricWrapper; -import com.dtstack.flink.sql.sink.kafka.serialization.JsonCRowSerializationSchema; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.formats.json.JsonRowSerializationSchema; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema; -import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.concurrent.atomic.AtomicLong; -public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { +public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { private static final Logger LOG = LoggerFactory.getLogger(CustomerKeyedSerializationSchema.class); @@ -32,41 +30,38 @@ public CustomerKeyedSerializationSchema(SerializationMetricWrapper serialization this.mapper = new ObjectMapper(); } - @Override - public byte[] serializeKey(CRow element) { - if (partitionKeys == null || partitionKeys.length <= 0) { + public byte[] serializeKey(Row element) { + if(partitionKeys == null || partitionKeys.length <=0){ return null; - } - SerializationSchema serializationSchema = serializationMetricWrapper.getSerializationSchema(); - if (serializationSchema instanceof JsonCRowSerializationSchema) { - return serializeJsonKey((JsonCRowSerializationSchema) serializationSchema, element); + } + SerializationSchema serializationSchema = serializationMetricWrapper.getSerializationSchema(); + if(serializationSchema instanceof JsonRowSerializationSchema){ + return serializeJsonKey((JsonRowSerializationSchema) serializationSchema, element); } return null; } - @Override - public byte[] serializeValue(CRow element) { + public byte[] serializeValue(Row element) { return this.serializationMetricWrapper.serialize(element); } - @Override - public String getTargetTopic(CRow element) { + public String getTargetTopic(Row element) { return null; } - private byte[] serializeJsonKey(JsonCRowSerializationSchema jsonCRowSerializationSchema, CRow element) { + private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationSchema, Row element) { try { - byte[] data = jsonCRowSerializationSchema.serialize(element); + byte[] data = jsonRowSerializationSchema.serialize(element); ObjectNode objectNode = mapper.readValue(data, ObjectNode.class); StringBuilder sb = new StringBuilder(); - for (String key : partitionKeys) { - if (objectNode.has(key)) { + for(String key : partitionKeys){ + if(objectNode.has(key)){ sb.append(objectNode.get(key.trim())); } } return sb.toString().getBytes(); - } catch (Exception e) { - if (COUNTER.getAndIncrement() % 1000 == 0) { + } catch (Exception e){ + if(COUNTER.getAndIncrement() % 1000 == 0){ LOG.error("serializeJsonKey error", e); } } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java deleted file mode 100644 index 692e208b5..000000000 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java +++ /dev/null @@ -1,366 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka.serialization; - -import com.dtstack.flink.sql.enums.EUpdateMode; -import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; -import org.apache.avro.SchemaParseException; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.Encoder; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.specific.SpecificData; -import org.apache.avro.specific.SpecificDatumWriter; -import org.apache.avro.specific.SpecificRecord; -import org.apache.avro.util.Utf8; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.sql.Date; -import java.sql.Time; -import java.sql.Timestamp; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.TimeZone; -import java.util.stream.Collectors; - -/** - * Serialization schema that serializes CROW into Avro bytes. - * - *

Serializes objects that are represented in (nested) Flink rows. It support types that - * are compatible with Flink's Table & SQL API. - ** - * @author maqi - */ -public class AvroCRowSerializationSchema implements SerializationSchema { - - /** - * Used for time conversions from SQL types. - */ - private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); - - /** - * Avro record class for serialization. Might be null if record class is not available. - */ - private Class recordClazz; - - /** - * Schema string for deserialization. - */ - private String schemaString; - - /** - * Avro serialization schema. - */ - private transient Schema schema; - - /** - * Writer to serialize Avro record into a byte array. - */ - private transient DatumWriter datumWriter; - - /** - * Output stream to serialize records into byte array. - */ - private transient ByteArrayOutputStream arrayOutputStream; - - /** - * Low-level class for serialization of Avro values. - */ - private transient Encoder encoder; - - private String updateMode; - - private String retractKey = "retract"; - - /** - * Creates an Avro serialization schema for the given specific record class. - * - * @param recordClazz Avro record class used to serialize Flink's row to Avro's record - */ - public AvroCRowSerializationSchema(Class recordClazz, String updateMode) { - Preconditions.checkNotNull(recordClazz, "Avro record class must not be null."); - this.recordClazz = recordClazz; - this.schema = SpecificData.get().getSchema(recordClazz); - this.schemaString = schema.toString(); - this.datumWriter = new SpecificDatumWriter<>(schema); - this.arrayOutputStream = new ByteArrayOutputStream(); - this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); - this.updateMode = updateMode; - } - - /** - * Creates an Avro serialization schema for the given Avro schema string. - * - * @param avroSchemaString Avro schema string used to serialize Flink's row to Avro's record - */ - public AvroCRowSerializationSchema(String avroSchemaString,String updateMode) { - Preconditions.checkNotNull(avroSchemaString, "Avro schema must not be null."); - this.recordClazz = null; - this.schemaString = avroSchemaString; - try { - this.schema = new Schema.Parser().parse(avroSchemaString); - } catch (SchemaParseException e) { - throw new IllegalArgumentException("Could not parse Avro schema string.", e); - } - this.datumWriter = new GenericDatumWriter<>(schema); - this.arrayOutputStream = new ByteArrayOutputStream(); - this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); - this.updateMode = updateMode; - } - - @Override - public byte[] serialize(CRow crow) { - try { - Row row = crow.row(); - boolean change = crow.change(); - - // convert to record - final GenericRecord record = convertRowToAvroRecord(schema, row); - - dealRetractField(change, record); - - arrayOutputStream.reset(); - datumWriter.write(record, encoder); - encoder.flush(); - return arrayOutputStream.toByteArray(); - } catch (Exception e) { - throw new RuntimeException("Failed to serialize row.", e); - } - } - - protected void dealRetractField(boolean change, GenericRecord record) { - schema.getFields() - .stream() - .filter(field -> StringUtils.equalsIgnoreCase(field.name(), retractKey)) - .findFirst() - .ifPresent(field -> { - if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { - record.put(retractKey, convertFlinkType(field.schema(), change)); - } - }); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final AvroCRowSerializationSchema that = (AvroCRowSerializationSchema) o; - return Objects.equals(recordClazz, that.recordClazz) && Objects.equals(schemaString, that.schemaString); - } - - @Override - public int hashCode() { - return Objects.hash(recordClazz, schemaString); - } - - // -------------------------------------------------------------------------------------------- - - private GenericRecord convertRowToAvroRecord(Schema schema, Row row) { - - final List fields = schema.getFields() - .stream() - .filter(field -> !StringUtils.equalsIgnoreCase(field.name(), retractKey)) - .collect(Collectors.toList()); - - final int length = fields.size(); - final GenericRecord record = new GenericData.Record(schema); - for (int i = 0; i < length; i++) { - final Schema.Field field = fields.get(i); - record.put(i, convertFlinkType(field.schema(), row.getField(i))); - } - return record; - } - - private Object convertFlinkType(Schema schema, Object object) { - if (object == null) { - return null; - } - switch (schema.getType()) { - case RECORD: - if (object instanceof Row) { - return convertRowToAvroRecord(schema, (Row) object); - } - throw new IllegalStateException("Row expected but was: " + object.getClass()); - case ENUM: - return new GenericData.EnumSymbol(schema, object.toString()); - case ARRAY: - final Schema elementSchema = schema.getElementType(); - final Object[] array = (Object[]) object; - final GenericData.Array convertedArray = new GenericData.Array<>(array.length, schema); - for (Object element : array) { - convertedArray.add(convertFlinkType(elementSchema, element)); - } - return convertedArray; - case MAP: - final Map map = (Map) object; - final Map convertedMap = new HashMap<>(); - for (Map.Entry entry : map.entrySet()) { - convertedMap.put( - new Utf8(entry.getKey().toString()), - convertFlinkType(schema.getValueType(), entry.getValue())); - } - return convertedMap; - case UNION: - final List types = schema.getTypes(); - final int size = types.size(); - final Schema actualSchema; - if (size == 2 && types.get(0).getType() == Schema.Type.NULL) { - actualSchema = types.get(1); - } else if (size == 2 && types.get(1).getType() == Schema.Type.NULL) { - actualSchema = types.get(0); - } else if (size == 1) { - actualSchema = types.get(0); - } else { - // generic type - return object; - } - return convertFlinkType(actualSchema, object); - case FIXED: - // check for logical type - if (object instanceof BigDecimal) { - return new GenericData.Fixed( - schema, - convertFromDecimal(schema, (BigDecimal) object)); - } - return new GenericData.Fixed(schema, (byte[]) object); - case STRING: - return new Utf8(object.toString()); - case BYTES: - // check for logical type - if (object instanceof BigDecimal) { - return ByteBuffer.wrap(convertFromDecimal(schema, (BigDecimal) object)); - } - return ByteBuffer.wrap((byte[]) object); - case INT: - // check for logical types - if (object instanceof Date) { - return convertFromDate(schema, (Date) object); - } else if (object instanceof Time) { - return convertFromTime(schema, (Time) object); - } - return object; - case LONG: - // check for logical type - if (object instanceof Timestamp) { - return convertFromTimestamp(schema, (Timestamp) object); - } - return object; - case FLOAT: - case DOUBLE: - case BOOLEAN: - return object; - } - throw new RuntimeException("Unsupported Avro type:" + schema); - } - - private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) { - final LogicalType logicalType = schema.getLogicalType(); - if (logicalType instanceof LogicalTypes.Decimal) { - final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType; - // rescale to target type - final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY); - // byte array must contain the two's-complement representation of the - // unscaled integer value in big-endian byte order - return decimal.unscaledValue().toByteArray(); - } else { - throw new RuntimeException("Unsupported decimal type."); - } - } - - private int convertFromDate(Schema schema, Date date) { - final LogicalType logicalType = schema.getLogicalType(); - if (logicalType == LogicalTypes.date()) { - // adopted from Apache Calcite - final long time = date.getTime(); - final long converted = time + (long) LOCAL_TZ.getOffset(time); - return (int) (converted / 86400000L); - } else { - throw new RuntimeException("Unsupported date type."); - } - } - - private int convertFromTime(Schema schema, Time date) { - final LogicalType logicalType = schema.getLogicalType(); - if (logicalType == LogicalTypes.timeMillis()) { - // adopted from Apache Calcite - final long time = date.getTime(); - final long converted = time + (long) LOCAL_TZ.getOffset(time); - return (int) (converted % 86400000L); - } else { - throw new RuntimeException("Unsupported time type."); - } - } - - private long convertFromTimestamp(Schema schema, Timestamp date) { - final LogicalType logicalType = schema.getLogicalType(); - if (logicalType == LogicalTypes.timestampMillis()) { - // adopted from Apache Calcite - final long time = date.getTime(); - return time + (long) LOCAL_TZ.getOffset(time); - } else { - throw new RuntimeException("Unsupported timestamp type."); - } - } - - private void writeObject(ObjectOutputStream outputStream) throws IOException { - outputStream.writeObject(recordClazz); - outputStream.writeObject(schemaString); // support for null - outputStream.writeObject(retractKey); - outputStream.writeObject(updateMode); - } - - @SuppressWarnings("unchecked") - private void readObject(ObjectInputStream inputStream) throws ClassNotFoundException, IOException { - recordClazz = (Class) inputStream.readObject(); - schemaString = (String) inputStream.readObject(); - if (recordClazz != null) { - schema = SpecificData.get().getSchema(recordClazz); - } else { - schema = new Schema.Parser().parse(schemaString); - } - retractKey = (String) inputStream.readObject(); - updateMode = (String) inputStream.readObject(); - - datumWriter = new SpecificDatumWriter<>(schema); - arrayOutputStream = new ByteArrayOutputStream(); - encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); - } -} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java deleted file mode 100644 index 4e57b6f2a..000000000 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka.serialization; - -import com.dtstack.flink.sql.enums.EUpdateMode; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.annotation.PublicEvolving; -import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; -import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeinfo.Types; -import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.formats.csv.CsvRowDeserializationSchema; -import org.apache.flink.formats.csv.CsvRowSchemaConverter; -import org.apache.flink.formats.csv.CsvRowSerializationSchema; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectWriter; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvMapper; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvSchema; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.runtime.types.CRowTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; - -import java.io.Serializable; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.Arrays; -import java.util.Iterator; -import java.util.Objects; -import java.util.stream.IntStream; -import java.util.stream.Stream; - -/** - * Serialization schema that serializes an object of Flink types into a CSV bytes. - * - *

Serializes the input row into a {@link ObjectNode} and - * converts it into byte[]. - * - *

Result byte[] messages can be deserialized using {@link CsvRowDeserializationSchema}. - */ -@PublicEvolving -public final class CsvCRowSerializationSchema implements SerializationSchema { - - private static final long serialVersionUID = 2098447220136965L; - - /** Type information describing the input CSV data. */ - private final RowTypeInfo typeInfo; - - /** Runtime instance that performs the actual work. */ - private final RuntimeConverter runtimeConverter; - - /** CsvMapper used to write {@link JsonNode} into bytes. */ - private final CsvMapper csvMapper; - - /** Schema describing the input CSV data. */ - private CsvSchema csvSchema; - - /** Object writer used to write rows. It is configured by {@link CsvSchema}. */ - private ObjectWriter objectWriter; - - /** Reusable object node. */ - private transient ObjectNode root; - - private String updateMode; - - private String retractKey = "retract"; - - private CsvCRowSerializationSchema( - RowTypeInfo typeInfo, - CsvSchema csvSchema, - String updateMode) { - this.typeInfo = typeInfo; - this.runtimeConverter = createRowRuntimeConverter(typeInfo, true); - this.csvMapper = new CsvMapper(); - this.csvSchema = csvSchema; - this.updateMode = updateMode; - this.objectWriter = csvMapper.writer(csvSchema); - } - - /** - * A builder for creating a {@link CsvRowSerializationSchema}. - */ - @PublicEvolving - public static class Builder { - - private final RowTypeInfo typeInfo; - private CsvSchema csvSchema; - private String updateMode; - - /** - * Creates a {@link CsvRowSerializationSchema} expecting the given {@link TypeInformation}. - * - * @param typeInfo type information used to create schema. - */ - public Builder(TypeInformation typeInfo) { - Preconditions.checkNotNull(typeInfo, "Type information must not be null."); - - if (!(typeInfo instanceof CRowTypeInfo)) { - throw new IllegalArgumentException("Row type information expected."); - } - RowTypeInfo rowTypeInfo = ((CRowTypeInfo) typeInfo).rowType(); - this.typeInfo = rowTypeInfo; - this.csvSchema = CsvRowSchemaConverter.convert(rowTypeInfo); - } - - public Builder setFieldDelimiter(char c) { - this.csvSchema = this.csvSchema.rebuild().setColumnSeparator(c).build(); - return this; - } - - public Builder setLineDelimiter(String delimiter) { - Preconditions.checkNotNull(delimiter, "Delimiter must not be null."); - if (!delimiter.equals("\n") && !delimiter.equals("\r") && !delimiter.equals("\r\n")) { - throw new IllegalArgumentException( - "Unsupported new line delimiter. Only \\n, \\r, or \\r\\n are supported."); - } - this.csvSchema = this.csvSchema.rebuild().setLineSeparator(delimiter).build(); - return this; - } - - public Builder setArrayElementDelimiter(String delimiter) { - Preconditions.checkNotNull(delimiter, "Delimiter must not be null."); - this.csvSchema = this.csvSchema.rebuild().setArrayElementSeparator(delimiter).build(); - return this; - } - - public Builder setQuoteCharacter(char c) { - this.csvSchema = this.csvSchema.rebuild().setQuoteChar(c).build(); - return this; - } - - public Builder setEscapeCharacter(char c) { - this.csvSchema = this.csvSchema.rebuild().setEscapeChar(c).build(); - return this; - } - - public Builder setNullLiteral(String s) { - this.csvSchema = this.csvSchema.rebuild().setNullValue(s).build(); - return this; - } - - public Builder setUpdateMode(String updateMode) { - this.updateMode = updateMode; - return this; - } - - public CsvCRowSerializationSchema build() { - return new CsvCRowSerializationSchema( - typeInfo, - csvSchema, - updateMode); - } - } - - @Override - public byte[] serialize(CRow crow) { - Row row = crow.row(); - boolean change = crow.change(); - if (root == null) { - root = csvMapper.createObjectNode(); - } - try { - runtimeConverter.convert(csvMapper, root, row); - if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { - fillRetractField(row, change); - } - - return objectWriter.writeValueAsBytes(root); - } catch (Throwable t) { - throw new RuntimeException("Could not serialize row '" + row + "'.", t); - } - } - - protected void fillRetractField(Row row, boolean change) { - root.put(retractKey, change); - CsvSchema.Builder newBuilder = new CsvSchema.Builder(csvSchema); - - CsvSchema.Column retractColumn = new CsvSchema.Column(row.getArity(), retractKey, CsvSchema.ColumnType.BOOLEAN); - newBuilder.addColumn(retractColumn); - csvSchema = newBuilder.build(); - - this.objectWriter = csvMapper.writer(csvSchema); - - } - - @Override - public boolean equals(Object o) { - if (o == null || o.getClass() != this.getClass()) { - return false; - } - if (this == o) { - return true; - } - final CsvCRowSerializationSchema that = (CsvCRowSerializationSchema) o; - final CsvSchema otherSchema = that.csvSchema; - - return typeInfo.equals(that.typeInfo) && - csvSchema.getColumnSeparator() == otherSchema.getColumnSeparator() && - Arrays.equals(csvSchema.getLineSeparator(), otherSchema.getLineSeparator()) && - csvSchema.getArrayElementSeparator().equals(otherSchema.getArrayElementSeparator()) && - csvSchema.getQuoteChar() == otherSchema.getQuoteChar() && - csvSchema.getEscapeChar() == otherSchema.getEscapeChar() && - Arrays.equals(csvSchema.getNullValue(), otherSchema.getNullValue()); - } - - @Override - public int hashCode() { - return Objects.hash( - typeInfo, - csvSchema.getColumnSeparator(), - csvSchema.getLineSeparator(), - csvSchema.getArrayElementSeparator(), - csvSchema.getQuoteChar(), - csvSchema.getEscapeChar(), - csvSchema.getNullValue()); - } - - // -------------------------------------------------------------------------------------------- - - private interface RuntimeConverter extends Serializable { - JsonNode convert(CsvMapper csvMapper, ContainerNode container, Object obj); - } - - private static RuntimeConverter createRowRuntimeConverter(RowTypeInfo rowTypeInfo, boolean isTopLevel) { - final TypeInformation[] fieldTypes = rowTypeInfo.getFieldTypes(); - final String[] fieldNames = rowTypeInfo.getFieldNames(); - - final RuntimeConverter[] fieldConverters = createFieldRuntimeConverters(fieldTypes); - - return assembleRowRuntimeConverter(isTopLevel, fieldNames, fieldConverters); - } - - private static RuntimeConverter[] createFieldRuntimeConverters(TypeInformation[] fieldTypes) { - final RuntimeConverter[] fieldConverters = new RuntimeConverter[fieldTypes.length]; - for (int i = 0; i < fieldTypes.length; i++) { - fieldConverters[i] = createNullableRuntimeConverter(fieldTypes[i]); - } - return fieldConverters; - } - - private static RuntimeConverter assembleRowRuntimeConverter( - boolean isTopLevel, - String[] fieldNames, - RuntimeConverter[] fieldConverters) { - final int rowArity = fieldNames.length; - // top level reuses the object node container - if (isTopLevel) { - return (csvMapper, container, obj) -> { - final Row row = (Row) obj; - - validateArity(rowArity, row.getArity()); - - final ObjectNode objectNode = (ObjectNode) container; - for (int i = 0; i < rowArity; i++) { - objectNode.set( - fieldNames[i], - fieldConverters[i].convert(csvMapper, container, row.getField(i))); - } - return objectNode; - }; - } else { - return (csvMapper, container, obj) -> { - final Row row = (Row) obj; - - validateArity(rowArity, row.getArity()); - - final ArrayNode arrayNode = csvMapper.createArrayNode(); - for (int i = 0; i < rowArity; i++) { - arrayNode.add(fieldConverters[i].convert(csvMapper, arrayNode, row.getField(i))); - } - return arrayNode; - }; - } - } - - private static RuntimeConverter createNullableRuntimeConverter(TypeInformation info) { - final RuntimeConverter valueConverter = createRuntimeConverter(info); - return (csvMapper, container, obj) -> { - if (obj == null) { - return container.nullNode(); - } - return valueConverter.convert(csvMapper, container, obj); - }; - } - - private static RuntimeConverter createRuntimeConverter(TypeInformation info) { - if (info.equals(Types.VOID)) { - return (csvMapper, container, obj) -> container.nullNode(); - } else if (info.equals(Types.STRING)) { - return (csvMapper, container, obj) -> container.textNode((String) obj); - } else if (info.equals(Types.BOOLEAN)) { - return (csvMapper, container, obj) -> container.booleanNode((Boolean) obj); - } else if (info.equals(Types.BYTE)) { - return (csvMapper, container, obj) -> container.numberNode((Byte) obj); - } else if (info.equals(Types.SHORT)) { - return (csvMapper, container, obj) -> container.numberNode((Short) obj); - } else if (info.equals(Types.INT)) { - return (csvMapper, container, obj) -> container.numberNode((Integer) obj); - } else if (info.equals(Types.LONG)) { - return (csvMapper, container, obj) -> container.numberNode((Long) obj); - } else if (info.equals(Types.FLOAT)) { - return (csvMapper, container, obj) -> container.numberNode((Float) obj); - } else if (info.equals(Types.DOUBLE)) { - return (csvMapper, container, obj) -> container.numberNode((Double) obj); - } else if (info.equals(Types.BIG_DEC)) { - return (csvMapper, container, obj) -> container.numberNode((BigDecimal) obj); - } else if (info.equals(Types.BIG_INT)) { - return (csvMapper, container, obj) -> container.numberNode((BigInteger) obj); - } else if (info.equals(Types.SQL_DATE)) { - return (csvMapper, container, obj) -> container.textNode(obj.toString()); - } else if (info.equals(Types.SQL_TIME)) { - return (csvMapper, container, obj) -> container.textNode(obj.toString()); - } else if (info.equals(Types.SQL_TIMESTAMP)) { - return (csvMapper, container, obj) -> container.textNode(obj.toString()); - } else if (info instanceof RowTypeInfo){ - return createRowRuntimeConverter((RowTypeInfo) info, false); - } else if (info instanceof BasicArrayTypeInfo) { - return createObjectArrayRuntimeConverter(((BasicArrayTypeInfo) info).getComponentInfo()); - } else if (info instanceof ObjectArrayTypeInfo) { - return createObjectArrayRuntimeConverter(((ObjectArrayTypeInfo) info).getComponentInfo()); - } else if (info instanceof PrimitiveArrayTypeInfo && - ((PrimitiveArrayTypeInfo) info).getComponentType() == Types.BYTE) { - return createByteArrayRuntimeConverter(); - } - else { - throw new RuntimeException("Unsupported type information '" + info + "'."); - } - } - - private static RuntimeConverter createObjectArrayRuntimeConverter(TypeInformation elementType) { - final RuntimeConverter elementConverter = createNullableRuntimeConverter(elementType); - return (csvMapper, container, obj) -> { - final Object[] array = (Object[]) obj; - final ArrayNode arrayNode = csvMapper.createArrayNode(); - for (Object element : array) { - arrayNode.add(elementConverter.convert(csvMapper, arrayNode, element)); - } - return arrayNode; - }; - } - - private static RuntimeConverter createByteArrayRuntimeConverter() { - return (csvMapper, container, obj) -> container.binaryNode((byte[]) obj); - } - - private static void validateArity(int expected, int actual) { - if (expected != actual) { - throw new RuntimeException("Row length mismatch. " + expected + - " fields expected but was " + actual + "."); - } - } -} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java deleted file mode 100644 index bfe801d52..000000000 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka.serialization; - -import com.dtstack.flink.sql.enums.EUpdateMode; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; -import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeinfo.Types; -import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.formats.json.JsonRowSchemaConverter; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.runtime.types.CRowTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.sql.Time; -import java.sql.Timestamp; -import java.text.SimpleDateFormat; -import java.util.Objects; - -/** - * - * Serialization schema that serializes an object of Flink types into a JSON bytes. - * - *

Serializes the input Flink object into a JSON string and - * converts it into byte[]. - * - */ -public class JsonCRowSerializationSchema implements SerializationSchema { - - private static final long serialVersionUID = -2885556750743978636L; - - /** Type information describing the input type. */ - private final TypeInformation typeInfo; - - /** Object mapper that is used to create output JSON objects. */ - private final ObjectMapper mapper = new ObjectMapper(); - - /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone, without milliseconds). */ - private SimpleDateFormat timeFormat = new SimpleDateFormat("HH:mm:ss'Z'"); - - /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone). */ - private SimpleDateFormat timeFormatWithMillis = new SimpleDateFormat("HH:mm:ss.SSS'Z'"); - - /** Formatter for RFC 3339-compliant string representation of a timestamp value (with UTC timezone). */ - private SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - - /** Reusable object node. */ - private transient ObjectNode node; - - private String updateMode; - - private final String retractKey = "retract"; - - public JsonCRowSerializationSchema(String jsonSchema, String updateMode) { - this(JsonRowSchemaConverter.convert(jsonSchema), updateMode); - } - - /** - * Creates a JSON serialization schema for the given type information. - * - * @param typeInfo The field names of {@link Row} are used to map to JSON properties. - */ - public JsonCRowSerializationSchema(TypeInformation typeInfo, String updateMode) { - Preconditions.checkNotNull(typeInfo, "Type information"); - this.typeInfo = typeInfo; - this.updateMode = updateMode; - } - - - @Override - public byte[] serialize(CRow crow) { - Row row = crow.row(); - boolean change = crow.change(); - if (node == null) { - node = mapper.createObjectNode(); - } - - RowTypeInfo rowTypeInfo = ((CRowTypeInfo) typeInfo).rowType(); - try { - convertRow(node, rowTypeInfo, row); - if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { - node.put(retractKey, change); - } - return mapper.writeValueAsBytes(node); - } catch (Throwable t) { - throw new RuntimeException("Could not serialize row '" + row + "'. " + - "Make sure that the schema matches the input.", t); - } - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final JsonCRowSerializationSchema that = (JsonCRowSerializationSchema) o; - return Objects.equals(typeInfo, that.typeInfo); - } - - @Override - public int hashCode() { - return Objects.hash(typeInfo); - } - - // -------------------------------------------------------------------------------------------- - - private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) { - if (reuse == null) { - reuse = mapper.createObjectNode(); - } - final String[] fieldNames = info.getFieldNames(); - - final TypeInformation[] fieldTypes = info.getFieldTypes(); - - // validate the row - if (row.getArity() != fieldNames.length) { - throw new IllegalStateException(String.format( - "Number of elements in the row '%s' is different from number of field names: %d", row, fieldNames.length)); - } - - for (int i = 0; i < fieldNames.length; i++) { - final String name = fieldNames[i]; - - final JsonNode fieldConverted = convert(reuse, reuse.get(name), fieldTypes[i], row.getField(i)); - reuse.set(name, fieldConverted); - } - - return reuse; - } - - private JsonNode convert(ContainerNode container, JsonNode reuse, TypeInformation info, Object object) { - if (info == Types.VOID || object == null) { - return container.nullNode(); - } else if (info == Types.BOOLEAN) { - return container.booleanNode((Boolean) object); - } else if (info == Types.STRING) { - return container.textNode((String) object); - } else if (info == Types.BIG_DEC) { - // convert decimal if necessary - if (object instanceof BigDecimal) { - return container.numberNode((BigDecimal) object); - } - return container.numberNode(BigDecimal.valueOf(((Number) object).doubleValue())); - } else if (info == Types.BIG_INT) { - // convert integer if necessary - if (object instanceof BigInteger) { - return container.numberNode((BigInteger) object); - } - return container.numberNode(BigInteger.valueOf(((Number) object).longValue())); - } else if (info == Types.SQL_DATE) { - return container.textNode(object.toString()); - } else if (info == Types.SQL_TIME) { - final Time time = (Time) object; - // strip milliseconds if possible - if (time.getTime() % 1000 > 0) { - return container.textNode(timeFormatWithMillis.format(time)); - } - return container.textNode(timeFormat.format(time)); - } else if (info == Types.SQL_TIMESTAMP) { - return container.textNode(timestampFormat.format((Timestamp) object)); - } else if (info instanceof RowTypeInfo) { - if (reuse != null && reuse instanceof ObjectNode) { - return convertRow((ObjectNode) reuse, (RowTypeInfo) info, (Row) object); - } else { - return convertRow(null, (RowTypeInfo) info, (Row) object); - } - } else if (info instanceof ObjectArrayTypeInfo) { - if (reuse != null && reuse instanceof ArrayNode) { - return convertObjectArray((ArrayNode) reuse, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); - } else { - return convertObjectArray(null, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); - } - } else if (info instanceof BasicArrayTypeInfo) { - if (reuse != null && reuse instanceof ArrayNode) { - return convertObjectArray((ArrayNode) reuse, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); - } else { - return convertObjectArray(null, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); - } - } else if (info instanceof PrimitiveArrayTypeInfo && ((PrimitiveArrayTypeInfo) info).getComponentType() == Types.BYTE) { - return container.binaryNode((byte[]) object); - } else { - // for types that were specified without JSON schema - // e.g. POJOs - try { - return mapper.valueToTree(object); - } catch (IllegalArgumentException e) { - throw new IllegalStateException("Unsupported type information '" + info + "' for object: " + object, e); - } - } - } - - private ArrayNode convertObjectArray(ArrayNode reuse, TypeInformation info, Object[] array) { - if (reuse == null) { - reuse = mapper.createArrayNode(); - } else { - reuse.removeAll(); - } - - for (Object object : array) { - reuse.add(convert(reuse, null, info, object)); - } - return reuse; - } -} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index 4ad8947a8..8ba21b953 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -18,10 +18,9 @@ package com.dtstack.flink.sql.sink.kafka.table; -import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.format.FormatType; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -33,13 +32,12 @@ * @author DocLi * @modifyer maqi */ -public class KafkaSinkParser extends AbstractTableParser { +public class KafkaSinkParser extends AbsTableParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KafkaSinkTableInfo kafkaSinkTableInfo = new KafkaSinkTableInfo(); kafkaSinkTableInfo.setName(tableName); kafkaSinkTableInfo.setType(MathUtil.getString(props.get(KafkaSinkTableInfo.TYPE_KEY.toLowerCase()))); - parseFieldsInfo(fieldsInfo, kafkaSinkTableInfo); if (props.get(KafkaSinkTableInfo.SINK_DATA_TYPE) != null) { @@ -48,14 +46,11 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map kafkaParam = new HashMap(); @@ -70,8 +59,6 @@ public class KafkaSinkTableInfo extends AbstractTargetTableInfo { private String partitionKeys; - private String updateMode; - public void addKafkaParam(String key, String value) { kafkaParam.put(key, value); } @@ -84,6 +71,7 @@ public Set getKafkaParamKeys() { return kafkaParam.keySet(); } + public String getBootstrapServers() { return bootstrapServers; } @@ -116,40 +104,14 @@ public void setFieldDelimiter(String fieldDelimiter) { this.fieldDelimiter = fieldDelimiter; } - public String getUpdateMode() { - return updateMode; - } - - public void setUpdateMode(String updateMode) { - this.updateMode = updateMode; - } - @Override public boolean check() { Preconditions.checkNotNull(getType(), "kafka of type is required"); Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); Preconditions.checkNotNull(topic, "kafka of topic is required"); - - if (StringUtils.equalsIgnoreCase(getSinkDataType(), FormatType.AVRO.name())) { - avroParamCheck(); - } - return false; } - public void avroParamCheck() { - Preconditions.checkNotNull(schemaString, "avro type schemaInfo is required"); - if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { - Schema schema = new Schema.Parser().parse(schemaString); - schema.getFields() - .stream() - .filter(field -> StringUtils.equalsIgnoreCase(field.name(), RETRACT_FIELD_KEY)) - .findFirst() - .orElseThrow(() -> - new NullPointerException(String.valueOf("arvo upsert mode the retract attribute must be contained in schemaInfo field "))); - } - } - public String getEnableKeyPartition() { return enableKeyPartition; } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java deleted file mode 100644 index 852a381e2..000000000 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.source.kafka; - -import com.dtstack.flink.sql.source.IStreamSourceGener; -import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; -import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; -import org.apache.flink.table.api.Table; -import org.apache.flink.types.Row; -import org.apache.kafka.clients.consumer.ConsumerConfig; - -import java.util.Map; -import java.util.Properties; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -/** - * Date: 2020/3/20 - * Company: www.dtstack.com - * @author maqi - */ -public abstract class AbstractKafkaSource implements IStreamSourceGener

{ - - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - protected Properties getKafkaProperties(KafkaSourceTableInfo kafkaSourceTableInfo) { - Properties props = new Properties(); - props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaSourceTableInfo.getBootstrapServers()); - - if (DtStringUtil.isJson(kafkaSourceTableInfo.getOffsetReset())) { - props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, EKafkaOffset.NONE.name().toLowerCase()); - } else { - props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, kafkaSourceTableInfo.getOffsetReset()); - } - - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())) { - props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, kafkaSourceTableInfo.getGroupId()); - } - - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - return props; - } - - protected String generateOperatorName(String tabName, String topicName) { - return SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", tabName); - } - - protected TypeInformation getRowTypeInformation(KafkaSourceTableInfo kafkaSourceTableInfo) { - Class[] fieldClasses = kafkaSourceTableInfo.getFieldClasses(); - TypeInformation[] types = IntStream.range(0, fieldClasses.length) - .mapToObj(i -> TypeInformation.of(fieldClasses[i])) - .toArray(TypeInformation[]::new); - - return new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); - } - - protected void setStartPosition(String offset, String topicName, FlinkKafkaConsumerBase kafkaSrc) { - if (StringUtils.equalsIgnoreCase(offset, EKafkaOffset.EARLIEST.name())) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJson(offset)) { - Map specificStartupOffsets = buildOffsetMap(offset, topicName); - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } else { - kafkaSrc.setStartFromLatest(); - } - } - - /** - * kafka offset,eg.. {"0":12312,"1":12321,"2":12312} - * @param offsetJson - * @param topicName - * @return - */ - protected Map buildOffsetMap(String offsetJson, String topicName) { - try { - Properties properties = PluginUtil.jsonStrToObject(offsetJson, Properties.class); - Map offsetMap = PluginUtil.objectToMap(properties); - Map specificStartupOffsets = offsetMap - .entrySet() - .stream() - .collect(Collectors.toMap( - (Map.Entry entry) -> new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), - (Map.Entry entry) -> Long.valueOf(entry.getValue().toString())) - ); - - return specificStartupOffsets; - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + offsetJson); - } - } - -} diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java index f08287cec..afa950c5b 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java @@ -77,7 +77,7 @@ protected void beforeDeserialize() throws IOException { protected void registerPtMetric(AbstractFetcher fetcher) throws Exception { - Field consumerThreadField = getConsumerThreadField(fetcher); + Field consumerThreadField = fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); consumerThreadField.setAccessible(true); KafkaConsumerThread consumerThread = (KafkaConsumerThread) consumerThreadField.get(fetcher); @@ -115,14 +115,6 @@ public Long getValue() { } } - private Field getConsumerThreadField(AbstractFetcher fetcher) throws NoSuchFieldException { - try { - return fetcher.getClass().getDeclaredField("consumerThread"); - } catch (Exception e) { - return fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); - } - } - public void setFetcher(AbstractFetcher fetcher) { this.fetcher = fetcher; } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index bef86f10d..005bfb4c2 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -19,14 +19,11 @@ package com.dtstack.flink.sql.source.kafka.table; -import com.dtstack.flink.sql.format.FormatType; -import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; -import com.dtstack.flink.sql.table.AbstractSourceParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSourceParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import java.util.stream.Collectors; /** * Reason: @@ -35,38 +32,33 @@ * * @author sishu.yss */ -public class KafkaSourceParser extends AbstractSourceParser { +public class KafkaSourceParser extends AbsSourceParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { KafkaSourceTableInfo kafkaSourceTableInfo = new KafkaSourceTableInfo(); - parseFieldsInfo(fieldsInfo, kafkaSourceTableInfo); - kafkaSourceTableInfo.setName(tableName); kafkaSourceTableInfo.setType(MathUtil.getString(props.get(KafkaSourceTableInfo.TYPE_KEY.toLowerCase()))); + parseFieldsInfo(fieldsInfo, kafkaSourceTableInfo); + kafkaSourceTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(KafkaSourceTableInfo.PARALLELISM_KEY.toLowerCase()))); - kafkaSourceTableInfo.setBootstrapServers(MathUtil.getString(props.get(KafkaSourceTableInfo.BOOTSTRAPSERVERS_KEY.toLowerCase()))); + String bootstrapServer = MathUtil.getString(props.get(KafkaSourceTableInfo.BOOTSTRAPSERVERS_KEY.toLowerCase())); + if (bootstrapServer == null || bootstrapServer.trim().equals("")) { + throw new Exception("BootstrapServers can not be empty!"); + } else { + kafkaSourceTableInfo.setBootstrapServers(bootstrapServer); + } kafkaSourceTableInfo.setGroupId(MathUtil.getString(props.get(KafkaSourceTableInfo.GROUPID_KEY.toLowerCase()))); kafkaSourceTableInfo.setTopic(MathUtil.getString(props.get(KafkaSourceTableInfo.TOPIC_KEY.toLowerCase()))); kafkaSourceTableInfo.setOffsetReset(MathUtil.getString(props.get(KafkaSourceTableInfo.OFFSETRESET_KEY.toLowerCase()))); kafkaSourceTableInfo.setTopicIsPattern(MathUtil.getBoolean(props.get(KafkaSourceTableInfo.TOPICISPATTERN_KEY.toLowerCase()), false)); - kafkaSourceTableInfo.setOffsetReset(MathUtil.getString(props.getOrDefault(KafkaSourceTableInfo.OFFSETRESET_KEY.toLowerCase(), EKafkaOffset.LATEST.name().toLowerCase()))); - kafkaSourceTableInfo.setTopicIsPattern(MathUtil.getBoolean(props.get(KafkaSourceTableInfo.TOPICISPATTERN_KEY.toLowerCase()))); kafkaSourceTableInfo.setTimeZone(MathUtil.getString(props.get(KafkaSourceTableInfo.TIME_ZONE_KEY.toLowerCase()))); - - kafkaSourceTableInfo.setSchemaString(MathUtil.getString(props.get(KafkaSourceTableInfo.SCHEMA_STRING_KEY.toLowerCase()))); - kafkaSourceTableInfo.setFieldDelimiter(MathUtil.getString(props.getOrDefault(KafkaSourceTableInfo.CSV_FIELD_DELIMITER_KEY.toLowerCase(), "|"))); - kafkaSourceTableInfo.setSourceDataType(MathUtil.getString(props.getOrDefault(KafkaSourceTableInfo.SOURCE_DATA_TYPE_KEY.toLowerCase(), FormatType.DT_NEST.name()))); - - Map kafkaParams = props.keySet().stream() - .filter(key -> !key.isEmpty() && key.startsWith("kafka.")) - .collect(Collectors.toMap( - key -> key.substring(6), key -> props.get(key).toString()) - ); - - kafkaSourceTableInfo.addKafkaParam(kafkaParams); + for (String key : props.keySet()) { + if (!key.isEmpty() && key.startsWith("kafka.")) { + kafkaSourceTableInfo.addKafkaParam(key.substring(6), props.get(key).toString()); + } + } kafkaSourceTableInfo.check(); - return kafkaSourceTableInfo; } } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index c27eee376..4f4851335 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -17,10 +17,11 @@ */ + package com.dtstack.flink.sql.source.kafka.table; import com.dtstack.flink.sql.format.FormatType; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; import com.google.common.base.Preconditions; import java.util.HashMap; @@ -34,126 +35,136 @@ * @author sishu.yss */ -public class KafkaSourceTableInfo extends AbstractSourceTableInfo { - - public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; +public class KafkaSourceTableInfo extends SourceTableInfo { - public static final String TOPIC_KEY = "topic"; + public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; - public static final String TYPE_KEY = "type"; + public static final String TOPIC_KEY = "topic"; - public static final String GROUPID_KEY = "groupId"; + public static final String TYPE_KEY = "type"; - public static final String OFFSETRESET_KEY = "offsetReset"; + public static final String GROUPID_KEY = "groupId"; - public static final String TOPICISPATTERN_KEY = "topicIsPattern"; + public static final String OFFSETRESET_KEY = "offsetReset"; - public static final String SCHEMA_STRING_KEY = "schemaInfo"; + public static final String TOPICISPATTERN_KEY = "topicIsPattern"; - public static final String CSV_FIELD_DELIMITER_KEY = "fieldDelimiter"; + private String bootstrapServers; - public static final String SOURCE_DATA_TYPE_KEY = "sourceDataType"; + private String topic; - private String bootstrapServers; + private String groupId; - private String topic; + //latest, earliest + private String offsetReset = "latest"; - private String groupId; + private String offset; - private String offsetReset; + private Boolean topicIsPattern = false; - private Boolean topicIsPattern = false; + private String sourceDataType = FormatType.DT_NEST.name(); - private String sourceDataType; + private String schemaString; - private String schemaString; + private String fieldDelimiter; - private String fieldDelimiter; + public String getBootstrapServers() { + return bootstrapServers; + } - public Map kafkaParam = new HashMap<>(); + public void setBootstrapServers(String bootstrapServers) { + this.bootstrapServers = bootstrapServers; + } + public String getTopic() { + return topic; + } - public String getBootstrapServers() { - return bootstrapServers; - } + public void setTopic(String topic) { + this.topic = topic; + } - public void setBootstrapServers(String bootstrapServers) { - this.bootstrapServers = bootstrapServers; - } + public String getGroupId() { + return groupId; + } - public String getTopic() { - return topic; - } + public void setGroupId(String groupId) { + this.groupId = groupId; + } - public void setTopic(String topic) { - this.topic = topic; - } + public String getOffsetReset() { + return offsetReset; + } - public String getGroupId() { - return groupId; - } + public void setOffsetReset(String offsetReset) { + if(offsetReset == null){ + return; + } + this.offsetReset = offsetReset; + } - public void setGroupId(String groupId) { - this.groupId = groupId; - } + public String getOffset() { + return offset; + } - public String getOffsetReset() { - return offsetReset; - } + public void setOffset(String offset) { + if (offsetReset == null) { + return; + } + this.offset = offset; + } - public void setOffsetReset(String offsetReset) { - this.offsetReset = offsetReset; - } + public Boolean getTopicIsPattern() { + return topicIsPattern; + } - public Boolean getTopicIsPattern() { - return topicIsPattern; - } + public void setTopicIsPattern(Boolean topicIsPattern) { + this.topicIsPattern = topicIsPattern; + } - public void setTopicIsPattern(Boolean topicIsPattern) { - this.topicIsPattern = topicIsPattern; - } + public Map kafkaParam = new HashMap<>(); - public void addKafkaParam(Map kafkaParam) { - kafkaParam.putAll(kafkaParam); - } + public void addKafkaParam(String key, String value) { + kafkaParam.put(key, value); + } - public String getKafkaParam(String key) { - return kafkaParam.get(key); - } + public String getKafkaParam(String key) { + return kafkaParam.get(key); + } - public Set getKafkaParamKeys() { - return kafkaParam.keySet(); - } + public Set getKafkaParamKeys() { + return kafkaParam.keySet(); + } - public String getSourceDataType() { - return sourceDataType; - } + public String getSourceDataType() { + return sourceDataType; + } - public void setSourceDataType(String sourceDataType) { - this.sourceDataType = sourceDataType; - } + public void setSourceDataType(String sourceDataType) { + this.sourceDataType = sourceDataType; + } - public String getSchemaString() { - return schemaString; - } + public String getSchemaString() { + return schemaString; + } - public void setSchemaString(String schemaString) { - this.schemaString = schemaString; - } + public void setSchemaString(String schemaString) { + this.schemaString = schemaString; + } - public String getFieldDelimiter() { - return fieldDelimiter; - } + public String getFieldDelimiter() { + return fieldDelimiter; + } - public void setFieldDelimiter(String fieldDelimiter) { - this.fieldDelimiter = fieldDelimiter; - } + public void setFieldDelimiter(String fieldDelimiter) { + this.fieldDelimiter = fieldDelimiter; + } - @Override - public boolean check() { - Preconditions.checkNotNull(getType(), "kafka of type is required"); - Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); - Preconditions.checkNotNull(topic, "kafka of topic is required"); - return false; - } + @Override + public boolean check() { + Preconditions.checkNotNull(getType(), "kafka of type is required"); + Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); + Preconditions.checkNotNull(topic, "kafka of topic is required"); + return false; + } } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java index 1cbbeafd9..71a9cc386 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java @@ -23,7 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,7 @@ * * @author maqi */ -public class KafkaProducer extends FlinkKafkaProducer { +public class KafkaProducer extends FlinkKafkaProducer { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer.class); @@ -45,7 +45,7 @@ public class KafkaProducer extends FlinkKafkaProducer { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] parititonKeys) { + public KafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] parititonKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, parititonKeys), producerConfig, customPartitioner); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java index 6bf9014df..f3a2f40f5 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java @@ -22,7 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -36,7 +36,7 @@ public class KafkaProducerFactory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 632bb720e..d0d477f4e 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,8 +18,23 @@ package com.dtstack.flink.sql.sink.kafka; +import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.table.utils.TableConnectorUtils; +import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -29,23 +44,104 @@ * @create: 2019-11-05 11:45 * @description: **/ -public class KafkaSink extends AbstractKafkaSink { +public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { + + protected String[] fieldNames; + + protected TypeInformation[] fieldTypes; + + protected String topic; + + protected int parallelism; + + protected Properties properties; + + protected FlinkKafkaProducer flinkKafkaProducer; + + /** The schema of the table. */ + private TableSchema schema; + + /** Partitioner to select Kafka partition for each item. */ + protected Optional> partitioner; + + private String[] partitionKeys; + @Override - public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafkaSinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - - Properties kafkaProperties = getKafkaProperties(kafkaSinkTableInfo); - this.tableName = kafkaSinkTableInfo.getName(); this.topic = kafkaSinkTableInfo.getTopic(); + + properties = new Properties(); + properties.setProperty("bootstrap.servers", kafkaSinkTableInfo.getBootstrapServers()); + + for (String key : kafkaSinkTableInfo.getKafkaParamKeys()) { + properties.setProperty(key, kafkaSinkTableInfo.getKafkaParam(key)); + } this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafkaSinkTableInfo); this.fieldNames = kafkaSinkTableInfo.getFields(); - this.fieldTypes = getTypeInformations(kafkaSinkTableInfo); - this.schema = buildTableSchema(fieldNames, fieldTypes); - this.parallelism = kafkaSinkTableInfo.getParallelism(); - this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); - this.kafkaProducer = new KafkaProducerFactory().createKafkaProducer(kafkaSinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); + TypeInformation[] types = new TypeInformation[kafkaSinkTableInfo.getFields().length]; + for (int i = 0; i < kafkaSinkTableInfo.getFieldClasses().length; i++) { + types[i] = TypeInformation.of(kafkaSinkTableInfo.getFieldClasses()[i]); + } + this.fieldTypes = types; + + TableSchema.Builder schemaBuilder = TableSchema.builder(); + for (int i=0;i) new KafkaProducerFactory().createKafkaProducer(kafkaSinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); return this; } + @Override + public TypeInformation getRecordType() { + return new RowTypeInfo(fieldTypes, fieldNames); + } + + @Override + public void emitDataStream(DataStream> dataStream) { + DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) + .map((Tuple2 record) -> record.f1) + .returns(getOutputType().getTypeAt(1)) + .setParallelism(parallelism); + + mapDataStream.addSink(flinkKafkaProducer).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); + } + + @Override + public TupleTypeInfo> getOutputType() { + return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); + } + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } + + @Override + public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + return this; + } + + private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ + if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } } diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 394ea86ee..a445a6529 100644 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -18,17 +18,24 @@ package com.dtstack.flink.sql.source.kafka; +import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; /** @@ -36,24 +43,75 @@ * @create: 2019-11-05 10:55 * @description: **/ -public class KafkaSource extends AbstractKafkaSource { +public class KafkaSource implements IStreamSourceGener
{ + private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; + + /** + * Get kafka data source, you need to provide the data field names, data types + * If you do not specify auto.offset.reset, the default use groupoffset + * + * @param sourceTableInfo + * @return + */ + @SuppressWarnings("rawtypes") @Override - public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); - Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); - TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); - FlinkKafkaConsumer kafkaSrc = (FlinkKafkaConsumer) new KafkaConsumerFactory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); + Properties props = new Properties(); + for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { + props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); + } + props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); + if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) { + props.setProperty("auto.offset.reset", "none"); + } else { + props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); + } + if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())) { + props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); + } - String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; + for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { + types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); + } + + TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + + FlinkKafkaConsumer kafkaSrc = (FlinkKafkaConsumer) new KafkaConsumerFactory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); + + + //earliest,latest + if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { + kafkaSrc.setStartFromEarliest(); + } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} + try { + Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); + Map offsetMap = PluginUtil.ObjectToMap(properties); + Map specificStartupOffsets = new HashMap<>(); + for (Map.Entry entry : offsetMap.entrySet()) { + specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); + } + kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); + } catch (Exception e) { + throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); + } + } else { + kafkaSrc.setStartFromLatest(); + } - setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); + String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + Integer parallelism = kafkaSourceTableInfo.getParallelism(); + if (parallelism != null) { + kafkaSource.setParallelism(parallelism); + } return tableEnv.fromDataStream(kafkaSource, fields); } } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java index bee1865dd..c815e134a 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java @@ -23,7 +23,6 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +37,7 @@ * * @author maqi */ -public class KafkaProducer09 extends FlinkKafkaProducer09 { +public class KafkaProducer09 extends FlinkKafkaProducer09 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer09.class); @@ -46,7 +45,7 @@ public class KafkaProducer09 extends FlinkKafkaProducer09 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer09(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + public KafkaProducer09(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner,String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.orElse(null)); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java index ee3423b07..7fb3909ee 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java @@ -22,7 +22,6 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -37,7 +36,7 @@ public class KafkaProducer09Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner,String[] partitionKeys) { return new KafkaProducer09(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index d22be3d59..52caa7bd2 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,8 +18,23 @@ package com.dtstack.flink.sql.sink.kafka; +import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.table.utils.TableConnectorUtils; +import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -31,22 +46,109 @@ * @author DocLi * @modifyer maqi */ -public class KafkaSink extends AbstractKafkaSink{ +public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { + + protected String[] fieldNames; + + protected TypeInformation[] fieldTypes; + + protected String topic; + + protected Properties properties; + + protected FlinkKafkaProducer09 kafkaProducer09; + + /** The schema of the table. */ + private TableSchema schema; + + /** Partitioner to select Kafka partition for each item. */ + protected Optional> partitioner; + + private String[] partitionKeys; + + protected int parallelism; + + + @Override - public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka09SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - - Properties kafkaProperties = getKafkaProperties(kafka09SinkTableInfo); - this.tableName = kafka09SinkTableInfo.getName(); this.topic = kafka09SinkTableInfo.getTopic(); + + properties = new Properties(); + properties.setProperty("bootstrap.servers", kafka09SinkTableInfo.getBootstrapServers()); + for (String key : kafka09SinkTableInfo.getKafkaParamKeys()) { + properties.setProperty(key, kafka09SinkTableInfo.getKafkaParam(key)); + } + this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka09SinkTableInfo); this.fieldNames = kafka09SinkTableInfo.getFields(); - this.fieldTypes = getTypeInformations(kafka09SinkTableInfo); - this.schema = buildTableSchema(fieldNames, fieldTypes); - this.parallelism = kafka09SinkTableInfo.getParallelism(); - this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); - this.kafkaProducer = new KafkaProducer09Factory().createKafkaProducer(kafka09SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); + TypeInformation[] types = new TypeInformation[kafka09SinkTableInfo.getFields().length]; + for (int i = 0; i < kafka09SinkTableInfo.getFieldClasses().length; i++) { + types[i] = TypeInformation.of(kafka09SinkTableInfo.getFieldClasses()[i]); + } + this.fieldTypes = types; + + TableSchema.Builder schemaBuilder = TableSchema.builder(); + for (int i=0;i) new KafkaProducer09Factory() + .createKafkaProducer(kafka09SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); return this; } + + @Override + public TypeInformation getRecordType() { + return new RowTypeInfo(fieldTypes, fieldNames); + } + + @Override + public void emitDataStream(DataStream> dataStream) { + DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) + .map((Tuple2 record) -> record.f1) + .returns(getOutputType().getTypeAt(1)) + .setParallelism(parallelism); + + mapDataStream.addSink(kafkaProducer09) + .name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); + } + + @Override + public TupleTypeInfo> getOutputType() { + return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); + } + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } + + @Override + public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + return this; + } + + private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ + if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } + } diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 9f8917761..21ed7c3e3 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -17,13 +17,17 @@ */ + package com.dtstack.flink.sql.source.kafka; +import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer09; @@ -32,6 +36,7 @@ import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; +import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -42,23 +47,74 @@ * @author xuchao */ -public class KafkaSource extends AbstractKafkaSource { +public class KafkaSource implements IStreamSourceGener
{ + + private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; + + /** + * Get kafka data source, you need to provide the data field names, data types + * If you do not specify auto.offset.reset, the default use groupoffset + * @param sourceTableInfo + * @return + */ + @SuppressWarnings("rawtypes") @Override - public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); - Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); - TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); - FlinkKafkaConsumer09 kafkaSrc = (FlinkKafkaConsumer09) new KafkaConsumer09Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); + Properties props = new Properties(); + for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { + props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); + } + props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); + if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ + props.setProperty("auto.offset.reset", "none"); + } else { + props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); + } + if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ + props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); + } + // only required for Kafka 0.8 + //TODO props.setProperty("zookeeper.connect", kafkaSourceTableInfo.) - String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; + for(int i = 0; i< kafkaSourceTableInfo.getFieldClasses().length; i++){ + types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); + } + + TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + FlinkKafkaConsumer09 kafkaSrc = (FlinkKafkaConsumer09) new KafkaConsumer09Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); + + //earliest,latest + if("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())){ + kafkaSrc.setStartFromEarliest(); + }else if(DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){// {"0":12312,"1":12321,"2":12312} + try { + Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); + Map offsetMap = PluginUtil.ObjectToMap(properties); + Map specificStartupOffsets = new HashMap<>(); + for(Map.Entry entry:offsetMap.entrySet()){ + specificStartupOffsets.put(new KafkaTopicPartition(topicName,Integer.valueOf(entry.getKey())),Long.valueOf(entry.getValue().toString())); + } + kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); + } catch (Exception e) { + throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); + } + }else { + kafkaSrc.setStartFromLatest(); + } - setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); + String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + Integer parallelism = kafkaSourceTableInfo.getParallelism(); + if (parallelism != null) { + kafkaSource.setParallelism(parallelism); + } return tableEnv.fromDataStream(kafkaSource, fields); } } \ No newline at end of file diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java index 3936575ef..3cdc45dec 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java @@ -23,7 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,7 @@ * * @author maqi */ -public class KafkaProducer010 extends FlinkKafkaProducer010 { +public class KafkaProducer010 extends FlinkKafkaProducer010 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer010.class); @@ -45,7 +45,7 @@ public class KafkaProducer010 extends FlinkKafkaProducer010 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.get()); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java index e0e023586..c44a9fe86 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java @@ -22,7 +22,6 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -37,7 +36,7 @@ public class KafkaProducer010Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer010(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index eea78e121..42f1045bf 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,8 +18,21 @@ package com.dtstack.flink.sql.sink.kafka; +import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.table.utils.TableConnectorUtils; +import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -33,22 +46,105 @@ * @modifyer maqi * */ -public class KafkaSink extends AbstractKafkaSink { - @Override - public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { - KafkaSinkTableInfo kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; +public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { + + + protected String[] fieldNames; + + protected TypeInformation[] fieldTypes; + + protected String topic; + + protected Properties properties; - Properties kafkaProperties = getKafkaProperties(kafka10SinkTableInfo); - this.tableName = kafka10SinkTableInfo.getName(); + protected int parallelism; + + protected KafkaSinkTableInfo kafka10SinkTableInfo; + + /** The schema of the table. */ + private TableSchema schema; + + private String[] partitionKeys; + + @Override + public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + this.kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; this.topic = kafka10SinkTableInfo.getTopic(); - this.partitioner = Optional.of(new CustomerFlinkPartition<>()); + + properties = new Properties(); + properties.setProperty("bootstrap.servers", kafka10SinkTableInfo.getBootstrapServers()); + + for (String key : kafka10SinkTableInfo.getKafkaParamKeys()) { + properties.setProperty(key, kafka10SinkTableInfo.getKafkaParam(key)); + } + this.partitionKeys = getPartitionKeys(kafka10SinkTableInfo); this.fieldNames = kafka10SinkTableInfo.getFields(); - this.fieldTypes = getTypeInformations(kafka10SinkTableInfo); - this.schema = buildTableSchema(fieldNames, fieldTypes); - this.parallelism = kafka10SinkTableInfo.getParallelism(); - this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); - this.kafkaProducer = new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); + TypeInformation[] types = new TypeInformation[kafka10SinkTableInfo.getFields().length]; + for (int i = 0; i < kafka10SinkTableInfo.getFieldClasses().length; i++) { + types[i] = TypeInformation.of(kafka10SinkTableInfo.getFieldClasses()[i]); + } + this.fieldTypes = types; + + + TableSchema.Builder schemaBuilder = TableSchema.builder(); + for (int i=0;i getRecordType() { + return new RowTypeInfo(fieldTypes, fieldNames); + } + + @Override + public void emitDataStream(DataStream> dataStream) { + + RichSinkFunction kafkaProducer010 = new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, + Optional.of(new CustomerFlinkPartition<>()), partitionKeys); + + DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) + .map((Tuple2 record) -> record.f1) + .returns(getOutputType().getTypeAt(1)) + .setParallelism(parallelism); + + mapDataStream.addSink(kafkaProducer010).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); + } + + @Override + public TupleTypeInfo> getOutputType() { + return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); + } + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } + + @Override + public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + return this; + } + private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ + if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } + } diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 23989ab7e..55cffbf1e 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -19,17 +19,24 @@ package com.dtstack.flink.sql.source.kafka; +import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; /** @@ -40,24 +47,74 @@ * @author sishu.yss */ -public class KafkaSource extends AbstractKafkaSource { +public class KafkaSource implements IStreamSourceGener
{ - @Override - public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; - String topicName = kafkaSourceTableInfo.getTopic(); + private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); - TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); - FlinkKafkaConsumer010 kafkaSrc = (FlinkKafkaConsumer010) new KafkaConsumer010Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); + /** + * Get kafka data source, you need to provide the data field names, data types + * If you do not specify auto.offset.reset, the default use groupoffset + * + * @param sourceTableInfo + * @return + */ + @SuppressWarnings("rawtypes") + @Override + public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; + String topicName = kafkaSourceTableInfo.getTopic(); - setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); - String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); + Properties props = new Properties(); + for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { + props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); + } + props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); + if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ + props.setProperty("auto.offset.reset", "none"); + } else { + props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); + } + if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ + props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); + } - return tableEnv.fromDataStream(kafkaSource, fields); - } + TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; + for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { + types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); + } + + TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + + FlinkKafkaConsumer010 kafkaSrc = (FlinkKafkaConsumer010) new KafkaConsumer010Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); + + //earliest,latest + if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { + kafkaSrc.setStartFromEarliest(); + } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} + try { + Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); + Map offsetMap = PluginUtil.ObjectToMap(properties); + Map specificStartupOffsets = new HashMap<>(); + for (Map.Entry entry : offsetMap.entrySet()) { + specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); + } + kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); + } catch (Exception e) { + throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); + } + } else { + kafkaSrc.setStartFromLatest(); + } + + String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); + String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); + + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + Integer parallelism = kafkaSourceTableInfo.getParallelism(); + if (parallelism != null) { + kafkaSource.setParallelism(parallelism); + } + return tableEnv.fromDataStream(kafkaSource, fields); + } } \ No newline at end of file diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java index 429d21a79..7880dd419 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java @@ -24,7 +24,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +38,7 @@ * * @author maqi */ -public class KafkaProducer011 extends FlinkKafkaProducer011 { +public class KafkaProducer011 extends FlinkKafkaProducer011 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer011.class); @@ -46,7 +46,7 @@ public class KafkaProducer011 extends FlinkKafkaProducer011 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer011(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + public KafkaProducer011(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java index 0cb11da82..e2272b16e 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java @@ -22,7 +22,6 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -37,7 +36,7 @@ public class KafkaProducer011Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer011(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index ea45280c7..c5da782d0 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,8 +18,23 @@ package com.dtstack.flink.sql.sink.kafka; +import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.table.utils.TableConnectorUtils; +import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -34,22 +49,106 @@ * @modifyer maqi * */ -public class KafkaSink extends AbstractKafkaSink { +public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { + + protected String[] fieldNames; + + protected TypeInformation[] fieldTypes; + + protected String topic; + + protected int parallelism; + + protected Properties properties; + + protected FlinkKafkaProducer011 kafkaProducer011; + + /** The schema of the table. */ + private TableSchema schema; + + /** Partitioner to select Kafka partition for each item. */ + protected Optional> partitioner; + private String[] partitionKeys; + + @Override - public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka11SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - - Properties kafkaProperties = getKafkaProperties(kafka11SinkTableInfo); - this.tableName = kafka11SinkTableInfo.getName(); this.topic = kafka11SinkTableInfo.getTopic(); + + properties = new Properties(); + properties.setProperty("bootstrap.servers", kafka11SinkTableInfo.getBootstrapServers()); + + for (String key : kafka11SinkTableInfo.getKafkaParamKeys()) { + properties.setProperty(key, kafka11SinkTableInfo.getKafkaParam(key)); + } this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka11SinkTableInfo); this.fieldNames = kafka11SinkTableInfo.getFields(); - this.fieldTypes = getTypeInformations(kafka11SinkTableInfo); - this.schema = buildTableSchema(fieldNames, fieldTypes); - this.parallelism = kafka11SinkTableInfo.getParallelism(); - this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); - this.kafkaProducer = new KafkaProducer011Factory().createKafkaProducer(kafka11SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); + TypeInformation[] types = new TypeInformation[kafka11SinkTableInfo.getFields().length]; + for (int i = 0; i < kafka11SinkTableInfo.getFieldClasses().length; i++) { + types[i] = TypeInformation.of(kafka11SinkTableInfo.getFieldClasses()[i]); + } + this.fieldTypes = types; + + TableSchema.Builder schemaBuilder = TableSchema.builder(); + for (int i=0;i) new KafkaProducer011Factory() + .createKafkaProducer(kafka11SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); return this; } + + @Override + public TypeInformation getRecordType() { + return new RowTypeInfo(fieldTypes, fieldNames); + } + + @Override + public void emitDataStream(DataStream> dataStream) { + DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) + .map((Tuple2 record) -> record.f1) + .returns(getOutputType().getTypeAt(1)) + .setParallelism(parallelism); + + mapDataStream.addSink(kafkaProducer011).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); + } + + @Override + public TupleTypeInfo> getOutputType() { + return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); + } + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } + + @Override + public TableSink>configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + return this; + } + + private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ + if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } + } diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java index c20f0678b..3f804fc6c 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java @@ -31,8 +31,8 @@ /** * company: www.dtstack.com + * author: toutian * create: 2019/12/24 - * @author: toutian */ public class KafkaConsumer011Factory extends AbstractKafkaConsumerFactory { diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index f58d59d05..1a4eed08a 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -19,16 +19,24 @@ package com.dtstack.flink.sql.source.kafka; +import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; + +import java.util.HashMap; +import java.util.Map; import java.util.Properties; /** @@ -39,24 +47,76 @@ * @author sishu.yss */ -public class KafkaSource extends AbstractKafkaSource { +public class KafkaSource implements IStreamSourceGener
{ + + private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; + + /** + * Get kafka data source, you need to provide the data field names, data types + * If you do not specify auto.offset.reset, the default use groupoffset + * + * @param sourceTableInfo + * @return + */ + @SuppressWarnings("rawtypes") + @Override + public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; + String topicName = kafkaSourceTableInfo.getTopic(); + + Properties props = new Properties(); + for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { + props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); + } + props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); + if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ + props.setProperty("auto.offset.reset", "none"); + } else { + props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); + } + if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ + props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); + } + // only required for Kafka 0.8 + //TODO props.setProperty("zookeeper.connect", kafka09SourceTableInfo.) + + TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; + for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { + types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); + } - @Override - public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; - String topicName = kafkaSourceTableInfo.getTopic(); + TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); - Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); - TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); - FlinkKafkaConsumer011 kafkaSrc = (FlinkKafkaConsumer011) new KafkaConsumer011Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); + FlinkKafkaConsumer011 kafkaSrc = (FlinkKafkaConsumer011) new KafkaConsumer011Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); - String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + //earliest,latest + if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { + kafkaSrc.setStartFromEarliest(); + } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} + try { + Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); + Map offsetMap = PluginUtil.ObjectToMap(properties); + Map specificStartupOffsets = new HashMap<>(); + for (Map.Entry entry : offsetMap.entrySet()) { + specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); + } + kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); + } catch (Exception e) { + throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); + } + } else { + kafkaSrc.setStartFromLatest(); + } - setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); - String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); + String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); + String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - return tableEnv.fromDataStream(kafkaSource, fields); - } + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + Integer parallelism = kafkaSourceTableInfo.getParallelism(); + if (parallelism != null) { + kafkaSource.setParallelism(parallelism); + } + return tableEnv.fromDataStream(kafkaSource, fields); + } } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 31f9aa13d..63ff31c50 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -1,23 +1,24 @@ package com.dtstack.flink.sql.side.kudu; -import com.dtstack.flink.sql.side.BaseAllReqRow; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; +import org.apache.flink.util.Preconditions; + +import com.dtstack.flink.sql.side.AllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; -import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.KuduClient; @@ -40,7 +41,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; -public class KuduAllReqRow extends BaseAllReqRow { +public class KuduAllReqRow extends AllReqRow { private static final long serialVersionUID = 6051774809356082219L; @@ -61,7 +62,7 @@ public class KuduAllReqRow extends BaseAllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public KuduAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public KuduAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new KuduAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -228,6 +229,9 @@ private KuduScanner getConn(KuduSideTableInfo tableInfo) { if (null != workerCount) { kuduClientBuilder.workerCount(workerCount); } + if (null != defaultSocketReadTimeoutMs) { + kuduClientBuilder.defaultSocketReadTimeoutMs(defaultSocketReadTimeoutMs); + } if (null != defaultOperationTimeoutMs) { kuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java index 1241a6f37..ddfa73201 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java @@ -2,8 +2,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; @@ -12,16 +12,16 @@ import java.util.List; -public class KuduAllSideInfo extends BaseSideInfo { +public class KuduAllSideInfo extends SideInfo { private static final long serialVersionUID = 9005389633060174746L; - public KuduAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public KuduAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { // no use } diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index e534a85f9..e1705d796 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -1,12 +1,20 @@ package com.dtstack.flink.sql.side.kudu; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.AsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; @@ -16,13 +24,6 @@ import com.stumbleupon.async.Deferred; import io.vertx.core.json.JsonArray; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.AsyncKuduClient; @@ -41,7 +42,7 @@ import java.util.List; import java.util.Map; -public class KuduAsyncReqRow extends BaseAsyncReqRow { +public class KuduAsyncReqRow extends AsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(KuduAsyncReqRow.class); /** @@ -64,7 +65,7 @@ public class KuduAsyncReqRow extends BaseAsyncReqRow { private AsyncKuduScanner.AsyncKuduScannerBuilder scannerBuilder; - public KuduAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public KuduAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new KuduAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -94,6 +95,9 @@ private void connKuDu() throws KuduException { if (null != workerCount) { asyncKuduClientBuilder.workerCount(workerCount); } + if (null != defaultSocketReadTimeoutMs) { + asyncKuduClientBuilder.defaultSocketReadTimeoutMs(defaultSocketReadTimeoutMs); + } if (null != defaultOperationTimeoutMs) { asyncKuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java index 5ace515f7..de75f23fb 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java @@ -2,8 +2,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; @@ -12,15 +12,15 @@ import java.util.List; -public class KuduAsyncSideInfo extends BaseSideInfo { +public class KuduAsyncSideInfo extends SideInfo { - public KuduAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public KuduAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { } @Override diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java index 62d215d87..c03c470a7 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java @@ -1,7 +1,7 @@ package com.dtstack.flink.sql.side.kudu.table; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; @@ -9,9 +9,9 @@ import java.sql.Timestamp; import java.util.Map; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; -public class KuduSideParser extends AbstractSideTableParser { +public class KuduSideParser extends AbsSideTableParser { public static final String KUDU_MASTERS = "kuduMasters"; @@ -52,7 +52,7 @@ public class KuduSideParser extends AbstractSideTableParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KuduSideTableInfo kuduSideTableInfo = new KuduSideTableInfo(); kuduSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, kuduSideTableInfo); @@ -107,7 +107,6 @@ public Class dbTypeConvertToJavaType(String fieldType) { return BigDecimal.class; case "binary": return byte[].class; - default: } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java index eb8df5ad9..c527ec268 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java @@ -1,9 +1,9 @@ package com.dtstack.flink.sql.side.kudu.table; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.google.common.base.Preconditions; -public class KuduSideTableInfo extends AbstractSideTableInfo { +public class KuduSideTableInfo extends SideTableInfo { private static final String CURR_TYPE = "kudu"; diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java index dfc02dee6..b750e2d4a 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java @@ -191,7 +191,6 @@ public static KuduPredicate buildKuduPredicate(Schema schema, PredicateInfo info return KuduPredicate.newIsNotNullPredicate(column); case "IS NULL": return KuduPredicate.newIsNullPredicate(column); - default: } return null; diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index 4543b9618..99d19cdef 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -18,11 +18,13 @@ package com.dtstack.flink.sql.sink.kudu; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; + +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import org.apache.kudu.client.AsyncKuduClient; import org.apache.kudu.client.AsyncKuduSession; import org.apache.kudu.client.KuduClient; @@ -42,7 +44,7 @@ * @author gituser * @modify xiuzhu */ -public class KuduOutputFormat extends AbstractDtRichOutputFormat { +public class KuduOutputFormat extends DtRichOutputFormat { private static final long serialVersionUID = 1L; @@ -123,7 +125,7 @@ public void writeRecord(Tuple2 record) throws IOException { Row row = tupleTrans.getField(1); if (row.getArity() != fieldNames.length) { if(outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0) { - LOG.error("record insert failed ..{}", row.toString()); + LOG.error("record insert failed:{}", row.toString()); LOG.error("cause by row.getArity() != fieldNames.length"); } outDirtyRecords.inc(); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java index 4c7c3bea8..0c90ffe94 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java @@ -3,7 +3,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kudu.table.KuduTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -39,7 +39,7 @@ public class KuduSink implements RetractStreamTableSink, Serializable, IStr private int parallelism = -1; @Override - public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public KuduSink genStreamSink(TargetTableInfo targetTableInfo) { KuduTableInfo kuduTableInfo = (KuduTableInfo) targetTableInfo; this.kuduMasters = kuduTableInfo.getKuduMasters(); this.tableName = kuduTableInfo.getTableName(); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java index 20302d44f..ea61bd325 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java @@ -1,8 +1,8 @@ package com.dtstack.flink.sql.sink.kudu.table; import com.dtstack.flink.sql.sink.kudu.KuduOutputFormat; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; @@ -10,9 +10,9 @@ import java.sql.Timestamp; import java.util.Map; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; -public class KuduSinkParser extends AbstractTableParser { +public class KuduSinkParser extends AbsTableParser { public static final String KUDU_MASTERS = "kuduMasters"; @@ -27,7 +27,7 @@ public class KuduSinkParser extends AbstractTableParser { public static final String SOCKET_READ_TIMEOUT_MS = "defaultSocketReadTimeoutMs"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KuduTableInfo kuduTableInfo = new KuduTableInfo(); kuduTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, kuduTableInfo); @@ -86,7 +86,6 @@ public Class dbTypeConvertToJavaType(String fieldType) { return BigDecimal.class; case "binary": return byte[].class; - default: } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java index 80e47761b..c086c7298 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java @@ -1,10 +1,10 @@ package com.dtstack.flink.sql.sink.kudu.table; import com.dtstack.flink.sql.sink.kudu.KuduOutputFormat; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; -public class KuduTableInfo extends AbstractTargetTableInfo { +public class KuduTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "kudu"; diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 8f0090db5..14cd847b5 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -21,11 +21,11 @@ import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.util.PluginUtil; -import com.esotericsoftware.minlog.Log; import org.apache.commons.io.Charsets; import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.MiniClusterClient; +import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; @@ -89,7 +89,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { if (StringUtils.isNotBlank(yarnConfDir)) { try { - config.setString("fs.hdfs.hadoopconf", yarnConfDir); + config.setString(ConfigConstants.PATH_HADOOP_CONFIG, yarnConfDir); FileSystem.initialize(config); YarnConfiguration yarnConf = YarnConfLoader.getYarnConf(yarnConfDir); @@ -109,7 +109,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { applicationId = getYarnClusterApplicationId(yarnClient); } - Log.info("applicationId={}", applicationId.toString()); + System.out.println("applicationId=" + applicationId.toString()); if (StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); @@ -166,7 +166,7 @@ private static ApplicationId getYarnClusterApplicationId(YarnClient yarnClient) private static ApplicationId toApplicationId(String appIdStr) { Iterator it = StringHelper._split(appIdStr).iterator(); - if (!"application".equals(it.next())) { + if (!(it.next()).equals("application")) { throw new IllegalArgumentException("Invalid ApplicationId prefix: " + appIdStr + ". The valid ApplicationId should start with prefix " + "application"); } else { try { diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 0e5089bc9..3de1cdbc5 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -21,7 +21,6 @@ import com.dtstack.flink.sql.enums.EPluginLoadMode; import com.dtstack.flink.sql.launcher.YarnConfLoader; import com.dtstack.flink.sql.option.Options; -import com.esotericsoftware.minlog.Log; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.cache.DistributedCache; import org.apache.flink.configuration.Configuration; @@ -80,7 +79,7 @@ public void init(String yarnConfDir, Configuration flinkConfig, Properties userC yarnClient.init(yarnConf); yarnClient.start(); - Log.info("----init yarn success ----"); + System.out.println("----init yarn success ----"); } public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJarPath, Options launcherOptions, JobGraph jobGraph) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index ca2d5bdd7..29cc4890a 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -73,6 +73,7 @@ public static String submit(Options launcherOptions, JobGraph jobGraph, Configur String flinkJobId = jobGraph.getJobID().toString(); String tips = String.format("deploy per_job with appId: %s, jobId: %s", applicationId, flinkJobId); + System.out.println(tips); LOG.info(tips); return applicationId; diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index 3373f3667..f90cfd973 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -18,17 +18,25 @@ package com.dtstack.flink.sql.side.mongo; -import com.dtstack.flink.sql.side.BaseAllReqRow; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; + +import com.dtstack.flink.sql.side.AllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.mongodb.BasicDBObject; import com.mongodb.MongoClient; -import com.mongodb.MongoClientURI; +import com.mongodb.MongoClientOptions; +import com.mongodb.MongoCredential; +import com.mongodb.ServerAddress; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; @@ -36,19 +44,13 @@ import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.SQLException; import java.sql.Timestamp; +import java.util.ArrayList; import java.util.Calendar; import java.util.List; import java.util.Map; @@ -60,7 +62,7 @@ * * @author xuqianjin */ -public class MongoAllReqRow extends BaseAllReqRow { +public class MongoAllReqRow extends AllReqRow { private static final long serialVersionUID = -675332795591842778L; @@ -76,7 +78,7 @@ public class MongoAllReqRow extends BaseAllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public MongoAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MongoAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new MongoAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -179,13 +181,34 @@ private String buildKey(Map val, List equalFieldList) { return sb.toString(); } - private MongoCollection getConn(String address, String database, String tableName) { + private MongoCollection getConn(String address, String userName, String password, String database, String tableName) { MongoCollection dbCollection; - mongoClient = new MongoClient(new MongoClientURI(address)); - db = mongoClient.getDatabase(database); - dbCollection = db.getCollection(tableName, Document.class); - return dbCollection; - + try { + MongoCredential credential; + String[] servers = StringUtils.split(address, ","); + String host; + Integer port; + String[] hostAndPort; + List lists = new ArrayList<>(); + for (String server : servers) { + hostAndPort = StringUtils.split(server, ":"); + host = hostAndPort[0]; + port = Integer.parseInt(hostAndPort[1]); + lists.add(new ServerAddress(host, port)); + } + if (!StringUtils.isEmpty(userName) || !StringUtils.isEmpty(password)) { + credential = MongoCredential.createCredential(userName, database, password.toCharArray()); + // To connect to mongodb server + mongoClient = new MongoClient(lists, credential, new MongoClientOptions.Builder().build()); + } else { + mongoClient = new MongoClient(lists); + } + db = mongoClient.getDatabase(database); + dbCollection = db.getCollection(tableName, Document.class); + return dbCollection; + } catch (Exception e) { + throw new RuntimeException("[connMongoDB]:" + e.getMessage()); + } } private void loadData(Map>> tmpCache) throws SQLException { @@ -195,7 +218,8 @@ private void loadData(Map>> tmpCache) throws SQ try { for (int i = 0; i < CONN_RETRY_NUM; i++) { try { - dbCollection = getConn(tableInfo.getAddress(), tableInfo.getDatabase(), tableInfo.getTableName()); + dbCollection = getConn(tableInfo.getAddress(), tableInfo.getUserName(), tableInfo.getPassword(), + tableInfo.getDatabase(), tableInfo.getTableName()); break; } catch (Exception e) { if (i == CONN_RETRY_NUM - 1) { diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java index 4a33f1a69..b2d5c2686 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; @@ -36,16 +36,16 @@ * * @author xuqianjin */ -public class MongoAllSideInfo extends BaseSideInfo { +public class MongoAllSideInfo extends SideInfo{ private static final long serialVersionUID = -1696860430075523841L; - public MongoAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MongoAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { } @Override diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index d85a9990d..42749e935 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -27,35 +27,33 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.AsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.google.common.collect.Lists; import com.mongodb.BasicDBObject; import com.mongodb.Block; -import com.mongodb.ConnectionString; +import com.mongodb.MongoCredential; +import com.mongodb.ServerAddress; import com.mongodb.async.SingleResultCallback; import com.mongodb.async.client.MongoClient; -import com.mongodb.MongoClientSettings; +import com.mongodb.async.client.MongoClientSettings; import com.mongodb.async.client.MongoClients; import com.mongodb.async.client.MongoCollection; import com.mongodb.async.client.MongoDatabase; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; +import com.mongodb.connection.ClusterSettings; +import com.mongodb.connection.ConnectionPoolSettings; +import org.apache.commons.lang3.StringUtils; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Timestamp; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -68,43 +66,68 @@ * * @author xuqianjin */ -public class MongoAsyncReqRow extends BaseAsyncReqRow { +public class MongoAsyncReqRow extends AsyncReqRow { private static final long serialVersionUID = -1183158242862673706L; private static final Logger LOG = LoggerFactory.getLogger(MongoAsyncReqRow.class); + private final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = 20; + private transient MongoClient mongoClient; private MongoDatabase db; - private MongoSideTableInfo mongoSideTableInfo; + private MongoSideTableInfo MongoSideTableInfo; - public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new MongoAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override public void open(Configuration parameters) throws Exception { super.open(parameters); - mongoSideTableInfo = (MongoSideTableInfo) sideInfo.getSideTableInfo(); - connMongoDb(); + MongoSideTableInfo = (MongoSideTableInfo) sideInfo.getSideTableInfo(); + connMongoDB(); } - public void connMongoDb() throws Exception { - String address = mongoSideTableInfo.getAddress(); - ConnectionString connectionString = new ConnectionString(address); - - MongoClientSettings settings = MongoClientSettings.builder() - .applyConnectionString(connectionString) + public void connMongoDB() throws Exception { + MongoCredential mongoCredential; + String[] servers = StringUtils.split(MongoSideTableInfo.getAddress(), ","); + String host; + Integer port; + String[] hostAndPort; + List lists = new ArrayList<>(); + for (String server : servers) { + hostAndPort = StringUtils.split(server, ":"); + host = hostAndPort[0]; + port = Integer.parseInt(hostAndPort[1]); + lists.add(new ServerAddress(host, port)); + } + ClusterSettings clusterSettings = ClusterSettings.builder().hosts(lists).build(); + ConnectionPoolSettings connectionPoolSettings = ConnectionPoolSettings.builder() + .maxSize(DEFAULT_MAX_DB_CONN_POOL_SIZE) .build(); - mongoClient = MongoClients.create(settings); - db = mongoClient.getDatabase(mongoSideTableInfo.getDatabase()); + if (!StringUtils.isEmpty(MongoSideTableInfo.getUserName()) || !StringUtils.isEmpty(MongoSideTableInfo.getPassword())) { + mongoCredential = MongoCredential.createCredential(MongoSideTableInfo.getUserName(), MongoSideTableInfo.getDatabase(), + MongoSideTableInfo.getPassword().toCharArray()); + MongoClientSettings settings = MongoClientSettings.builder().credential(mongoCredential) + .clusterSettings(clusterSettings) + .connectionPoolSettings(connectionPoolSettings) + .build(); + mongoClient = MongoClients.create(settings); + } else { + MongoClientSettings settings = MongoClientSettings.builder().clusterSettings(clusterSettings) + .connectionPoolSettings(connectionPoolSettings) + .build(); + mongoClient = MongoClients.create(settings); + } + db = mongoClient.getDatabase(MongoSideTableInfo.getDatabase()); } @Override public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { CRow inputCopy = new CRow(input.row(), input.change()); - BasicDBObject basicDbObject = new BasicDBObject(); + BasicDBObject basicDBObject = new BasicDBObject(); for (int i = 0; i < sideInfo.getEqualFieldList().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = inputCopy.row().getField(conValIndex); @@ -112,14 +135,14 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce dealMissKey(inputCopy, resultFuture); return; } - basicDbObject.put(sideInfo.getEqualFieldList().get(i), equalObj); + basicDBObject.put(sideInfo.getEqualFieldList().get(i), equalObj); } try { // 填充谓词 sideInfo.getSideTableInfo().getPredicateInfoes().stream().map(info -> { BasicDBObject filterCondition = MongoUtil.buildFilterObject(info); if (null != filterCondition) { - basicDbObject.append(info.getFieldName(), filterCondition); + basicDBObject.append(info.getFieldName(), filterCondition); } return info; }).count(); @@ -127,7 +150,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce LOG.info("add predicate infoes error ", e); } - String key = buildCacheKey(basicDbObject.values()); + String key = buildCacheKey(basicDBObject.values()); if (openCache()) { CacheObj val = getFromCache(key); if (val != null) { @@ -149,7 +172,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce } } AtomicInteger atomicInteger = new AtomicInteger(0); - MongoCollection dbCollection = db.getCollection(mongoSideTableInfo.getTableName(), Document.class); + MongoCollection dbCollection = db.getCollection(MongoSideTableInfo.getTableName(), Document.class); List cacheContent = Lists.newArrayList(); Block printDocumentBlock = new Block() { @Override @@ -175,7 +198,7 @@ public void onResult(final Void result, final Throwable t) { } } }; - dbCollection.find(basicDbObject).forEach(printDocumentBlock, callbackWhenFinished); + dbCollection.find(basicDBObject).forEach(printDocumentBlock, callbackWhenFinished); } @Override diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java index 19534d4e1..de81ed1f1 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java @@ -20,13 +20,16 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; +import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; +import com.google.common.collect.Lists; import java.util.List; @@ -36,16 +39,16 @@ * * @author xuqianjin */ -public class MongoAsyncSideInfo extends BaseSideInfo { +public class MongoAsyncSideInfo extends SideInfo { private static final long serialVersionUID = -3694857194254465989L; - public MongoAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MongoAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { } diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java index 4fe1ebee1..5edc12419 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java @@ -19,13 +19,15 @@ package com.dtstack.flink.sql.side.mongo.table; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; /** * Reason: @@ -33,7 +35,7 @@ * * @author xuqianjin */ -public class MongoSideParser extends AbstractSideTableParser { +public class MongoSideParser extends AbsSideTableParser { public static final String ADDRESS_KEY = "address"; @@ -46,7 +48,7 @@ public class MongoSideParser extends AbstractSideTableParser { public static final String DATABASE_KEY = "database"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { MongoSideTableInfo mongoSideTableInfo = new MongoSideTableInfo(); mongoSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, mongoSideTableInfo); diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java index c83de7bbc..a5c834469 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.mongo.table; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.google.common.base.Preconditions; /** @@ -28,7 +28,7 @@ * * @author xuqianjin */ -public class MongoSideTableInfo extends AbstractSideTableInfo { +public class MongoSideTableInfo extends SideTableInfo { private static final long serialVersionUID = -1L; diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java index cbcba8f97..2f1a2092d 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java @@ -55,7 +55,6 @@ public static BasicDBObject buildFilterObject(PredicateInfo info) { return new BasicDBObject("$exists", true); case "IS NULL": return new BasicDBObject("$exists", false); - default: } return null; } diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index 3f1f9f7e2..337fbad59 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -19,16 +19,20 @@ package com.dtstack.flink.sql.sink.mongo; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; -import com.mongodb.MongoClient; -import com.mongodb.MongoClientURI; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoDatabase; -import com.mongodb.client.result.UpdateResult; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; + +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.mongodb.MongoClient; +import com.mongodb.MongoClientOptions; +import com.mongodb.MongoCredential; +import com.mongodb.ServerAddress; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.result.UpdateResult; +import org.apache.commons.lang3.StringUtils; import org.bson.Document; import org.bson.types.ObjectId; import org.slf4j.Logger; @@ -44,7 +48,7 @@ * * @author xuqianjin */ -public class MongoOutputFormat extends AbstractDtRichOutputFormat { +public class MongoOutputFormat extends DtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(MongoOutputFormat.class); private String address; @@ -121,8 +125,30 @@ public void close() { } private void establishConnection() { - mongoClient = new MongoClient(new MongoClientURI(address)); - db = mongoClient.getDatabase(database); + try { + MongoCredential credential; + String[] servers = StringUtils.split(address, ","); + String host; + Integer port; + String[] hostAndPort; + List lists = new ArrayList<>(); + for (String server : servers) { + hostAndPort = StringUtils.split(server, ":"); + host = hostAndPort[0]; + port = Integer.parseInt(hostAndPort[1]); + lists.add(new ServerAddress(host, port)); + } + if (!StringUtils.isEmpty(userName) || !StringUtils.isEmpty(password)) { + credential = MongoCredential.createCredential(userName, database, password.toCharArray()); + // To connect to mongodb server + mongoClient = new MongoClient(lists, credential, new MongoClientOptions.Builder().build()); + } else { + mongoClient = new MongoClient(lists); + } + db = mongoClient.getDatabase(database); + } catch (Exception e) { + throw new IllegalArgumentException("[connMongoDB]:" + e.getMessage()); + } } private MongoOutputFormat() { diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java index 3f34055ec..4e28d8fd2 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.mongo.table.MongoTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -54,7 +54,7 @@ public MongoSink() { } @Override - public MongoSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public MongoSink genStreamSink(TargetTableInfo targetTableInfo) { MongoTableInfo mongoTableInfo = (MongoTableInfo) targetTableInfo; this.address = mongoTableInfo.getAddress(); this.tableName = mongoTableInfo.getTableName(); diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java index f093b70aa..d4810fb6d 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java @@ -19,13 +19,13 @@ package com.dtstack.flink.sql.sink.mongo.table; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; /** * Reason: @@ -34,7 +34,7 @@ * @author xuqianjin */ -public class MongoSinkParser extends AbstractTableParser { +public class MongoSinkParser extends AbsTableParser { private static final String CURR_TYPE = "mongo"; @@ -49,7 +49,7 @@ public class MongoSinkParser extends AbstractTableParser { public static final String DATABASE_KEY = "database"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { MongoTableInfo mongoTableInfo = new MongoTableInfo(); mongoTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, mongoTableInfo); diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java index c9d94dfe6..02a96d6bb 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.mongo.table; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; /** @@ -30,7 +30,7 @@ */ -public class MongoTableInfo extends AbstractTargetTableInfo { +public class MongoTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "mongo"; diff --git a/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java new file mode 100644 index 000000000..33a0233ac --- /dev/null +++ b/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flinkx; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * Unit test for simple App. + */ +public class AppTest + extends TestCase +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public AppTest( String testName ) + { + super( testName ); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite( AppTest.class ); + } + + /** + * Rigourous Test :-) + */ + public void testApp() + { + assertTrue( true ); + } +} diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java index eb294e64e..b6b7f45cb 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; @@ -41,7 +41,7 @@ * @author xuchao */ -public class MysqlAllReqRow extends AbstractRdbAllReqRow { +public class MysqlAllReqRow extends RdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -49,18 +49,18 @@ public class MysqlAllReqRow extends AbstractRdbAllReqRow { private static final String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; - public MysqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MysqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new MysqlAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbUrl, String userName, String password) { + public Connection getConn(String dbURL, String userName, String password) { try { Class.forName(MYSQL_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java index 503dbf9a3..07b06ba38 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class MysqlAllSideInfo extends RdbAllSideInfo { - public MysqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MysqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index d52f1918f..6120767ef 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -30,6 +30,8 @@ import io.vertx.ext.jdbc.JDBCClient; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; @@ -42,9 +44,12 @@ */ public class MysqlAsyncReqRow extends RdbAsyncReqRow { + + private static final Logger LOG = LoggerFactory.getLogger(MysqlAsyncReqRow.class); + private final static String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; - public MysqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MysqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new MysqlAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -56,7 +61,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); mysqlClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", MYSQL_DRIVER) - .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -68,10 +73,10 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setRdbSQLClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); } } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java index 9555897aa..5d6ed370d 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -36,7 +36,7 @@ public class MysqlAsyncSideInfo extends RdbAsyncSideInfo { - public MysqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MysqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java index b0a38e344..40f68e7e4 100644 --- a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java +++ b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.mysql.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -37,8 +37,8 @@ public class MysqlSideParser extends RdbSideParser { private static final String CURR_TYPE = "mysql"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java index 0a1749f04..8c1146ede 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; @@ -33,7 +33,7 @@ * @author xuchao */ -public class MysqlSink extends AbstractRdbSink implements IStreamSinkGener { +public class MysqlSink extends RdbSink implements IStreamSinkGener { public MysqlSink() { super(new MySQLDialect()); @@ -42,7 +42,7 @@ public MysqlSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(dbUrl) + .setDBUrl(dbURL) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java index 49105a7a8..2247eb8cc 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.mysql.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -36,8 +36,8 @@ public class MysqlSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "mysql"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java new file mode 100644 index 000000000..33a0233ac --- /dev/null +++ b/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flinkx; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * Unit test for simple App. + */ +public class AppTest + extends TestCase +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public AppTest( String testName ) + { + super( testName ); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite( AppTest.class ); + } + + /** + * Rigourous Test :-) + */ + public void testApp() + { + assertTrue( true ); + } +} diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java index 3be6687ec..18d9ba045 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; @@ -36,22 +36,22 @@ /** * side operator with cache for all(period reload) */ -public class OracleAllReqRow extends AbstractRdbAllReqRow { +public class OracleAllReqRow extends RdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(OracleAllReqRow.class); private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; - public OracleAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public OracleAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new OracleAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbUrl, String userName, String password) { + public Connection getConn(String dbURL, String userName, String password) { try { Class.forName(ORACLE_DRIVER); Map addParams = Maps.newHashMap(); - String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java index 342533681..361366929 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java @@ -19,17 +19,18 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; public class OracleAllSideInfo extends RdbAllSideInfo { - public OracleAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public OracleAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index 938fd870c..f973ffee2 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -30,14 +30,19 @@ import io.vertx.ext.jdbc.JDBCClient; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; public class OracleAsyncReqRow extends RdbAsyncReqRow { + + private static final Logger LOG = LoggerFactory.getLogger(OracleAsyncReqRow.class); + private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; - public OracleAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public OracleAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new OracleAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -48,7 +53,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); oracleClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", ORACLE_DRIVER) - .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -60,9 +65,9 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); + setRdbSQLClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); } } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java index 658020b1c..8e801970f 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java @@ -20,20 +20,19 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; - import java.util.List; public class OracleAsyncSideInfo extends RdbAsyncSideInfo { - public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -56,7 +55,7 @@ public String wrapperPlaceholder(String fieldName) { String rpadFormat = "rpad(?, %d, ' ')"; if (StringUtils.contains(type.toLowerCase(), "char")) { - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = sideTableInfo.getFieldExtraInfoList().get(pos); + TableInfo.FieldExtraInfo fieldExtraInfo = sideTableInfo.getFieldExtraInfoList().get(pos); int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); if (charLength > 0) { return String.format(rpadFormat, charLength); diff --git a/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java b/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java index 897b77d30..f9124bd10 100644 --- a/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java +++ b/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side.oracle.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -27,8 +27,8 @@ public class OracleSideParser extends RdbSideParser { private static final String CURR_TYPE = "oracle"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); oracleTableInfo.setType(CURR_TYPE); return oracleTableInfo; } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java index 3a320658c..a687aa012 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.oracle; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; @@ -41,7 +41,7 @@ public class OracleDialect implements JDBCDialect { private List fieldList; private List fieldTypeList; - private List fieldExtraInfoList; + private List fieldExtraInfoList; @Override public boolean canHandle(String url) { @@ -133,7 +133,7 @@ public String wrapperPlaceholder(String fieldName) { String type = fieldTypeList.get(pos); if (StringUtils.contains(type.toLowerCase(), DEAL_CHAR_KEY)) { - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfoList.get(pos); + TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfoList.get(pos); int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); if (charLength > 0) { return String.format(RPAD_FORMAT, charLength); @@ -151,7 +151,7 @@ public void setFieldTypeList(List fieldTypeList) { this.fieldTypeList = fieldTypeList; } - public void setFieldExtraInfoList(List fieldExtraInfoList) { + public void setFieldExtraInfoList(List fieldExtraInfoList) { this.fieldExtraInfoList = fieldExtraInfoList; } } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index f916b55f2..7a37aa6cd 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** @@ -29,7 +29,7 @@ * * @author maqi */ -public class OracleSink extends AbstractRdbSink implements IStreamSinkGener { +public class OracleSink extends RdbSink implements IStreamSinkGener { public OracleSink() { super(new OracleDialect()); @@ -42,7 +42,7 @@ public JDBCUpsertOutputFormat getOutputFormat() { ((OracleDialect) jdbcDialect).setFieldExtraInfoList(fieldExtraInfoList); JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(dbUrl) + .setDBUrl(dbURL) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java index 1732c8cc2..aff096bd3 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.oracle.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -33,8 +33,8 @@ public class OracleSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "oracle"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); oracleTableInfo.setType(CURR_TYPE); return oracleTableInfo; } diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java index 5fbf0824b..f9e0950dd 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java @@ -1,26 +1,9 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.collect.Maps; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -32,12 +15,7 @@ import java.util.List; import java.util.Map; -/** - * Date: 2019/12/20 - * Company: www.dtstack.com - * @author yinxi - */ -public class PolardbAllReqRow extends AbstractRdbAllReqRow { +public class PolardbAllReqRow extends RdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -45,18 +23,18 @@ public class PolardbAllReqRow extends AbstractRdbAllReqRow { private static final String POLARDB_DRIVER = "com.mysql.cj.jdbc.Driver"; - public PolardbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PolardbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new PolardbAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbUrl, String userName, String password) { + public Connection getConn(String dbURL, String userName, String password) { try { Class.forName(POLARDB_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java index c46c9dfd6..822cbaa70 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java @@ -1,37 +1,15 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; -/** - * Date: 2019/12/20 - * Company: www.dtstack.com - * @author yinxi - */ public class PolardbAllSideInfo extends RdbAllSideInfo { - public PolardbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PolardbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 13d25b111..3cbb0b368 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -1,25 +1,8 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -33,18 +16,13 @@ import java.util.List; -/** - * Date: 2019/12/20 - * Company: www.dtstack.com - * @author yinxi - */ public class PolardbAsyncReqRow extends RdbAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(PolardbAsyncReqRow.class); private final static String POLARDB_DRIVER = "com.mysql.cj.jdbc.Driver"; - public PolardbAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PolardbAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new PolardbAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -55,7 +33,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); mysqlClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", POLARDB_DRIVER) - .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -67,9 +45,9 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setRdbSQLClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); } } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java index 15237956b..bad71375f 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java @@ -1,38 +1,16 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; -/** - * Date: 2019/12/20 - * Company: www.dtstack.com - * @author yinxi - */ public class PolardbAsyncSideInfo extends RdbAsyncSideInfo { - public PolardbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PolardbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java index 71aad7cbb..348b3aed9 100644 --- a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java +++ b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java @@ -1,39 +1,16 @@ - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.dtstack.flink.sql.side.polardb.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; -/** - * Date: 2019/12/20 - * Company: www.dtstack.com - * @author yinxi - */ public class PolardbSideParser extends RdbSideParser { private static final String CURR_TYPE = "polardb"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java index c69f31e8f..44d558140 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java @@ -1,33 +1,11 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package com.dtstack.flink.sql.sink.polardb; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -/** - * Date: 2019/12/20 - * Company: www.dtstack.com - * @author yinxi - */ -public class PolardbSink extends AbstractRdbSink { + +public class PolardbSink extends RdbSink { public PolardbSink() { super(new PolardbDialect()); } @@ -35,7 +13,7 @@ public PolardbSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(dbUrl) + .setDBUrl(dbURL) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java index b4f02665e..68eead938 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java @@ -1,38 +1,16 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.dtstack.flink.sql.sink.polardb.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; -/** - * Date: 2019/12/20 - * Company: www.dtstack.com - * @author yinxi - */ public class PolardbSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "polardb"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo polardbTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo polardbTableInfo = super.getTableInfo(tableName, fieldsInfo, props); polardbTableInfo.setType(CURR_TYPE); return polardbTableInfo; } diff --git a/pom.xml b/pom.xml index de963240c..2794f9004 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,6 @@ mysql hbase elasticsearch5 - elasticsearch6 mongo redis5 launcher @@ -34,6 +33,7 @@ impala db2 polardb + elasticsearch6 @@ -54,20 +54,6 @@ - - org.jacoco - jacoco-maven-plugin - 0.7.8 - - - - prepare-agent - report - - - - - pl.project13.maven git-commit-id-plugin diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java index 0ec8f96ce..6d68cfdca 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; @@ -41,7 +41,7 @@ * @author tcm */ -public class PostgresqlAllReqRow extends AbstractRdbAllReqRow { +public class PostgresqlAllReqRow extends RdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -49,18 +49,18 @@ public class PostgresqlAllReqRow extends AbstractRdbAllReqRow { private static final String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - public PostgresqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PostgresqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new PostgresqlAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbUrl, String userName, String password) { + public Connection getConn(String dbURL, String userName, String password) { try { Class.forName(POSTGRESQL_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java index a4a52eabc..d383ee46d 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class PostgresqlAllSideInfo extends RdbAllSideInfo { - public PostgresqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PostgresqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index 42aa39a51..86fed91fa 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -48,7 +48,7 @@ public class PostgresqlAsyncReqRow extends RdbAsyncReqRow { private final static String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - public PostgresqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PostgresqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new PostgresqlAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -60,7 +60,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); pgClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", POSTGRESQL_DRIVER) - .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) @@ -69,9 +69,9 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, pgClientConfig)); + setRdbSQLClient(JDBCClient.createNonShared(vertx, pgClientConfig)); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java index f19a488bc..1d89f4894 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -36,7 +36,7 @@ public class PostgresqlAsyncSideInfo extends RdbAsyncSideInfo { - public PostgresqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PostgresqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java b/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java index be0c5ea8b..faee2c704 100644 --- a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java +++ b/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.postgresql.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -37,8 +37,8 @@ public class PostgresqlSideParser extends RdbSideParser { private static final String CURR_TYPE = "postgresql"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); pgTableInfo.setType(CURR_TYPE); return pgTableInfo; } diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java index bb731152c..cf45a6bd9 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java @@ -22,13 +22,13 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** * @author maqi */ -public class PostgresqlSink extends AbstractRdbSink implements IStreamSinkGener { +public class PostgresqlSink extends RdbSink implements IStreamSinkGener { public PostgresqlSink() { super(new PostgresqlDialect()); } @@ -36,7 +36,7 @@ public PostgresqlSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(dbUrl) + .setDBUrl(dbURL) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java index e94cfc8a0..5943b5e52 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.postgresql.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -35,8 +35,8 @@ public class PostgresqlSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "postgresql"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); pgTableInfo.setType(CURR_TYPE); return pgTableInfo; } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java similarity index 92% rename from rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java rename to rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java index 2e9986742..60e3d817f 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java @@ -19,15 +19,13 @@ package com.dtstack.flink.sql.side.rdb.all; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.configuration.Configuration; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; - -import com.dtstack.flink.sql.side.BaseAllReqRow; -import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import com.google.common.collect.Lists; @@ -58,11 +56,11 @@ * @author maqi */ -public abstract class AbstractRdbAllReqRow extends BaseAllReqRow { +public abstract class RdbAllReqRow extends AllReqRow { private static final long serialVersionUID = 2098635140857937718L; - private static final Logger LOG = LoggerFactory.getLogger(AbstractRdbAllReqRow.class); + private static final Logger LOG = LoggerFactory.getLogger(RdbAllReqRow.class); private static final int CONN_RETRY_NUM = 3; @@ -70,18 +68,10 @@ public abstract class AbstractRdbAllReqRow extends BaseAllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public AbstractRdbAllReqRow(BaseSideInfo sideInfo) { + public RdbAllReqRow(SideInfo sideInfo) { super(sideInfo); } - @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - LOG.info("rdb dim table config info: {} ", tableInfo.toString()); - } - - @Override protected void initCache() throws SQLException { Map>> newCache = Maps.newConcurrentMap(); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 143017133..3b7444b77 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -23,16 +23,14 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; @@ -46,21 +44,20 @@ * @author maqi */ -public class RdbAllSideInfo extends BaseSideInfo { +public class RdbAllSideInfo extends SideInfo { private static final long serialVersionUID = -5858335638589472159L; - private static final Logger LOG = LoggerFactory.getLogger(RdbAllSideInfo.class.getSimpleName()); - public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideTableInfo; sqlCondition = getSelectFromStatement(getTableName(rdbSideTableInfo), Arrays.asList(StringUtils.split(sideSelectFields, ",")), sideTableInfo.getPredicateInfoes()); - LOG.info("--------dimension sql query-------\n{}" + sqlCondition); + System.out.println("--------dimension sql query-------\n" + sqlCondition); } public String getAdditionalWhereClause() { diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 2ba34ad6b..2d767ccd9 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -20,11 +20,8 @@ package com.dtstack.flink.sql.side.rdb.async; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; -import com.dtstack.flink.sql.side.CacheMissVal; -import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; -import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import com.dtstack.flink.sql.util.DateUtil; import io.vertx.core.json.JsonArray; @@ -32,7 +29,6 @@ import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import com.google.common.collect.Lists; -import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -53,7 +49,7 @@ * @author maqi */ -public class RdbAsyncReqRow extends BaseAsyncReqRow { +public class RdbAsyncReqRow extends AsyncReqRow { private static final long serialVersionUID = 2098635244857937720L; @@ -63,9 +59,7 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { public final static int DEFAULT_VERTX_WORKER_POOL_SIZE = Runtime.getRuntime().availableProcessors() * 2; - public final static int DEFAULT_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; - - public final static int MAX_DB_CONN_POOL_SIZE_LIMIT = 20; + public final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; public final static int DEFAULT_IDLE_CONNECTION_TEST_PEROID = 60; @@ -75,25 +69,10 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { public final static String PREFERRED_TEST_QUERY_SQL = "select 1 from dual"; - private transient SQLClient rdbSqlClient; + private transient SQLClient rdbSQLClient; - public RdbAsyncReqRow(BaseSideInfo sideInfo) { + public RdbAsyncReqRow(SideInfo sideInfo) { super(sideInfo); - init(sideInfo); - } - - protected void init(BaseSideInfo sideInfo) { - RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - int defaultAsyncPoolSize = Math.min(MAX_DB_CONN_POOL_SIZE_LIMIT, DEFAULT_DB_CONN_POOL_SIZE); - int rdbPoolSize = rdbSideTableInfo.getAsyncPoolSize() > 0 ? rdbSideTableInfo.getAsyncPoolSize() : defaultAsyncPoolSize; - rdbSideTableInfo.setAsyncPoolSize(rdbPoolSize); - } - - @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - LOG.info("rdb dim table config info: {} ", rdbSideTableInfo.toString()); } @Override @@ -130,7 +109,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce } } - rdbSqlClient.getConnection(conn -> { + rdbSQLClient.getConnection(conn -> { if (conn.failed()) { //Treatment failures resultFuture.completeExceptionally(conn.cause()); @@ -197,9 +176,9 @@ private Object convertDataType(Object val) { } else if (val instanceof Instant) { } else if (val instanceof Timestamp) { - val = DateUtil.timestampToString((Timestamp) val); + val = DateUtil.getStringFromTimestamp((Timestamp) val); } else if (val instanceof java.util.Date) { - val = DateUtil.dateToString((java.util.Date)val); + val = DateUtil.getStringFromDate((java.sql.Date) val); } else { val = val.toString(); } @@ -248,8 +227,8 @@ public Row fillData(Row input, Object line) { @Override public void close() throws Exception { super.close(); - if (rdbSqlClient != null) { - rdbSqlClient.close(); + if (rdbSQLClient != null) { + rdbSQLClient.close(); } } @@ -264,8 +243,8 @@ public String buildCacheKey(JsonArray jsonArray) { return sb.toString(); } - public void setRdbSqlClient(SQLClient rdbSqlClient) { - this.rdbSqlClient = rdbSqlClient; + public void setRdbSQLClient(SQLClient rdbSQLClient) { + this.rdbSQLClient = rdbSQLClient; } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java index 9cbd43989..9e8c13080 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java @@ -23,8 +23,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; @@ -33,8 +33,6 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; @@ -49,18 +47,16 @@ * @author maqi */ -public class RdbAsyncSideInfo extends BaseSideInfo { +public class RdbAsyncSideInfo extends SideInfo { private static final long serialVersionUID = 1942629132469918611L; - private static final Logger LOG = LoggerFactory.getLogger(RdbAsyncSideInfo.class); - - public RdbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public RdbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideTableInfo; String sideTableName = joinInfo.getSideTableName(); @@ -78,7 +74,7 @@ public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInf sqlCondition = getSelectFromStatement(getTableName(rdbSideTableInfo), Arrays.asList(StringUtils.split(sideSelectFields, ",")), equalFieldList, sqlJoinCompareOperate, sideTableInfo.getPredicateInfoes()); - LOG.info("----------dimension sql query-----------\n{}", sqlCondition); + System.out.println("----------dimension sql query-----------\n" + sqlCondition); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java index 542258257..fcca4f0ef 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java @@ -25,19 +25,13 @@ import javax.sql.DataSource; import java.beans.PropertyVetoException; import java.sql.SQLException; -/** - * Date: 2019/9/17 - * Company: www.dtstack.com - * @author maqi - */ + public class DTC3P0DataSourceProvider extends C3P0DataSourceProvider { @Override public DataSource getDataSource(JsonObject config) throws SQLException { String url = config.getString("url"); - if (url == null) { - throw new NullPointerException("url cannot be null"); - } + if (url == null) throw new NullPointerException("url cannot be null"); String driverClass = config.getString("driver_class"); String user = config.getString("user"); String password = config.getString("password"); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java index 79942ae11..b9811b0ee 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.side.rdb.table; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -33,10 +33,10 @@ * @author maqi */ -public class RdbSideParser extends AbstractSideTableParser { +public class RdbSideParser extends AbsSideTableParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RdbSideTableInfo rdbTableInfo = new RdbSideTableInfo(); rdbTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, rdbTableInfo); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java index 43d0a6ee8..2c8c18fcd 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java @@ -17,7 +17,7 @@ */ package com.dtstack.flink.sql.side.rdb.table; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.google.common.base.Preconditions; /** @@ -27,7 +27,7 @@ * * @author maqi */ -public class RdbSideTableInfo extends AbstractSideTableInfo { +public class RdbSideTableInfo extends SideTableInfo { private static final long serialVersionUID = -1L; @@ -101,15 +101,4 @@ public String getPassword() { public void setPassword(String password) { this.password = password; } - - @Override - public String toString() { - String cacheInfo = super.toString(); - String connectionInfo = "Rdb Side Connection Info{" + - "url='" + url + '\'' + - ", tableName='" + tableName + '\'' + - ", schema='" + schema + '\'' + - '}'; - return cacheInfo + " , " + connectionInfo; - } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java index 49d163d4d..1205c5521 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java @@ -78,7 +78,6 @@ public static Object getTarget(Object obj, String targetType) { case "timestamp": case "datetime": return MathUtil.getTimestamp(obj); - default: } return obj; } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java index f476bfadf..e9a4f391f 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java @@ -29,7 +29,7 @@ public class JDBCOptions { - private String dbUrl; + private String dbURL; private String tableName; private String driverName; private String username; @@ -37,9 +37,9 @@ public class JDBCOptions { private String schema; private JDBCDialect dialect; - private JDBCOptions(String dbUrl, String tableName, String driverName, String username, + private JDBCOptions(String dbURL, String tableName, String driverName, String username, String password, String schema, JDBCDialect dialect) { - this.dbUrl = dbUrl; + this.dbURL = dbURL; this.tableName = tableName; this.driverName = driverName; this.username = username; @@ -48,8 +48,8 @@ private JDBCOptions(String dbUrl, String tableName, String driverName, String us this.dialect = dialect; } - public String getDbUrl() { - return dbUrl; + public String getDbURL() { + return dbURL; } public String getTableName() { @@ -84,7 +84,7 @@ public static Builder builder() { public boolean equals(Object o) { if (o instanceof JDBCOptions) { JDBCOptions options = (JDBCOptions) o; - return Objects.equals(dbUrl, options.dbUrl) && + return Objects.equals(dbURL, options.dbURL) && Objects.equals(tableName, options.tableName) && Objects.equals(driverName, options.driverName) && Objects.equals(username, options.username) && @@ -100,7 +100,7 @@ public boolean equals(Object o) { * Builder of {@link JDBCOptions}. */ public static class Builder { - private String dbUrl; + private String dbURL; private String tableName; private String driverName; private String username; @@ -152,8 +152,8 @@ public Builder setSchema(String schema) { /** * required, JDBC DB url. */ - public Builder setDbUrl(String dbUrl) { - this.dbUrl = dbUrl; + public Builder setDBUrl(String dbURL) { + this.dbURL = dbURL; return this; } @@ -163,7 +163,7 @@ public Builder setDialect(JDBCDialect dialect) { } public JDBCOptions build() { - checkNotNull(dbUrl, "No dbURL supplied."); + checkNotNull(dbURL, "No dbURL supplied."); checkNotNull(tableName, "No tableName supplied."); if (this.driverName == null) { @@ -173,7 +173,7 @@ public JDBCOptions build() { }); } - return new JDBCOptions(dbUrl, tableName, driverName, username, password, schema, dialect); + return new JDBCOptions(dbURL, tableName, driverName, username, password, schema, dialect); } } } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java similarity index 90% rename from rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java rename to rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java index ee51c22d1..84eea68ff 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; -import com.dtstack.flink.sql.table.AbstractTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -44,8 +44,11 @@ * * @author maqi */ -public abstract class AbstractRdbSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { - protected String dbUrl; +public abstract class RdbSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { + + protected String driverName; + + protected String dbURL; protected String userName; @@ -83,20 +86,20 @@ public abstract class AbstractRdbSink implements RetractStreamTableSink, Se public List fieldList; public List fieldTypeList; - public List fieldExtraInfoList; + public List fieldExtraInfoList; - public AbstractRdbSink(JDBCDialect jdbcDialect) { + public RdbSink(JDBCDialect jdbcDialect) { this.jdbcDialect = jdbcDialect; } @Override - public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { RdbTableInfo rdbTableInfo = (RdbTableInfo) targetTableInfo; this.batchNum = rdbTableInfo.getBatchSize() == null ? batchNum : rdbTableInfo.getBatchSize(); this.batchWaitInterval = rdbTableInfo.getBatchWaitInterval() == null ? batchWaitInterval : rdbTableInfo.getBatchWaitInterval(); this.parallelism = rdbTableInfo.getParallelism() == null ? parallelism : rdbTableInfo.getParallelism(); - this.dbUrl = rdbTableInfo.getUrl(); + this.dbURL = rdbTableInfo.getUrl(); this.userName = rdbTableInfo.getUserName(); this.password = rdbTableInfo.getPassword(); this.tableName = rdbTableInfo.getTableName(); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java index 2a3ce5e90..73f17d8fd 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.format; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; @@ -36,7 +36,7 @@ * @see Row * @see DriverManager */ -public abstract class AbstractJDBCOutputFormat extends AbstractDtRichOutputFormat { +public abstract class AbstractJDBCOutputFormat extends DtRichOutputFormat { private static final long serialVersionUID = 1L; public static final int DEFAULT_FLUSH_MAX_SIZE = 100; @@ -47,16 +47,16 @@ public abstract class AbstractJDBCOutputFormat extends AbstractDtRichOutputFo protected final String username; protected final String password; - private final String driverName; + private final String drivername; protected final String dbURL; protected transient Connection connection; - public AbstractJDBCOutputFormat(String username, String password, String driverName, String dbUrl) { + public AbstractJDBCOutputFormat(String username, String password, String drivername, String dbURL) { this.username = username; this.password = password; - this.driverName = driverName; - this.dbURL = dbUrl; + this.drivername = drivername; + this.dbURL = dbURL; } @Override @@ -64,7 +64,7 @@ public void configure(Configuration parameters) { } protected void establishConnection() throws SQLException, ClassNotFoundException, IOException { - JDBCUtils.forName(driverName, getClass().getClassLoader()); + JDBCUtils.forName(drivername, getClass().getClassLoader()); if (username == null) { connection = DriverManager.getConnection(dbURL); } else { diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index e8b6dc8a4..e1b2cc6f4 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -25,7 +25,7 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.dtstack.flink.sql.sink.rdb.writer.AppendOnlyWriter; import com.dtstack.flink.sql.sink.rdb.writer.JDBCWriter; -import com.dtstack.flink.sql.sink.rdb.writer.AbstractUpsertWriter; +import com.dtstack.flink.sql.sink.rdb.writer.UpsertWriter; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; @@ -44,7 +44,6 @@ /** * An upsert OutputFormat for JDBC. - * @author maqi */ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat> { @@ -84,7 +83,7 @@ public JDBCUpsertOutputFormat( long flushIntervalMills, boolean allReplace, String updateMode) { - super(options.getUsername(), options.getPassword(), options.getDriverName(), options.getDbUrl()); + super(options.getUsername(), options.getPassword(), options.getDriverName(), options.getDbURL()); this.schema = options.getSchema(); this.tableName = options.getTableName(); this.dialect = options.getDialect(); @@ -112,11 +111,12 @@ public void open(int taskNumber, int numTasks) throws IOException { initMetric(); if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.APPEND.name()) || keyFields == null || keyFields.length == 0) { - String insertSql = dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields); - LOG.info("execute insert sql： {}", insertSql); - jdbcWriter = new AppendOnlyWriter(insertSql, fieldTypes, this); + String insertSQL = dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields); + LOG.info("execute insert sql： {}", insertSQL); + System.out.println("execute insert sql :" + insertSQL); + jdbcWriter = new AppendOnlyWriter(insertSQL, fieldTypes, this); } else { - jdbcWriter = AbstractUpsertWriter.create( + jdbcWriter = UpsertWriter.create( dialect, schema, tableName, fieldNames, fieldTypes, keyFields, partitionFields, getRuntimeContext().getExecutionConfig().isObjectReuseEnabled(), allReplace, this); } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java deleted file mode 100644 index e69de29bb..000000000 diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java index aca7acb6c..9bbff3b4d 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java @@ -17,8 +17,9 @@ */ package com.dtstack.flink.sql.sink.rdb.table; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.constrant.ConfigConstrant; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -30,9 +31,9 @@ * * @author maqi */ -public class RdbSinkParser extends AbstractTableParser { +public class RdbSinkParser extends AbsTableParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RdbTableInfo rdbTableInfo = new RdbTableInfo(); rdbTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, rdbTableInfo); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java index ccdf28e48..e490978e4 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java @@ -18,10 +18,12 @@ package com.dtstack.flink.sql.sink.rdb.table; import com.dtstack.flink.sql.enums.EUpdateMode; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; +import java.util.Arrays; + /** * Reason: * Date: 2018/11/27 @@ -29,7 +31,7 @@ * * @author maqi */ -public class RdbTableInfo extends AbstractTargetTableInfo { +public class RdbTableInfo extends TargetTableInfo { public static final String URL_KEY = "url"; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 3559d4376..ca2bd4c66 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.writer; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -34,7 +34,6 @@ /** * Just append record to jdbc, can not receive retract/delete message. - * @author maqi */ public class AppendOnlyWriter implements JDBCWriter { @@ -42,16 +41,16 @@ public class AppendOnlyWriter implements JDBCWriter { private static final Logger LOG = LoggerFactory.getLogger(AppendOnlyWriter.class); - private final String insertSql; + private final String insertSQL; private final int[] fieldTypes; private transient PreparedStatement statement; private transient List rows; // only use metric - private transient AbstractDtRichOutputFormat metricOutputFormat; + private transient DtRichOutputFormat metricOutputFormat; - public AppendOnlyWriter(String insertSql, int[] fieldTypes, AbstractDtRichOutputFormat metricOutputFormat) { - this.insertSql = insertSql; + public AppendOnlyWriter(String insertSQL, int[] fieldTypes, DtRichOutputFormat metricOutputFormat) { + this.insertSQL = insertSQL; this.fieldTypes = fieldTypes; this.metricOutputFormat = metricOutputFormat; } @@ -64,7 +63,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { - this.statement = connection.prepareStatement(insertSql); + this.statement = connection.prepareStatement(insertSQL); } /** diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java similarity index 84% rename from rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java rename to rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java index 2c070b680..ce3a1c680 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.writer; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; @@ -41,15 +41,14 @@ /** * Upsert writer to deal with upsert, delete message.dd - * @author maqi */ -public abstract class AbstractUpsertWriter implements JDBCWriter { +public abstract class UpsertWriter implements JDBCWriter { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(AbstractUpsertWriter.class); + private static final Logger LOG = LoggerFactory.getLogger(UpsertWriter.class); - public static AbstractUpsertWriter create( + public static UpsertWriter create( JDBCDialect dialect, String schema, String tableName, @@ -59,7 +58,7 @@ public static AbstractUpsertWriter create( String[] partitionFields, boolean objectReuse, boolean allReplace, - AbstractDtRichOutputFormat metricOutputFormat) { + DtRichOutputFormat metricOutputFormat) { checkNotNull(keyFields); @@ -68,18 +67,20 @@ public static AbstractUpsertWriter create( int[] pkTypes = fieldTypes == null ? null : Arrays.stream(pkFields).map(f -> fieldTypes[f]).toArray(); - String deleteSql = dialect.getDeleteStatement(schema, tableName, keyFields); - LOG.info("deleteSQL is :{}", deleteSql); + String deleteSQL = dialect.getDeleteStatement(schema, tableName, keyFields); + LOG.info("deleteSQL is :{}", deleteSQL); + System.out.println("deleteSQL is :" + deleteSQL); - Optional upsertSql = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); - LOG.info("execute UpsertStatement: {}", upsertSql.orElse("use UsingInsertUpdateStatement")); + Optional upsertSQL = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); + LOG.info("execute UpsertStatement: {}", upsertSQL.orElse("use UsingInsertUpdateStatement")); + System.out.println("execute UpsertStatement: " + upsertSQL.orElse("use UsingInsertUpdateStatement")); - return upsertSql.map((Function) sql -> + return upsertSQL.map((Function) sql -> new UpsertWriterUsingUpsertStatement( - fieldTypes, pkFields, pkTypes, objectReuse, deleteSql, sql, metricOutputFormat)) + fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, sql, metricOutputFormat)) .orElseGet(() -> new UpsertWriterUsingInsertUpdateStatement( - fieldTypes, pkFields, pkTypes, objectReuse, deleteSql, + fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, dialect.getRowExistsStatement(tableName, keyFields), dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields), dialect.getUpdateStatement(tableName, fieldNames, keyFields), @@ -89,19 +90,19 @@ public static AbstractUpsertWriter create( final int[] fieldTypes; final int[] pkTypes; private final int[] pkFields; - private final String deleteSql; + private final String deleteSQL; private final boolean objectReuse; private transient Map> keyToRows; private transient PreparedStatement deleteStatement; // only use metric - private transient AbstractDtRichOutputFormat metricOutputFormat; + private transient DtRichOutputFormat metricOutputFormat; - private AbstractUpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSql, boolean objectReuse, AbstractDtRichOutputFormat metricOutputFormat) { + private UpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSQL, boolean objectReuse, DtRichOutputFormat metricOutputFormat) { this.fieldTypes = fieldTypes; this.pkFields = pkFields; this.pkTypes = pkTypes; - this.deleteSql = deleteSql; + this.deleteSQL = deleteSQL; this.objectReuse = objectReuse; this.metricOutputFormat = metricOutputFormat; } @@ -114,7 +115,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { - this.deleteStatement = connection.prepareStatement(deleteSql); + this.deleteStatement = connection.prepareStatement(deleteSQL); } @Override @@ -169,6 +170,7 @@ public void executeUpdate(Connection connection) throws SQLException { } connection.commit(); } catch (Exception e) { + System.out.println(e.getCause()); // deal pg error: current transaction is aborted, commands ignored until end of transaction block connection.rollback(); connection.commit(); @@ -205,10 +207,10 @@ private Row getPrimaryKey(Row row) { // ---------------------------------------------------------------------------------------- - private static final class UpsertWriterUsingUpsertStatement extends AbstractUpsertWriter { + private static final class UpsertWriterUsingUpsertStatement extends UpsertWriter { private static final long serialVersionUID = 1L; - private final String upsertSql; + private final String upsertSQL; private transient PreparedStatement upsertStatement; @@ -217,11 +219,11 @@ private UpsertWriterUsingUpsertStatement( int[] pkFields, int[] pkTypes, boolean objectReuse, - String deleteSql, - String upsertSql, - AbstractDtRichOutputFormat metricOutputFormat) { - super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); - this.upsertSql = upsertSql; + String deleteSQL, + String upsertSQL, + DtRichOutputFormat metricOutputFormat) { + super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); + this.upsertSQL = upsertSQL; } @Override @@ -232,7 +234,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { super.prepareStatement(connection); - upsertStatement = connection.prepareStatement(upsertSql); + upsertStatement = connection.prepareStatement(upsertSQL); } @Override @@ -262,12 +264,12 @@ public void close() throws SQLException { } } - private static final class UpsertWriterUsingInsertUpdateStatement extends AbstractUpsertWriter { + private static final class UpsertWriterUsingInsertUpdateStatement extends UpsertWriter { private static final long serialVersionUID = 1L; - private final String existSql; - private final String insertSql; - private final String updateSql; + private final String existSQL; + private final String insertSQL; + private final String updateSQL; private transient PreparedStatement existStatement; private transient PreparedStatement insertStatement; @@ -278,15 +280,15 @@ private UpsertWriterUsingInsertUpdateStatement( int[] pkFields, int[] pkTypes, boolean objectReuse, - String deleteSql, - String existSql, - String insertSql, - String updateSql, - AbstractDtRichOutputFormat metricOutputFormat) { - super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); - this.existSql = existSql; - this.insertSql = insertSql; - this.updateSql = updateSql; + String deleteSQL, + String existSQL, + String insertSQL, + String updateSQL, + DtRichOutputFormat metricOutputFormat) { + super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); + this.existSQL = existSQL; + this.insertSQL = insertSQL; + this.updateSQL = updateSQL; } @Override @@ -297,9 +299,9 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { super.prepareStatement(connection); - existStatement = connection.prepareStatement(existSql); - insertStatement = connection.prepareStatement(insertSql); - updateStatement = connection.prepareStatement(updateSql); + existStatement = connection.prepareStatement(existSQL); + insertStatement = connection.prepareStatement(insertSQL); + updateStatement = connection.prepareStatement(updateSQL); } @Override diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 5d09c8f3d..0d257c5b9 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -18,10 +18,6 @@ package com.dtstack.flink.sql.side.redis; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.BaseAllReqRow; -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.redis.enums.RedisType; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; @@ -33,34 +29,22 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.java.typeutils.RowTypeInfo; +import com.google.common.collect.Maps; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.HostAndPort; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.JedisCommands; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisSentinelPool; +import redis.clients.jedis.*; import java.io.Closeable; import java.io.IOException; import java.sql.SQLException; - -import java.util.Calendar; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; +import java.util.*; import java.util.concurrent.atomic.AtomicReference; -/** - * @author yanxi - */ -public class RedisAllReqRow extends BaseAllReqRow { +import java.util.stream.Collectors; + +public class RedisAllReqRow extends AllReqRow{ private static final long serialVersionUID = 7578879189085344807L; @@ -78,7 +62,7 @@ public class RedisAllReqRow extends BaseAllReqRow { private RedisSideReqRow redisSideReqRow; - public RedisAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public RedisAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new RedisAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); this.redisSideReqRow = new RedisSideReqRow(super.sideInfo); } @@ -106,7 +90,7 @@ protected void reloadCache() { } cacheRef.set(newCache); - LOG.info("----- Redis all cacheRef reload end:{}", Calendar.getInstance()); + LOG.info("----- Redis all cacheRef reload end:{}", newCache.size()); } @Override diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java index ec0fa0d15..81e7b1f4f 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java @@ -20,27 +20,25 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import java.util.List; -/** - * @author yanxi - */ -public class RedisAllSideInfo extends BaseSideInfo { + +public class RedisAllSideInfo extends SideInfo { private static final long serialVersionUID = 1998703966487857613L; - public RedisAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public RedisAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index d53bbed8c..f2e164922 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -18,18 +18,8 @@ package com.dtstack.flink.sql.side.redis; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; - import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.CacheMissVal; -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.redis.enums.RedisType; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; @@ -39,20 +29,24 @@ import io.lettuce.core.api.StatefulRedisConnection; import io.lettuce.core.api.async.RedisHashAsyncCommands; import io.lettuce.core.api.async.RedisKeyAsyncCommands; +import io.lettuce.core.api.async.RedisStringAsyncCommands; import io.lettuce.core.cluster.RedisClusterClient; import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Consumer; -/** - * @author yanxi - */ -public class RedisAsyncReqRow extends BaseAsyncReqRow { + +public class RedisAsyncReqRow extends AsyncReqRow { private static final long serialVersionUID = -2079908694523987738L; @@ -70,7 +64,7 @@ public class RedisAsyncReqRow extends BaseAsyncReqRow { private RedisSideReqRow redisSideReqRow; - public RedisAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public RedisAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new RedisAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); redisSideReqRow = new RedisSideReqRow(super.sideInfo); } diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java index c23035ec7..9bdefe8c5 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java @@ -20,26 +20,24 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import java.util.List; -/** - * @author yanxi - */ -public class RedisAsyncSideInfo extends BaseSideInfo { + +public class RedisAsyncSideInfo extends SideInfo { private static final long serialVersionUID = -4851348392924455039L; - public RedisAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public RedisAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java index 514984f0a..f8a83b868 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java @@ -18,19 +18,16 @@ package com.dtstack.flink.sql.side.redis.table; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -/** - * @author yanxi - */ -public class RedisSideParser extends AbstractSideTableParser { +public class RedisSideParser extends AbsSideTableParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RedisSideTableInfo redisSideTableInfo = new RedisSideTableInfo(); redisSideTableInfo.setName(tableName); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java index 17557f916..5007868e9 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.redis.table; import com.dtstack.flink.sql.side.ISideReqRow; -import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.SideInfo; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; @@ -40,9 +40,9 @@ public class RedisSideReqRow implements ISideReqRow, Serializable { private static final long serialVersionUID = 3751171828444748982L; - private BaseSideInfo sideInfo; + private SideInfo sideInfo; - public RedisSideReqRow(BaseSideInfo sideInfo){ + public RedisSideReqRow(SideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -72,7 +72,7 @@ public Row fillData(Row input, Object sideInput) { return row; } - public void setRowField(Row row, Integer index, BaseSideInfo sideInfo, String value) { + public void setRowField(Row row, Integer index, SideInfo sideInfo, String value) { Integer keyIndex = sideInfo.getSideFieldIndex().get(index); String classType = sideInfo.getSideTableInfo().getFieldClassList().get(keyIndex).getName(); switch (classType){ diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java index 5224134c8..88cfcb8a2 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java @@ -18,12 +18,10 @@ package com.dtstack.flink.sql.side.redis.table; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.google.common.base.Preconditions; -/** - * @author yanxi - */ -public class RedisSideTableInfo extends AbstractSideTableInfo { + +public class RedisSideTableInfo extends SideTableInfo { private static final long serialVersionUID = -1L; diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index ae4fe5a4b..0bd053473 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -18,13 +18,14 @@ package com.dtstack.flink.sql.sink.redis; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; + +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.HostAndPort; @@ -41,10 +42,7 @@ import java.util.List; import java.util.Set; -/** - * @author yanxi - */ -public class RedisOutputFormat extends AbstractDtRichOutputFormat { +public class RedisOutputFormat extends DtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(RedisOutputFormat.class); private String url; @@ -142,8 +140,7 @@ private void establishConnection() { break; //集群 case 3: - jedis = new JedisCluster(addresses, timeout, timeout, 10, password, poolConfig); - default: + jedis = new JedisCluster(addresses, timeout, timeout,10, password, poolConfig); } } @@ -159,10 +156,10 @@ public void writeRecord(Tuple2 record) throws IOException { return; } - HashMap map = new HashMap<>(8); - for (String primaryKey : primaryKeys) { - for (int i = 0; i < fieldNames.length; i++) { - if (fieldNames[i].equals(primaryKey)) { + HashMap map = new HashMap<>(); + for (String primaryKey : primaryKeys){ + for (int i=0; i kvList = new LinkedList<>(); for (String primaryKey : primaryKeys){ - StringBuilder primaryKv = new StringBuilder(); + StringBuilder primaryKV = new StringBuilder(); int index = map.get(primaryKey).intValue(); - primaryKv.append(primaryKey).append(":").append(row.getField(index)); - kvList.add(primaryKv.toString()); + primaryKV.append(primaryKey).append(":").append(row.getField(index)); + kvList.add(primaryKV.toString()); } String perKey = String.join(":", kvList); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java index cc49a3ba8..d2e28c01f 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.redis.table.RedisTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -33,9 +33,7 @@ import org.apache.flink.types.Row; import java.util.List; -/** - * @author yanxi - */ + public class RedisSink implements RetractStreamTableSink, IStreamSinkGener { protected String[] fieldNames; @@ -69,7 +67,7 @@ public RedisSink(){ } @Override - public RedisSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public RedisSink genStreamSink(TargetTableInfo targetTableInfo) { RedisTableInfo redisTableInfo = (RedisTableInfo) targetTableInfo; this.url = redisTableInfo.getUrl(); this.database = redisTableInfo.getDatabase(); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index e965eeecb..0c9b0a64a 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.sink.redis.table; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; @@ -27,12 +27,10 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -/** - * @author yanxi - */ -public class RedisSinkParser extends AbstractTableParser { + +public class RedisSinkParser extends AbsTableParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RedisTableInfo redisTableInfo = new RedisTableInfo(); redisTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, redisTableInfo); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java index 2425f8c38..60d6dd12f 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java @@ -18,12 +18,10 @@ package com.dtstack.flink.sql.sink.redis.table; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; -/** - * @author yanxi - */ -public class RedisTableInfo extends AbstractTargetTableInfo { + +public class RedisTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "redis"; diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java index b032fdce6..e46e0a76f 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.format.DeserializationMetricWrapper; import com.dtstack.flink.sql.format.dtnest.DtNestRowDeserializationSchema; import com.dtstack.flink.sql.source.serversocket.table.ServersocketSourceTableInfo; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.flink.types.Row; @@ -67,7 +67,7 @@ public class CustomerSocketTextStreamFunction implements SourceFunction { ServersocketSourceTableInfo tableInfo; public CustomerSocketTextStreamFunction(ServersocketSourceTableInfo tableInfo, TypeInformation typeInfo, - Map rowAndFieldMapping, List fieldExtraInfos) { + Map rowAndFieldMapping, List fieldExtraInfos) { this.tableInfo = tableInfo; this.deserializationSchema = new DtNestRowDeserializationSchema(typeInfo, rowAndFieldMapping, fieldExtraInfos); this.deserializationMetricWrapper = new DeserializationMetricWrapper(typeInfo, deserializationSchema); @@ -94,7 +94,7 @@ public void run(SourceContext ctx) throws Exception { while (buffer.length() >= delimiter.length() && (delimPos = buffer.indexOf(delimiter)) != -1) { String record = buffer.substring(0, delimPos); // truncate trailing carriage return - if ("\n".equals(delimiter) && "\r".endsWith(record)) { + if (delimiter.equals("\n") && record.endsWith("\r")) { record = record.substring(0, record.length() - 1); } try { diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java index b4dda5f3b..3ab01fb42 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.serversocket.table.ServersocketSourceTableInfo; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -37,7 +37,7 @@ */ public class ServersocketSource implements IStreamSourceGener
{ @Override - public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { ServersocketSourceTableInfo serversocketSourceTableInfo = (ServersocketSourceTableInfo) sourceTableInfo; String tableName = serversocketSourceTableInfo.getName(); diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java index f2d861b17..b52e38e59 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java @@ -17,8 +17,8 @@ */ package com.dtstack.flink.sql.source.serversocket.table; -import com.dtstack.flink.sql.table.AbstractSourceParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSourceParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -30,9 +30,9 @@ * * @author maqi */ -public class ServersocketSourceParser extends AbstractSourceParser { +public class ServersocketSourceParser extends AbsSourceParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ServersocketSourceTableInfo serversocketSourceTableInfo = new ServersocketSourceTableInfo(); serversocketSourceTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, serversocketSourceTableInfo); diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java index 3123b477f..796728eb3 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java @@ -17,7 +17,7 @@ */ package com.dtstack.flink.sql.source.serversocket.table; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; import com.google.common.base.Preconditions; /** @@ -27,8 +27,8 @@ * * @author maqi */ -public class ServersocketSourceTableInfo extends AbstractSourceTableInfo { - +public class ServersocketSourceTableInfo extends SourceTableInfo { + //version private static final String CURR_TYPE = "serversocket"; public static final String HOSTNAME_KEY = "host"; diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java index dbae271e4..961539fde 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java @@ -20,43 +20,39 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.sql.Connection; import java.sql.DriverManager; import java.util.List; import java.util.Map; - /** - * side operator with cache for all(period reload) - * Date: 2019/11/26 - * Company: www.dtstack.com - * @author maqi + * side operator with cache for all(period reload) */ -public class SqlserverAllReqRow extends AbstractRdbAllReqRow { +public class SqlserverAllReqRow extends RdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(SqlserverAllReqRow.class); private static final String SQLSERVER_DRIVER = "net.sourceforge.jtds.jdbc.Driver"; - public SqlserverAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public SqlserverAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new SqlserverAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbUrl, String userName, String password) { + public Connection getConn(String dbURL, String userName, String password) { try { Class.forName(SQLSERVER_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); - String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); + //addParams.put("useCursorFetch", "true"); + String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java index a40af5697..2b9913e7e 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java @@ -20,18 +20,14 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; -/** - * Date: 2019/11/26 - * Company: www.dtstack.com - * @author maqi - */ + public class SqlserverAllSideInfo extends RdbAllSideInfo { - public SqlserverAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public SqlserverAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index a63d28607..cb4240ca6 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -33,18 +33,14 @@ import java.util.List; -/** - * Date: 2019/11/26 - * Company: www.dtstack.com - * @author maqi - */ + public class SqlserverAsyncReqRow extends RdbAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(SqlserverAsyncReqRow.class); private final static String SQLSERVER_DRIVER = "net.sourceforge.jtds.jdbc.Driver"; - public SqlserverAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public SqlserverAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new SqlserverAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -55,7 +51,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); sqlserverClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", SQLSERVER_DRIVER) - .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -68,9 +64,9 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); + setRdbSQLClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); } } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java index e8a487721..347ca9bf4 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java @@ -20,21 +20,16 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; - import java.util.List; -/** - * Date: 2019/11/26 - * Company: www.dtstack.com - * @author maqi - */ + public class SqlserverAsyncSideInfo extends RdbAsyncSideInfo { - public SqlserverAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public SqlserverAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java index e06c13898..3631dcd8a 100644 --- a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java +++ b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java @@ -18,20 +18,16 @@ package com.dtstack.flink.sql.side.sqlserver.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; -/** - * Date: 2019/11/26 - * Company: www.dtstack.com - * @author maqi - */ + public class SqlserverSideParser extends RdbSideParser { private static final String CURR_TYPE = "sqlserver"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo sqlServerTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo sqlServerTableInfo = super.getTableInfo(tableName, fieldsInfo, props); sqlServerTableInfo.setType(CURR_TYPE); return sqlServerTableInfo; } diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java index 3c331cdd0..dbbeb820e 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** @@ -29,7 +29,7 @@ * * @author maqi */ -public class SqlserverSink extends AbstractRdbSink implements IStreamSinkGener { +public class SqlserverSink extends RdbSink implements IStreamSinkGener { public SqlserverSink() { super(new SqlserverDialect()); @@ -38,7 +38,7 @@ public SqlserverSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(dbUrl) + .setDBUrl(dbURL) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java index 5300884bc..a695d6c3b 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.sqlserver.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -33,8 +33,8 @@ public class SqlserverSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "sqlserver"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo sqlserverTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo sqlserverTableInfo = super.getTableInfo(tableName, fieldsInfo, props); sqlserverTableInfo.setType(CURR_TYPE); return sqlserverTableInfo; } From 03a491311e6f56fb1eab2c6287e7ee2d2b140a15 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 8 Apr 2020 17:38:59 +0800 Subject: [PATCH 235/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dsonar=E6=89=AB?= =?UTF-8?q?=E6=8F=8F=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java | 2 +- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 2 -- .../java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java index b8530f224..56225cefb 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java @@ -121,7 +121,7 @@ public ReplaceInfo getReplaceInfo(String field){ private List makeFormula(String formula){ if(formula == null || formula.length() <= 0){ - Lists.newArrayList(); + return Lists.newArrayList(); } List result = Lists.newArrayList(); for(String meta: splitIgnoreQuotaBrackets(formula, "\\+")){ diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index a3189b0a6..e89ec0b46 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -22,7 +22,6 @@ import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; @@ -41,7 +40,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.List; import java.util.Map; import java.util.Set; diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java index 89667ee28..7af51be3d 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java @@ -130,7 +130,7 @@ public ReplaceInfo getReplaceInfo(String field){ private List makeFormula(String formula){ if(formula == null || formula.length() <= 0){ - Lists.newArrayList(); + return Lists.newArrayList(); } List result = Lists.newArrayList(); for(String meta: splitIgnoreQuotaBrackets(formula, "\\+")){ From a0e7d5b31e697b5162f13bc4f7efb537b8c93e35 Mon Sep 17 00:00:00 2001 From: xuchao Date: Wed, 8 Apr 2020 17:56:08 +0800 Subject: [PATCH 236/523] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E7=BC=BA=E5=B0=91=E7=9A=84=E4=BF=AE=E6=94=B9=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 823cd5143..3c9f37811 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -146,6 +146,8 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExeEnv(paramsInfo.getConfProp(), paramsInfo.getDeployMode()); StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); StreamQueryConfig streamQueryConfig = StreamEnvConfigManager.getStreamQueryConfig(tableEnv, paramsInfo.getConfProp()); + // init global flinkPlanner + FlinkPlanner.createFlinkPlanner(tableEnv.getFrameworkConfig(), tableEnv.getPlanner(), tableEnv.getTypeFactory()); SqlParser.setLocalSqlPluginRoot(paramsInfo.getLocalSqlPluginPath()); SqlTree sqlTree = SqlParser.parseSql(paramsInfo.getSql()); From 8aacd7e948c284d0ad15eec4e107ed7c4566ebfa Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Thu, 9 Apr 2020 10:43:15 +0800 Subject: [PATCH 237/523] fix import * --- .../flink/sql/side/rdb/all/AbstractRdbAllReqRow.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 5a1158125..bab9cc928 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -43,7 +43,11 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Calendar; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; From 5962508b3cb9010cec6d0f0a956adadc956cf23c Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 9 Apr 2020 10:58:54 +0800 Subject: [PATCH 238/523] fix import * --- .../dtstack/flink/sql/launcher/ClusterClientFactory.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 1da0db7a3..391e1a3f6 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -44,7 +44,12 @@ import java.net.InetSocketAddress; import java.net.URLDecoder; -import java.util.*; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Set; +import java.util.List; +import java.util.Properties; +import java.util.Iterator; /** * @author sishu.yss From 1eeea8adbc7a3a338bcce65fd78f001b3d0b382b Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 9 Apr 2020 11:27:10 +0800 Subject: [PATCH 239/523] =?UTF-8?q?add=20isHighAvailability=20=E5=88=A4?= =?UTF-8?q?=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/launcher/ClusterClientFactory.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 391e1a3f6..a40b0155c 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -60,6 +60,12 @@ public class ClusterClientFactory { private static final String HA_CLUSTER_ID = "high-availability.cluster-id"; + private static final String HIGH_AVAILABILITY = "high-availability"; + + private static final String NODE = "NONE"; + + private static final String ZOOKEEPER = "zookeeper"; + private static final String HADOOP_CONF = "fs.hdfs.hadoopconf"; public static ClusterClient createClusterClient(Options launcherOptions) throws Exception { @@ -96,6 +102,8 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { if (StringUtils.isNotBlank(yarnConfDir)) { try { + boolean isHighAvailability; + config.setString(HADOOP_CONF, yarnConfDir); FileSystem.initialize(config); @@ -123,7 +131,9 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { throw new RuntimeException("No flink session found on yarn cluster."); } - if (config.getString(HA_CLUSTER_ID, null) == null) { + isHighAvailability = config.getString(HIGH_AVAILABILITY, NODE).equals(ZOOKEEPER); + + if (isHighAvailability && config.getString(HA_CLUSTER_ID, null) == null) { config.setString(HA_CLUSTER_ID, applicationId.toString()); } From 90a77555d41a577b31e792bc0777d88df3703c3e Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 10 Apr 2020 09:34:55 +0800 Subject: [PATCH 240/523] fix --- .../java/com/dtstack/flink/sql/launcher/LauncherMain.java | 5 +---- .../dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index da2e881a6..650a22629 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -19,8 +19,6 @@ package com.dtstack.flink.sql.launcher; -//import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; - import com.dtstack.flink.sql.constrant.ConfigConstrant; import com.google.common.collect.Lists; import com.alibaba.fastjson.JSON; @@ -50,7 +48,6 @@ import java.io.IOException; import java.io.InputStreamReader; import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -129,7 +126,7 @@ private static String[] parseJson(String[] args) { StringBuilder lastStr = new StringBuilder(); try { FileInputStream fileInputStream = new FileInputStream(args[0]); - InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, StandardCharsets.UTF_8); + InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, Charsets.UTF_8); reader = new BufferedReader(inputStreamReader); String tempString; while ((tempString = reader.readLine()) != null) { diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index 287844c87..956495ef2 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -75,7 +75,6 @@ public static String submit(Options launcherOptions, JobGraph jobGraph, Configur String flinkJobId = jobGraph.getJobID().toString(); String tips = String.format("deploy per_job with appId: %s, jobId: %s", applicationId, flinkJobId); - System.out.println(tips); LOG.info(tips); return applicationId; From 49159c7cf175d3a29d03519d9647a2610b069f4c Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 10 Apr 2020 09:42:18 +0800 Subject: [PATCH 241/523] fix --- .../perjob/PerJobClusterClientBuilder.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index c40f9432d..9556956eb 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -67,7 +67,7 @@ public class PerJobClusterClientBuilder { public void init(String yarnConfDir, Configuration flinkConfig, Properties userConf) throws Exception { - if(Strings.isNullOrEmpty(yarnConfDir)) { + if (Strings.isNullOrEmpty(yarnConfDir)) { throw new RuntimeException("parameters of yarn is required"); } userConf.forEach((key, val) -> flinkConfig.setString(key.toString(), val.toString())); @@ -79,8 +79,7 @@ public void init(String yarnConfDir, Configuration flinkConfig, Properties userC yarnClient.init(yarnConf); yarnClient.start(); - System.out.println("----init yarn success ----"); -// LOG.info("----init yarn success ----"); + LOG.info("----init yarn success ----"); } public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJarPath, Options launcherOptions, JobGraph jobGraph) @@ -131,8 +130,8 @@ public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJ private static void fillJobGraphClassPath(JobGraph jobGraph) throws MalformedURLException { Map jobCacheFileConfig = jobGraph.getUserArtifacts(); - for(Map.Entry tmp : jobCacheFileConfig.entrySet()){ - if(tmp.getKey().startsWith("class_path")){ + for (Map.Entry tmp : jobCacheFileConfig.entrySet()) { + if (tmp.getKey().startsWith("class_path")) { jobGraph.getClasspaths().add(new URL("file:" + tmp.getValue().filePath)); } } @@ -141,8 +140,8 @@ private static void fillJobGraphClassPath(JobGraph jobGraph) throws MalformedURL private List getPluginPathToShipFiles(JobGraph jobGraph) { List shipFiles = new ArrayList<>(); Map jobCacheFileConfig = jobGraph.getUserArtifacts(); - for(Map.Entry tmp : jobCacheFileConfig.entrySet()){ - if(tmp.getKey().startsWith("class_path")){ + for (Map.Entry tmp : jobCacheFileConfig.entrySet()) { + if (tmp.getKey().startsWith("class_path")) { shipFiles.add(new File(tmp.getValue().filePath)); } } From d7baa87ed2b6ae691aed67805134d9d8d545c684 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 10 Apr 2020 09:46:44 +0800 Subject: [PATCH 242/523] fix --- .../com/dtstack/flink/sql/launcher/LauncherMain.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 3811d349d..750356db2 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -40,11 +40,12 @@ import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -60,6 +61,8 @@ public class LauncherMain { private static final String CORE_JAR = "core"; + private static final Logger LOG = LoggerFactory.getLogger(LauncherMain.class); + private static String SP = File.separator; private static String getLocalCoreJarPath(String localSqlRootJar) throws Exception { @@ -80,7 +83,9 @@ public static void main(String[] args) throws Exception { confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - if(mode.equals(ClusterMode.local.name())) { + LOG.info("current job mode is {}", mode); + + if (mode.equals(ClusterMode.local.name())) { String[] localArgs = argList.toArray(new String[0]); Main.main(localArgs); return; From c4ce9357300195d20e2a2a72fe59a99c3a4998c4 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 10 Apr 2020 10:43:35 +0800 Subject: [PATCH 243/523] add zk session expired error --- .../com/dtstack/flink/sql/enums/EConnectionErrorCode.java | 2 +- .../com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java b/core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java index feed69785..a0a281c4a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java @@ -30,7 +30,7 @@ */ public enum EConnectionErrorCode { ERROR_NOT_MATCH(0, "错误信息未匹配", new String[]{}), - CONN_DB_INVALID(1, "数据库连接失效，请重新打开", new String[]{"the last packet successfully received from the server was"}), + CONN_DB_INVALID(1, "数据库连接失效，请重新打开", new String[]{"the last packet successfully received from the server was", "Zookeeper session has been expired"}), CONN_DB_FAILED(2, "数据库连接失败，请检查用户名或密码是否正确", new String[]{"Access denied for user"}), DB_TABLE_NOT_EXIST(3, "操作的表名不存在", new String[]{"doesn't exist"}); diff --git a/core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java b/core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java index a43b253df..6ae55c425 100644 --- a/core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java +++ b/core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java @@ -34,6 +34,10 @@ public void testResolveErrorCodeFromException(){ EConnectionErrorCode errorCode = EConnectionErrorCode.resolveErrorCodeFromException(new Exception("The last packet successfully received from the server was 179 milliseconds")); + EConnectionErrorCode ckSessionExpired = + EConnectionErrorCode.resolveErrorCodeFromException(new Exception("Excepetion: Zookeeper session has been expired")); + Assert.assertEquals(errorCode, EConnectionErrorCode.CONN_DB_INVALID); + Assert.assertEquals(ckSessionExpired, EConnectionErrorCode.CONN_DB_INVALID); } } From 4a4f21c0d31fe6614ddd94c741abfd88daa5ff30 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 10 Apr 2020 12:00:19 +0800 Subject: [PATCH 244/523] rdb async connection pool size --- .../clickhouse/ClickhouseAsyncReqRow.java | 4 +-- .../dtstack/flink/sql/side/SideTableInfo.java | 30 +++++++++++++++++++ .../flink/sql/table/AbsSideTableParser.java | 8 +++++ .../flink/sql/side/db2/Db2AsyncReqRow.java | 4 +-- .../sql/side/impala/ImpalaAsyncReqRow.java | 29 ++++++++---------- .../sql/side/mysql/MysqlAsyncReqRow.java | 4 +-- .../sql/side/oracle/OracleAsyncReqRow.java | 4 +-- .../sql/side/polardb/PolardbAsyncReqRow.java | 4 +-- .../postgresql/PostgresqlAsyncReqRow.java | 4 +-- .../flink/sql/side/rdb/all/RdbAllReqRow.java | 8 +++++ .../sql/side/rdb/async/RdbAsyncReqRow.java | 13 +++++++- .../sql/side/rdb/table/RdbSideTableInfo.java | 12 ++++++++ .../side/sqlserver/SqlserverAsyncReqRow.java | 4 +-- 13 files changed, 96 insertions(+), 32 deletions(-) diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index 2b5223412..db49e4337 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -48,7 +48,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); clickhouseClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", CLICKHOUSE_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -59,7 +59,7 @@ public void open(Configuration parameters) throws Exception { System.setProperty("vertx.disableFileCPResolving", "true"); VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSQLClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java index 8c78578fb..32f065123 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java @@ -55,6 +55,8 @@ public abstract class SideTableInfo extends TableInfo implements Serializable { public static final String ASYNC_TIMEOUT_NUM_KEY = "asyncTimeoutNum"; + public static final String ASYNC_REQ_POOL_KEY = "asyncPoolSize"; + private String cacheType = "none";//None or LRU or ALL private int cacheSize = 10000; @@ -65,6 +67,11 @@ public abstract class SideTableInfo extends TableInfo implements Serializable { private int asyncTimeout=10000; + /** + * async operator req outside conn pool size, egg rdb conn pool size + */ + private int asyncPoolSize = 0; + private int asyncTimeoutNumLimit = Integer.MAX_VALUE; private boolean partitionedJoin = false; @@ -156,4 +163,27 @@ public void setAsyncTimeoutNumLimit(int asyncTimeoutNumLimit) { this.asyncTimeoutNumLimit = asyncTimeoutNumLimit; } + public int getAsyncPoolSize() { + return asyncPoolSize; + } + + public void setAsyncPoolSize(int asyncPoolSize) { + this.asyncPoolSize = asyncPoolSize; + } + + @Override + public String toString() { + return "Cache Info{" + + "cacheType='" + cacheType + '\'' + + ", cacheSize=" + cacheSize + + ", cacheTimeout=" + cacheTimeout + + ", asyncCapacity=" + asyncCapacity + + ", asyncTimeout=" + asyncTimeout + + ", asyncPoolSize=" + asyncPoolSize + + ", asyncTimeoutNumLimit=" + asyncTimeoutNumLimit + + ", partitionedJoin=" + partitionedJoin + + ", cacheMode='" + cacheMode + '\'' + + '}'; + } + } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java index bf0b8a7a4..403360400 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java @@ -23,6 +23,8 @@ import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.MathUtil; +import org.apache.flink.util.Preconditions; + import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -114,6 +116,12 @@ protected void parseCacheProp(SideTableInfo sideTableInfo, Map p sideTableInfo.setAsyncTimeoutNumLimit(asyncTimeoutNum); } } + + if (props.containsKey(SideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())) { + Integer asyncPoolSize = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())); + Preconditions.checkArgument(asyncPoolSize > 0 && asyncPoolSize <= 20, "asyncPoolSize size limit (0,20]"); + sideTableInfo.setAsyncPoolSize(asyncPoolSize); + } } } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index 3d85c6fbd..cda2ed128 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -59,7 +59,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); db2lientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", DB2_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -71,7 +71,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSQLClient(JDBCClient.createNonShared(vertx, db2lientConfig)); diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 725e7c3ee..2b76ec26c 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -57,33 +57,28 @@ public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List>> newCache = Maps.newConcurrentMap(); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 2d767ccd9..971285e52 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -22,6 +22,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import com.dtstack.flink.sql.util.DateUtil; import io.vertx.core.json.JsonArray; @@ -59,7 +60,9 @@ public class RdbAsyncReqRow extends AsyncReqRow { public final static int DEFAULT_VERTX_WORKER_POOL_SIZE = Runtime.getRuntime().availableProcessors() * 2; - public final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; + public final static int DEFAULT_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; + + public final static int MAX_DB_CONN_POOL_SIZE_LIMIT = 20; public final static int DEFAULT_IDLE_CONNECTION_TEST_PEROID = 60; @@ -73,6 +76,14 @@ public class RdbAsyncReqRow extends AsyncReqRow { public RdbAsyncReqRow(SideInfo sideInfo) { super(sideInfo); + init(sideInfo); + } + + protected void init(SideInfo sideInfo) { + RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + int defaultAsyncPoolSize = Math.min(MAX_DB_CONN_POOL_SIZE_LIMIT, DEFAULT_DB_CONN_POOL_SIZE); + int rdbPoolSize = rdbSideTableInfo.getAsyncPoolSize() > 0 ? rdbSideTableInfo.getAsyncPoolSize() : defaultAsyncPoolSize; + rdbSideTableInfo.setAsyncPoolSize(rdbPoolSize); } @Override diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java index 2c8c18fcd..fdb024b1d 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java @@ -101,4 +101,16 @@ public String getPassword() { public void setPassword(String password) { this.password = password; } + + @Override + public String toString() { + String cacheInfo = super.toString(); + String connectionInfo = "Rdb Side Connection Info{" + + "url='" + url + '\'' + + ", tableName='" + tableName + '\'' + + ", schema='" + schema + '\'' + + '}'; + return cacheInfo + " , " + connectionInfo; + } + } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index cb4240ca6..9fc2c35cd 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -51,7 +51,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); sqlserverClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", SQLSERVER_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -64,7 +64,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSQLClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); From 330be96a729c630506a8737ce198d5ee6a77b99e Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 10 Apr 2020 12:56:06 +0800 Subject: [PATCH 245/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=AE=B0=E8=BF=B0?= =?UTF-8?q?=E5=92=8C=E8=B6=85=E8=BF=87=E6=AC=A1=E6=95=B0=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E9=80=80=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/table/AbstractSideTableParser.java | 6 ---- .../sql/side/rdb/async/RdbAsyncReqRow.java | 33 ++++++++++--------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 7df86b3cb..de46e8409 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -108,12 +108,6 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0){ - sideTableInfo.setAsyncTimeoutNumLimit(asyncTimeoutNum); - } - } if(props.containsKey(AbstractSideTableInfo.ASYNC_FAIL_MAX_NUM_KEY.toLowerCase())){ Long asyncFailNum = MathUtil.getLongVal(props.get(AbstractSideTableInfo.ASYNC_FAIL_MAX_NUM_KEY.toLowerCase())); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index c86434528..ca6dcc38b 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.side.rdb.async; import com.dtstack.flink.sql.enums.ECacheContentType; +import com.dtstack.flink.sql.metric.MetricConstant; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.CacheMissVal; @@ -31,6 +32,7 @@ import io.vertx.ext.sql.SQLConnection; import org.apache.calcite.sql.JoinType; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -77,10 +79,12 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private final static AtomicBoolean CONN_STATUS = new AtomicBoolean(true); - private final static AtomicLong TIMOUT_NUM = new AtomicLong(0); + private final static AtomicLong FAIL_NUM = new AtomicLong(0); private Logger logger = LoggerFactory.getLogger(getClass()); + private Counter counter = getRuntimeContext().getMetricGroup().counter(MetricConstant.DT_NUM_SIDE_PARSE_ERROR_RECORDS); + public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); } @@ -123,7 +127,12 @@ private void connectWithRetry(Map inputParams, CRow input, Resul logger.error("getConnection error", conn.cause()); } if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3L)){ - outByJoinType(resultFuture, conn.cause()); + if(FAIL_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ + counter.inc(); + resultFuture.completeExceptionally(conn.cause()); + } else { + dealMissKey(input, resultFuture); + } finishFlag.set(true); } conn.result().close(); @@ -135,6 +144,7 @@ private void connectWithRetry(Map inputParams, CRow input, Resul handleQuery(conn.result(), inputParams, input, resultFuture); finishFlag.set(true); } catch (Exception e) { + dealFillDataError(resultFuture, e, null); logger.error("", e); } finally { latch.countDown(); @@ -200,12 +210,13 @@ private void handleQuery(SQLConnection connection,Map inputParam JsonArray params = new JsonArray(Lists.newArrayList(inputParams.values())); connection.queryWithParams(sideInfo.getSqlCondition(), params, rs -> { if (rs.failed()) { - if(TIMOUT_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ - outByJoinType(resultFuture, rs.cause()); - return; + if(FAIL_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ + LOG.error("Cannot retrieve the data from the database", rs.cause()); + counter.inc(); + resultFuture.completeExceptionally(rs.cause()); + } else { + dealMissKey(input, resultFuture); } - LOG.error("Cannot retrieve the data from the database", rs.cause()); - resultFuture.complete(null); return; } @@ -244,12 +255,4 @@ private void handleQuery(SQLConnection connection,Map inputParam }); } - private void outByJoinType(ResultFuture resultFuture, Throwable e){ - if(sideInfo.getJoinType() == JoinType.LEFT){ - resultFuture.complete(null); - return; - } - resultFuture.completeExceptionally(e); - } - } From 2027b415a930d35417575669a31212b5f5b93523 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 13 Apr 2020 19:08:49 +0800 Subject: [PATCH 246/523] parse where case --- .../flink/sql/parser/InsertSqlParser.java | 16 +++------- .../flink/sql/side/SidePredicatesParser.java | 21 +++++++------ .../flink/sql/util/FieldReplaceUtil.java | 17 ++++++++++- .../dtstack/flink/sql/util/TableUtils.java | 30 ++++++++++++------- 4 files changed, 52 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java index e2940c4f7..e29ac4bef 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java @@ -26,6 +26,7 @@ import org.apache.calcite.sql.parser.SqlParser; import org.apache.commons.lang3.StringUtils; import com.google.common.collect.Lists; +import org.apache.flink.table.calcite.FlinkPlannerImpl; import java.util.List; @@ -53,18 +54,9 @@ public static InsertSqlParser newInstance(){ @Override public void parseSql(String sql, SqlTree sqlTree) { - SqlParser.Config config = SqlParser - .configBuilder() - .setLex(Lex.MYSQL) - .build(); - - SqlParser sqlParser = SqlParser.create(sql,config); - SqlNode sqlNode = null; - try { - sqlNode = sqlParser.parseStmt(); - } catch (SqlParseException e) { - throw new RuntimeException("", e); - } + + FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); + SqlNode sqlNode = flinkPlanner.parse(sql); SqlParseResult sqlParseResult = new SqlParseResult(); parseNode(sqlNode, sqlParseResult); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java index 0902bf39f..99dc3507b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java @@ -138,16 +138,19 @@ private void extractPredicateInfo(SqlNode whereNode, List predica private void fillPredicateInfoToList(SqlBasicCall whereNode, List predicatesInfoList, String operatorName, SqlKind operatorKind, int fieldIndex, int conditionIndex) { - SqlIdentifier fieldFullPath = (SqlIdentifier) whereNode.getOperands()[fieldIndex]; - if (fieldFullPath.names.size() == 2) { - String ownerTable = fieldFullPath.names.get(0); - String fieldName = fieldFullPath.names.get(1); - String content = (operatorKind == SqlKind.BETWEEN) ? whereNode.getOperands()[conditionIndex].toString() + " AND " + - whereNode.getOperands()[2].toString() : whereNode.getOperands()[conditionIndex].toString(); + SqlNode sqlNode = whereNode.getOperands()[fieldIndex]; + if (sqlNode.getKind() == SqlKind.IDENTIFIER) { + SqlIdentifier fieldFullPath = (SqlIdentifier) sqlNode; + if (fieldFullPath.names.size() == 2) { + String ownerTable = fieldFullPath.names.get(0); + String fieldName = fieldFullPath.names.get(1); + String content = (operatorKind == SqlKind.BETWEEN) ? whereNode.getOperands()[conditionIndex].toString() + " AND " + + whereNode.getOperands()[2].toString() : whereNode.getOperands()[conditionIndex].toString(); - PredicateInfo predicateInfo = PredicateInfo.builder().setOperatorName(operatorName).setOperatorKind(operatorKind.toString()) - .setOwnerTable(ownerTable).setFieldName(fieldName).setCondition(content).build(); - predicatesInfoList.add(predicateInfo); + PredicateInfo predicateInfo = PredicateInfo.builder().setOperatorName(operatorName).setOperatorKind(operatorKind.toString()) + .setOwnerTable(ownerTable).setFieldName(fieldName).setCondition(content).build(); + predicatesInfoList.add(predicateInfo); + } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java index 10919ca5b..697954dd6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java @@ -146,7 +146,22 @@ private static SqlNode replaceNodeInfo(SqlNode groupNode, } return sqlBasicCall; - }else{ + } else if (groupNode.getKind() == CASE) { + SqlCase sqlCase = (SqlCase) groupNode; + + for (int i = 0; i < sqlCase.getWhenOperands().size(); i++) { + SqlNode sqlNode = sqlCase.getWhenOperands().getList().get(i); + SqlNode replaceNode = replaceSelectFieldName(sqlNode, oldTbName, newTbName, mappingField); + sqlCase.getWhenOperands().set(i,replaceNode); + } + + for (int i = 0; i < sqlCase.getThenOperands().size(); i++) { + SqlNode sqlNode = sqlCase.getThenOperands().getList().get(i); + SqlNode replaceNode = replaceSelectFieldName(sqlNode, oldTbName, newTbName, mappingField); + sqlCase.getThenOperands().set(i,replaceNode); + } + return sqlCase; + } else { return groupNode; } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 1527c2db7..579beaf63 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -568,7 +568,12 @@ private static void replaceConditionNode(SqlNode selectNode, String oldTbName, S }else if(selectNode.getKind() == OTHER){ //不处理 return; - }else{ + } else if (selectNode.getKind() == CASE) { + SqlCase sqlCase = (SqlCase) selectNode; + + sqlCase.getWhenOperands().getList().forEach(sqlNode -> replaceConditionNode(sqlNode, oldTbName, newTbName, fieldReplaceRef)); + sqlCase.getThenOperands().getList().forEach(sqlNode -> replaceConditionNode(sqlNode, oldTbName, newTbName, fieldReplaceRef)); + } else { throw new RuntimeException(String.format("not support node kind of %s to replace name now.", selectNode.getKind())); } } @@ -579,14 +584,14 @@ private static void replaceConditionNode(SqlNode selectNode, String oldTbName, S * @param fieldInfos */ public static void getConditionRefTable(SqlNode selectNode, Set fieldInfos) { - if(selectNode.getKind() == IDENTIFIER){ + if (selectNode.getKind() == IDENTIFIER) { SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; fieldInfos.add(sqlIdentifier.toString()); return; - }else if(selectNode.getKind() == LITERAL || selectNode.getKind() == LITERAL_CHAIN){//字面含义 + } else if (selectNode.getKind() == LITERAL || selectNode.getKind() == LITERAL_CHAIN) {//字面含义 return; - }else if( AGGREGATE.contains(selectNode.getKind()) + } else if (AGGREGATE.contains(selectNode.getKind()) || AVG_AGG_FUNCTIONS.contains(selectNode.getKind()) || COMPARISON.contains(selectNode.getKind()) || selectNode.getKind() == OTHER_FUNCTION @@ -616,15 +621,15 @@ public static void getConditionRefTable(SqlNode selectNode, Set fieldInf || selectNode.getKind() == TIMESTAMP_DIFF || selectNode.getKind() == LIKE - ){ + ) { SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; - for(int i=0; i fieldInf } return; - }else if(selectNode.getKind() == OTHER){ + } else if (selectNode.getKind() == OTHER) { //不处理 return; - }else{ + } else if (selectNode.getKind() == CASE) { + SqlCase sqlCase = (SqlCase) selectNode; + + sqlCase.getWhenOperands().getList().forEach(sqlNode -> getConditionRefTable(sqlNode, fieldInfos)); + sqlCase.getThenOperands().getList().forEach(sqlNode -> getConditionRefTable(sqlNode, fieldInfos)); + } else { throw new RuntimeException(String.format("not support node kind of %s to replace name now.", selectNode.getKind())); } } From a7ea7f3c8eda717d50c9e1edf8baefe883686363 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 13 Apr 2020 21:34:38 +0800 Subject: [PATCH 247/523] add log --- core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index fae0ba9aa..6f0dce58a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -66,6 +66,7 @@ public void open(Configuration parameters) throws Exception { super.open(parameters); initCache(); initMetric(); + LOG.info("async dim table config info: {} ", sideInfo.getSideTableInfo().toString()); } private void initCache(){ From 63f3ab158c2900deed4063c123a08f12754f1430 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 14 Apr 2020 10:05:10 +0800 Subject: [PATCH 248/523] fix spelling and kafka params map name --- .../dtstack/flink/sql/exec/ExecuteProcessHelper.java | 10 +++++----- .../flink/sql/sink/kafka/table/KafkaSinkTableInfo.java | 8 ++++---- .../sql/source/kafka/table/KafkaSourceTableInfo.java | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 7ffe7d1dc..4bd33363c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -266,7 +266,7 @@ public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrl */ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { - Set pluginClassPatshSets = Sets.newHashSet(); + Set pluginClassPathSets = Sets.newHashSet(); WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); for (AbstractTableInfo tableInfo : sqlTree.getTableInfoMap().values()) { @@ -304,7 +304,7 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment registerTableCache.put(tableInfo.getName(), regTable); URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractSourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); - pluginClassPatshSets.add(sourceTablePathUrl); + pluginClassPathSets.add(sourceTablePathUrl); } else if (tableInfo instanceof AbstractTargetTableInfo) { TableSink tableSink = StreamSinkFactory.getTableSink((AbstractTargetTableInfo) tableInfo, localSqlPluginPath); @@ -312,18 +312,18 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); - pluginClassPatshSets.add(sinkTablePathUrl); + pluginClassPathSets.add(sinkTablePathUrl); } else if (tableInfo instanceof AbstractSideTableInfo) { String sideOperator = ECacheType.ALL.name().equals(((AbstractSideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; sideTableMap.put(tableInfo.getName(), (AbstractSideTableInfo) tableInfo); URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, AbstractSideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); - pluginClassPatshSets.add(sideTablePathUrl); + pluginClassPathSets.add(sideTablePathUrl); } else { throw new RuntimeException("not support table type:" + tableInfo.getType()); } } - return pluginClassPatshSets; + return pluginClassPathSets; } /** diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java index f1bef088e..2804c71de 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java @@ -58,7 +58,7 @@ public class KafkaSinkTableInfo extends AbstractTargetTableInfo { private String bootstrapServers; - public Map kafkaParam = new HashMap(); + public Map kafkaParams = new HashMap(); private String topic; @@ -73,15 +73,15 @@ public class KafkaSinkTableInfo extends AbstractTargetTableInfo { private String updateMode; public void addKafkaParam(String key, String value) { - kafkaParam.put(key, value); + kafkaParams.put(key, value); } public String getKafkaParam(String key) { - return kafkaParam.get(key); + return kafkaParams.get(key); } public Set getKafkaParamKeys() { - return kafkaParam.keySet(); + return kafkaParams.keySet(); } public String getBootstrapServers() { diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index 1210aba40..62453166f 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -69,7 +69,7 @@ public class KafkaSourceTableInfo extends AbstractSourceTableInfo { private String fieldDelimiter; - public Map kafkaParam = new HashMap<>(); + public Map kafkaParams = new HashMap<>(); public String getBootstrapServers() { @@ -113,15 +113,15 @@ public void setTopicIsPattern(Boolean topicIsPattern) { } public void addKafkaParam(Map kafkaParam) { - kafkaParam.putAll(kafkaParam); + kafkaParams.putAll(kafkaParam); } public String getKafkaParam(String key) { - return kafkaParam.get(key); + return kafkaParams.get(key); } public Set getKafkaParamKeys() { - return kafkaParam.keySet(); + return kafkaParams.keySet(); } public String getSourceDataType() { From 86bd4b6a06a925f05955fcdb6e61263063e6316d Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 14 Apr 2020 11:41:04 +0800 Subject: [PATCH 249/523] fix 24602 --- .../flink/sql/side/rdb/all/RdbAllReqRow.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java index 60e3d817f..2fc1d3b23 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java @@ -83,12 +83,12 @@ protected void initCache() throws SQLException { protected void reloadCache() { //reload cacheRef and replace to old cacheRef Map>> newCache = Maps.newConcurrentMap(); - cacheRef.set(newCache); try { loadData(newCache); } catch (SQLException e) { throw new RuntimeException(e); } + cacheRef.set(newCache); LOG.info("----- rdb all cacheRef reload end:{}", Calendar.getInstance()); } @@ -112,9 +112,9 @@ public void flatMap(CRow value, Collector out) throws Exception { List> cacheList = cacheRef.get().get(cacheKey); if (CollectionUtils.isEmpty(cacheList) && sideInfo.getJoinType() == JoinType.LEFT) { out.collect(new CRow(fillData(value.row(), null), value.change())); + } else if (!CollectionUtils.isEmpty(cacheList)) { + cacheList.forEach(one -> out.collect(new CRow(fillData(value.row(), one), value.change()))); } - - cacheList.stream().forEach(one -> out.collect(new CRow(fillData(value.row(), one), value.change()))); } @Override @@ -141,8 +141,8 @@ public Row fillData(Row input, Object sideInput) { } /** - * covert flink time attribute.Type information for indicating event or processing time. - * However, it behaves like a regular SQL timestamp but is serialized as Long. + * covert flink time attribute.Type information for indicating event or processing time. + * However, it behaves like a regular SQL timestamp but is serialized as Long. * * @param entry * @param obj @@ -222,7 +222,8 @@ public int getFetchSize() { } /** - * get jdbc connection + * get jdbc connection + * * @param dbURL * @param userName * @param password From 6943b54f1b9964cbceeae35a8b2309c1fb08b249 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 14 Apr 2020 15:54:21 +0800 Subject: [PATCH 250/523] =?UTF-8?q?fix=20=E5=88=AB=E5=90=8D=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/dtstack/flink/sql/side/SideInfo.java | 7 ++++--- .../dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java index 029c86e25..d8a22e022 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java @@ -93,7 +93,7 @@ public void parseSelectFields(JoinInfo joinInfo){ if(fieldInfo.getTable().equalsIgnoreCase(sideTableName)){ String sideFieldName = sideTableInfo.getPhysicalFields().getOrDefault(fieldInfo.getFieldName(), fieldInfo.getFieldName()); fields.add(sideFieldName); - sideSelectFieldsType.put(sideTableFieldIndex, getTargetFieldType(sideFieldName)); + sideSelectFieldsType.put(sideTableFieldIndex, getTargetFieldType(fieldInfo.getFieldName())); sideFieldIndex.put(i, sideTableFieldIndex); sideFieldNameIndex.put(i, sideFieldName); sideTableFieldIndex++; @@ -132,12 +132,13 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName){ String leftTableName = left.getComponent(0).getSimple(); String leftField = left.getComponent(1).getSimple(); + Map physicalFields = sideTableInfo.getPhysicalFields(); String rightTableName = right.getComponent(0).getSimple(); String rightField = right.getComponent(1).getSimple(); if(leftTableName.equalsIgnoreCase(sideTableName)){ - equalFieldList.add(leftField); + equalFieldList.add(physicalFields.get(leftField)); int equalFieldIndex = -1; for(int i=0; i physicalFields = sideTableInfo.getPhysicalFields(); String leftTableName = left.getComponent(0).getSimple(); String leftField = left.getComponent(1).getSimple(); @@ -94,7 +96,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { String rightField = right.getComponent(1).getSimple(); if (leftTableName.equalsIgnoreCase(sideTableName)) { - equalFieldList.add(leftField); + equalFieldList.add(physicalFields.get(leftField)); int equalFieldIndex = -1; for (int i = 0; i < rowTypeInfo.getFieldNames().length; i++) { String fieldName = rowTypeInfo.getFieldNames()[i]; @@ -110,7 +112,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { } else if (rightTableName.equalsIgnoreCase(sideTableName)) { - equalFieldList.add(rightField); + equalFieldList.add(physicalFields.get(rightField)); int equalFieldIndex = -1; for (int i = 0; i < rowTypeInfo.getFieldNames().length; i++) { String fieldName = rowTypeInfo.getFieldNames()[i]; From 75adb0b0cbb56e57797d915a9aadfe87ee0dc635 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 14 Apr 2020 17:04:14 +0800 Subject: [PATCH 251/523] side predicate skip where case --- .../com/dtstack/flink/sql/side/SidePredicatesParser.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java index 99dc3507b..4d6112aa8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java @@ -30,6 +30,7 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.table.calcite.FlinkPlannerImpl; import java.util.List; @@ -147,6 +148,10 @@ private void fillPredicateInfoToList(SqlBasicCall whereNode, List String content = (operatorKind == SqlKind.BETWEEN) ? whereNode.getOperands()[conditionIndex].toString() + " AND " + whereNode.getOperands()[2].toString() : whereNode.getOperands()[conditionIndex].toString(); + if (StringUtils.containsIgnoreCase(content,SqlKind.CASE.toString())) { + return; + } + PredicateInfo predicateInfo = PredicateInfo.builder().setOperatorName(operatorName).setOperatorKind(operatorKind.toString()) .setOwnerTable(ownerTable).setFieldName(fieldName).setCondition(content).build(); predicatesInfoList.add(predicateInfo); From 9bf9a659b25dc4b3c22511e53cef7fe00e3def6b Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 14 Apr 2020 17:46:52 +0800 Subject: [PATCH 252/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcoalesce=E5=87=BD?= =?UTF-8?q?=E6=95=B0=E4=B8=8D=E8=83=BD=E7=94=A8=E9=97=AE=E9=A2=98=E3=80=90?= =?UTF-8?q?http://redmine.prod.dtstack.cn/issues/24635=3Fissue=5Fcount=3D2?= =?UTF-8?q?&issue=5Fposition=3D1&next=5Fissue=5Fid=3D24600=E3=80=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java index 10919ca5b..4b05840d6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java @@ -253,6 +253,7 @@ public static SqlNode replaceSelectFieldName(SqlNode selectNode, || selectNode.getKind() == TIMESTAMP_ADD || selectNode.getKind() == TIMESTAMP_DIFF || selectNode.getKind() == LIKE + || selectNode.getKind() == COALESCE ){ SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; From 04550b05a4b7c47d56536f6b75d86ac17f0b2bbf Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 15 Apr 2020 14:44:58 +0800 Subject: [PATCH 253/523] =?UTF-8?q?=E5=A4=84=E7=90=86=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/AbstractSideTableInfo.java | 2 -- .../flink/sql/side/BaseAsyncReqRow.java | 23 +++++++------- .../sql/side/rdb/async/RdbAsyncReqRow.java | 30 +++++-------------- 3 files changed, 21 insertions(+), 34 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index 71a309411..19d3cc5fb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -54,8 +54,6 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements public static final String ASYNC_TIMEOUT_KEY = "asyncTimeout"; - public static final String ASYNC_TIMEOUT_NUM_KEY = "asyncTimeoutNum"; - public static final String ASYNC_FAIL_MAX_NUM_KEY = "asyncFailMaxNum"; private String cacheType = "none";//None or LRU or ALL diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index bfe14c3ea..7f8478183 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -51,6 +51,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; /** * All interfaces inherit naming rules: type + "AsyncReqRow" such as == "MysqlAsyncReqRow @@ -64,10 +65,9 @@ public abstract class BaseAsyncReqRow extends RichAsyncFunction impl private static final Logger LOG = LoggerFactory.getLogger(BaseAsyncReqRow.class); private static final long serialVersionUID = 2098635244857937717L; private RuntimeContext runtimeContext; - + private final static AtomicLong FAIL_NUM = new AtomicLong(0); private static int TIMEOUT_LOG_FLUSH_NUM = 10; private int timeOutNum = 0; - protected BaseSideInfo sideInfo; protected transient Counter parseErrorRecords; @@ -125,7 +125,7 @@ protected void dealMissKey(CRow input, ResultFuture resultFuture){ Row row = fillData(input.row(), null); resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, input); + dealFillDataError(input, resultFuture, e); } }else{ resultFuture.complete(null); @@ -205,7 +205,7 @@ private void invokeWithCache(Map inputParams, CRow input, Result Row row = fillData(input.row(), val); resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, input); + dealFillDataError(input, resultFuture, e); } } else if (ECacheContentType.MultiLine == val.getType()) { try { @@ -216,7 +216,7 @@ private void invokeWithCache(Map inputParams, CRow input, Result } resultFuture.complete(rowList); } catch (Exception e) { - dealFillDataError(resultFuture, e, input); + dealFillDataError(input, resultFuture, e); } } else { resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); @@ -256,11 +256,14 @@ protected void cancelTimerWhenComplete(ResultFuture resultFuture, Schedule } } - protected void dealFillDataError(ResultFuture resultFuture, Exception e, Object sourceData) { - LOG.debug("source data {} join side table error ", sourceData); - LOG.debug("async buid row error..{}", e); - parseErrorRecords.inc(); - resultFuture.complete(Collections.emptyList()); + protected void dealFillDataError(CRow input, ResultFuture resultFuture, Throwable e) { + if(FAIL_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ + LOG.info("dealFillDataError", e); + parseErrorRecords.inc(); + resultFuture.completeExceptionally(e); + } else { + dealMissKey(input, resultFuture); + } } @Override diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index ca6dcc38b..9571d7962 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -20,7 +20,6 @@ package com.dtstack.flink.sql.side.rdb.async; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.metric.MetricConstant; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.CacheMissVal; @@ -30,9 +29,7 @@ import io.vertx.core.json.JsonArray; import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; -import org.apache.calcite.sql.JoinType; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -79,12 +76,8 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private final static AtomicBoolean CONN_STATUS = new AtomicBoolean(true); - private final static AtomicLong FAIL_NUM = new AtomicLong(0); - private Logger logger = LoggerFactory.getLogger(getClass()); - private Counter counter = getRuntimeContext().getMetricGroup().counter(MetricConstant.DT_NUM_SIDE_PARSE_ERROR_RECORDS); - public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); } @@ -98,7 +91,12 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { + AtomicLong networkLogCounter = new AtomicLong(0L); while (!CONN_STATUS.get()){//network is unhealth + //todo:统一计数 + if(networkLogCounter.getAndIncrement() % 1000 == 0){ + LOG.info("network unhealth to block task"); + } Thread.sleep(100); } rdbSqlClient.getConnection(conn -> { @@ -127,12 +125,7 @@ private void connectWithRetry(Map inputParams, CRow input, Resul logger.error("getConnection error", conn.cause()); } if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3L)){ - if(FAIL_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ - counter.inc(); - resultFuture.completeExceptionally(conn.cause()); - } else { - dealMissKey(input, resultFuture); - } + dealFillDataError(input, resultFuture, conn.cause()); finishFlag.set(true); } conn.result().close(); @@ -144,8 +137,7 @@ private void connectWithRetry(Map inputParams, CRow input, Resul handleQuery(conn.result(), inputParams, input, resultFuture); finishFlag.set(true); } catch (Exception e) { - dealFillDataError(resultFuture, e, null); - logger.error("", e); + dealFillDataError(input, resultFuture, e); } finally { latch.countDown(); } @@ -210,13 +202,7 @@ private void handleQuery(SQLConnection connection,Map inputParam JsonArray params = new JsonArray(Lists.newArrayList(inputParams.values())); connection.queryWithParams(sideInfo.getSqlCondition(), params, rs -> { if (rs.failed()) { - if(FAIL_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ - LOG.error("Cannot retrieve the data from the database", rs.cause()); - counter.inc(); - resultFuture.completeExceptionally(rs.cause()); - } else { - dealMissKey(input, resultFuture); - } + dealFillDataError(input, resultFuture, rs.cause()); return; } From 96936193e3e18ad7d85b536401afffd30d95c8c1 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 15 Apr 2020 15:29:49 +0800 Subject: [PATCH 254/523] update doc quickStart --- README.md | 4 +- docs/quickStart.md | 224 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 210 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e5d4622eb..30c95552c 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ FlinkStreamSQL * 自定义create view 语法 * 自定义create function 语法 * 实现了流与维表的join - * 支持原生FLinkSQL所有的语法 - * 扩展了输入和输出的性能指标到promethus + * 支持原生FlinkSQL所有的语法 + * 扩展了输入和输出的性能指标到Task metrics ## 目录 diff --git a/docs/quickStart.md b/docs/quickStart.md index dda046062..f1607c5e7 100644 --- a/docs/quickStart.md +++ b/docs/quickStart.md @@ -8,7 +8,7 @@ ### 1.2 执行环境 * Java: JDK8及以上 -* Flink集群: 1.4,1.5,1.8（单机模式不需要安装Flink集群） +* Flink集群: 1.4,1.5,1.8,1.9,1.10（单机模式不需要安装Flink集群） * 操作系统：理论上不限 * kerberos环境需要在flink-conf.yaml配置security.kerberos.login.keytab以及security.kerberos.login.principal参数，配置案例: ``` @@ -29,7 +29,6 @@ zookeeper.sasl.login-context-name: Client ``` mvn clean package -Dmaven.test.skip - ``` ####可运行的目录结构: @@ -37,9 +36,9 @@ mvn clean package -Dmaven.test.skip | |-----bin | |--- submit.sh 任务启动脚本 -|-----lib -| |--- sql.launcher.jar 包存储路径，是任务提交的入口（需要手动移动到该目录） -|-----plugins: 插件包存储路径(mvn 打包之后会自动将jar移动到该目录下) +|-----lib: launcher包存储路径，是任务提交的入口 +| |--- sql.launcher.jar +|-----sqlplugins: 插件包存储路径(mvn 打包之后会自动将jar移动到该目录下) | |--- core.jar | |--- xxxsource | |--- xxxsink @@ -49,15 +48,208 @@ mvn clean package -Dmaven.test.skip #### 1.4.1 启动命令 +```shell script +# 脚本启动 +sh submit.sh + -mode yarn + -name flink1.10_yarnSession + -sql F:\dtstack\stressTest\flinkStreamSql\1.10_dev\sql\flink1100.sql + -localSqlPluginPath F:\dtstack\project\flinkStreamSQL\plugins + -remoteSqlPluginPath F:\dtstack\project\flinkStreamSQL\plugins + -flinkconf F:\dtstack\flink\flink-1.10.0\conf + -yarnconf F:\dtstack\flinkStreamSql\yarnConf_node1 + -flinkJarPath F:\dtstack\flink\flink-1.10.0\lib + -pluginLoadMode shipfile + -confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} + -yarnSessionConf {\"yid\":\"application_1586851105774_0014\"} +``` +```shell script +# 通过idea启动 程序入口类LaucherMain +# Run/Debug Configurations中设置Program arguments +-mode yarnPer +-sql /home/wen/Desktop/flink_stream_sql_conf/sql/stressTest.sql +-name stressTestAll +-localSqlPluginPath /home/wen/IdeaProjects/flinkStreamSQL/plugins +-remoteSqlPluginPath /home/wen/IdeaProjects/flinkStreamSQL/plugins +-flinkconf /home/wen/Desktop/flink_stream_sql_conf/flinkConf +-yarnconf /home/wen/Desktop/flink_stream_sql_conf/yarnConf_node1 +-flinkJarPath /home/wen/Desktop/dtstack/flink-1.8.1/lib +-pluginLoadMode shipfile +-confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} +-queue c +``` +#### 1.4.2 命令参数说明 +* **mode** + * 描述：执行模式，也就是flink集群的工作模式 + * local: 本地模式 + * standalone: 提交到独立部署模式的flink集群 + * yarn: 提交到yarn模式的flink集群，该模式下需要提前启动一个yarn-session，使用默认名"Flink session cluster" + * yarnPer: yarn per_job模式提交(即创建新flink application)，默认名为flink任务名称 + * 必选：否 + * 默认值：local + +* **name** + * 描述：flink 任务对应名称。 + * 必选：是 + * 默认值：无 + +* **sql** + * 描述：待执行的flink sql所在路径 。 + * 必选：是 + * 默认值：无 + +* **localSqlPluginPath** + * 描述：本地插件根目录地址，也就是打包后产生的plugins目录。 + * 必选：是 + * 默认值：无 + +* **remoteSqlPluginPath** + * 描述：flink执行集群上的插件根目录地址（将打包好的插件存放到各个flink节点上,如果是yarn集群需要存放到所有的nodemanager上）。 + * 必选：否 + * 默认值：无 + +* **addjar** + * 描述：扩展jar路径,当前主要是UDF定义的jar； + * 必选：否 + * 默认值：无 + +* **confProp** + * 描述：一些参数设置 + * 必选：是 （如无参数填写空json即可） + * 默认值：{} + * 可选参数: + * sql.ttl.min: 最小过期时间,大于0的整数,如1d、1h(d\D:天,h\H:小时,m\M:分钟,s\s:秒) + * sql.ttl.max: 最大过期时间,大于0的整数,如2d、2h(d\D:天,h\H:小时,m\M:分钟,s\s:秒),需同时设置最小时间,且比最小时间大5分钟 + * state.backend: 任务状态后端，可选为MEMORY,FILESYSTEM,ROCKSDB，默认为flinkconf中的配置。 + * state.checkpoints.dir: FILESYSTEM,ROCKSDB状态后端文件系统存储路径，例如：hdfs://ns1/dtInsight/flink180/checkpoints。 + * state.backend.incremental: ROCKSDB状态后端是否开启增量checkpoint,默认为true。 + * sql.env.parallelism: 默认并行度设置 + * sql.max.env.parallelism: 最大并行度设置 + * time.characteristic: 可选值[ProcessingTime|IngestionTime|EventTime] + * sql.checkpoint.interval: 设置了该参数表明开启checkpoint(ms) + * sql.checkpoint.mode: 可选值[EXACTLY_ONCE|AT_LEAST_ONCE] + * sql.checkpoint.timeout: 生成checkpoint的超时时间(ms) + * sql.max.concurrent.checkpoints: 最大并发生成checkpoint数 + * sql.checkpoint.cleanup.mode: 默认是不会将checkpoint存储到外部存储,[true(任务cancel之后会删除外部存储)|false(外部存储需要手动删除)] + * flinkCheckpointDataURI: 设置checkpoint的外部存储路径,根据实际的需求设定文件路径,hdfs://, file:// + * jobmanager.memory.mb: per_job模式下指定jobmanager的内存大小(单位MB, 默认值:768) + * taskmanager.memory.mb: per_job模式下指定taskmanager的内存大小(单位MB, 默认值:768) + * taskmanager.num: per_job模式下指定taskmanager的实例数(默认1) + * taskmanager.slots：per_job模式下指定每个taskmanager对应的slot数量(默认1) + * savePointPath：任务恢复点的路径（默认无） + * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） + * restore.enable：是否失败重启（默认是true） + * failure.interval：衡量失败率的时间段，单位分钟（默认6m） + * delay.interval：连续两次重启尝试间的间隔，单位是秒（默认10s） + * logLevel: 日志级别动态配置（默认info） + * [prometheus 相关参数](docs/prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 + + +* **flinkconf** + * 描述：flink配置文件所在的目录（单机模式下不需要），如/hadoop/flink-1.10.0/conf + * 必选：否 + * 默认值：无 + +* **yarnconf** + * 描述：Hadoop配置文件（包括hdfs和yarn）所在的目录（单机模式下不需要），如/hadoop/etc/hadoop + * 必选：否 + * 默认值：无 + +* **flinkJarPath** + * 描述：per_job 模式提交需要指定本地的flink jar存放路径 + * 必选：否 + * 默认值：无 + +* **queue** + * 描述：per_job 模式下指定的yarn queue + * 必选：否 + * 默认值：default + +* **pluginLoadMode** + * 描述：per_job 模式下的插件包加载方式。classpath:从每台机器加载插件包，shipfile:将需要插件从提交的节点上传到hdfs,不需要每台安装插件 + * 必选：否 + * 默认值：classpath + +* **yarnSessionConf** + * 描述：yarn session 模式下指定的运行的一些参数，[可参考](https://ci.apache.org/projects/flink/flink-docs-release-1.8/ops/cli.html),目前只支持指定yid + * 必选：否 + * 默认值：{} + +## 1.5 任务样例 + ``` -sh submit.sh -sql D:\sideSql.txt --name xctest --remoteSqlPluginPath /opt/dtstack/150_flinkplugin/sqlplugin --localSqlPluginPath D:\gitspace\flinkStreamSQL\plugins --addjar \["udf.jar\"\] --mode yarn --flinkconf D:\flink_home\kudu150etc --yarnconf D:\hadoop\etc\hadoopkudu --confProp \{\"time.characteristic\":\"EventTime\",\"sql.checkpoint.interval\":10000\} --yarnSessionConf \{\"yid\":\"application_1564971615273_38182\"} -``` \ No newline at end of file +# 一个kafka join all维表 sink kafka的样例 +CREATE TABLE MyTable( + id bigint, + name varchar, + address varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_in', + groupId = 'flink_sql', + timezone = 'Asia/Shanghai', + topicIsPattern = 'false', + parallelism = '1' +); + +CREATE TABLE sideTable( + id bigint, + school varchar, + home varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME +)WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/tiezhu', + userName='dtstack', + password='you-guess', + tableName='stressTest', + cache='ALL', + parallelism='1', + asyncCapacity='100' +); + +CREATE TABLE MyResult( + id bigint, + name varchar, + address varchar, + home varchar, + school varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_out', + parallelism = '1' +); + + +insert +into + MyResult + select + t1.id AS id, + t1.name AS name, + t1.address AS address, + t2.school AS school, + t2.home AS home + from + ( + select + id, + name, + address + from + MyTable + ) t1 + left join sideTable t2 + on t2.id = t2.id; +``` + +# 招聘 +1.大数据平台开发工程师，想了解岗位详细信息可以添加本人微信号ysqwhiletrue,注明招聘，如有意者发送简历至sishu@dtstack.com。 + From e496e3376be02a05badcc88cea7d0d2950fa9eda Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 15 Apr 2020 15:37:24 +0800 Subject: [PATCH 255/523] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=B8=AA=E5=88=AB?= =?UTF-8?q?=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/quickStart.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/quickStart.md b/docs/quickStart.md index f1607c5e7..52fc3af4e 100644 --- a/docs/quickStart.md +++ b/docs/quickStart.md @@ -22,11 +22,8 @@ zookeeper.sasl.service-name: zookeeper zookeeper.sasl.login-context-name: Client ``` - ### 1.3 打包 - 进入项目根目录，使用maven打包： - ``` mvn clean package -Dmaven.test.skip ``` @@ -38,13 +35,13 @@ mvn clean package -Dmaven.test.skip | |--- submit.sh 任务启动脚本 |-----lib: launcher包存储路径，是任务提交的入口 | |--- sql.launcher.jar -|-----sqlplugins: 插件包存储路径(mvn 打包之后会自动将jar移动到该目录下) +|-----plugins: 插件包存储路径(mvn 打包之后会自动将jar移动到该目录下) | |--- core.jar | |--- xxxsource | |--- xxxsink | |--- xxxside ``` -### 1.4 启动 +### 1.4 快速启动 #### 1.4.1 启动命令 @@ -115,7 +112,7 @@ sh submit.sh * **confProp** * 描述：一些参数设置 - * 必选：是 （如无参数填写空json即可） + * 必选：否 * 默认值：{} * 可选参数: * sql.ttl.min: 最小过期时间,大于0的整数,如1d、1h(d\D:天,h\H:小时,m\M:分钟,s\s:秒) @@ -176,9 +173,8 @@ sh submit.sh * 默认值：{} ## 1.5 任务样例 - ``` -# 一个kafka join all维表 sink kafka的样例 +# 一个 kafka source join all维表 到 kafka sink的样例 CREATE TABLE MyTable( id bigint, name varchar, From 88d73ed3122c3f5cede49eef0abfb8cf2864e36e Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 16 Apr 2020 09:42:52 +0800 Subject: [PATCH 256/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9config=E4=B8=AD?= =?UTF-8?q?=E9=83=A8=E5=88=86=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/config.md | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/docs/config.md b/docs/config.md index 48aeefc5a..3fb6ac723 100644 --- a/docs/config.md +++ b/docs/config.md @@ -9,8 +9,8 @@ sh submit.sh -key1 val1 -key2 val2 * 描述：执行模式，也就是flink集群的工作模式 * local: 本地模式 * standalone: 提交到独立部署模式的flink集群 - * yarn: 提交到yarn模式的flink集群(即提交到已有flink集群) - * yarnPer: yarn per_job模式提交(即创建新flink application) + * yarn: 提交到yarn模式的flink集群，该模式下需要提前启动一个yarn-session，使用默认名"Flink session cluster" + * yarnPer: yarn per_job模式提交(即创建新flink application)，默认名为flink任务名称 * 必选：否 * 默认值：local @@ -28,7 +28,7 @@ sh submit.sh -key1 val1 -key2 val2 * 默认值：无 * **sql** - * 描述：执行flink sql 的主体语句。 + * 描述：待执行的flink sql所在路径 。 * 必选：是 * 默认值：无 @@ -44,15 +44,13 @@ sh submit.sh -key1 val1 -key2 val2 * **addjar** * 描述：扩展jar路径,当前主要是UDF定义的jar； - * 格式：json * 必选：否 * 默认值：无 * **confProp** * 描述：一些参数设置 - * 格式: json - * 必选：是 （如无参数填写空json即可） - * 默认值：无 + * 必选：否 + * 默认值：{} * 可选参数: * sql.ttl.min: 最小过期时间,大于0的整数,如1d、1h(d\D:天,h\H:小时,m\M:分钟,s\s:秒) * sql.ttl.max: 最大过期时间,大于0的整数,如2d、2h(d\D:天,h\H:小时,m\M:分钟,s\s:秒),需同时设置最小时间,且比最小时间大5分钟 @@ -79,7 +77,7 @@ sh submit.sh -key1 val1 -key2 val2 * **flinkconf** - * 描述：flink配置文件所在的目录（单机模式下不需要），如/hadoop/flink-1.4.0/conf + * 描述：flink配置文件所在的目录（单机模式下不需要），如/hadoop/flink-1.10.0/conf * 必选：否 * 默认值：无 @@ -91,15 +89,15 @@ sh submit.sh -key1 val1 -key2 val2 * **flinkJarPath** * 描述：yarnPer 模式提交需要指定本地的flink jar存放路径 * 必选：否 - * 默认值：false + * 默认值：无 * **queue** * 描述：yarnPer 模式下指定的yarn queue * 必选：否 - * 默认值：false + * 默认值：default * **yarnSessionConf** * 描述：yarn session 模式下指定的运行的一些参数，[可参考](https://ci.apache.org/projects/flink/flink-docs-release-1.8/ops/cli.html),目前只支持指定yid * 必选：否 - * 默认值：false + * 默认值：{} \ No newline at end of file From 9b451a8ef1ef906e9efef9f5d95050cb47719713 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 16 Apr 2020 09:59:32 +0800 Subject: [PATCH 257/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9quickStart=E6=96=87?= =?UTF-8?q?=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/quickStart.md | 202 ++++----------------------------------------- 1 file changed, 16 insertions(+), 186 deletions(-) diff --git a/docs/quickStart.md b/docs/quickStart.md index 52fc3af4e..a51f234e3 100644 --- a/docs/quickStart.md +++ b/docs/quickStart.md @@ -11,7 +11,7 @@ * Flink集群: 1.4,1.5,1.8,1.9,1.10（单机模式不需要安装Flink集群） * 操作系统：理论上不限 * kerberos环境需要在flink-conf.yaml配置security.kerberos.login.keytab以及security.kerberos.login.principal参数，配置案例: -``` +```yaml #提交到hadoop环境一定要配置fs.hdfs.hadoopconf参数 fs.hdfs.hadoopconf: /Users/maqi/tmp/hadoopconf/hadoop_250 security.kerberos.login.use-ticket-cache: true @@ -24,7 +24,7 @@ zookeeper.sasl.login-context-name: Client ### 1.3 打包 进入项目根目录，使用maven打包： -``` +```shell script mvn clean package -Dmaven.test.skip ``` @@ -59,193 +59,23 @@ sh submit.sh -pluginLoadMode shipfile -confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} -yarnSessionConf {\"yid\":\"application_1586851105774_0014\"} -``` -```shell script -# 通过idea启动 程序入口类LaucherMain -# Run/Debug Configurations中设置Program arguments --mode yarnPer --sql /home/wen/Desktop/flink_stream_sql_conf/sql/stressTest.sql --name stressTestAll --localSqlPluginPath /home/wen/IdeaProjects/flinkStreamSQL/plugins --remoteSqlPluginPath /home/wen/IdeaProjects/flinkStreamSQL/plugins --flinkconf /home/wen/Desktop/flink_stream_sql_conf/flinkConf --yarnconf /home/wen/Desktop/flink_stream_sql_conf/yarnConf_node1 --flinkJarPath /home/wen/Desktop/dtstack/flink-1.8.1/lib --pluginLoadMode shipfile --confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} --queue c -``` -#### 1.4.2 命令参数说明 -* **mode** - * 描述：执行模式，也就是flink集群的工作模式 - * local: 本地模式 - * standalone: 提交到独立部署模式的flink集群 - * yarn: 提交到yarn模式的flink集群，该模式下需要提前启动一个yarn-session，使用默认名"Flink session cluster" - * yarnPer: yarn per_job模式提交(即创建新flink application)，默认名为flink任务名称 - * 必选：否 - * 默认值：local - -* **name** - * 描述：flink 任务对应名称。 - * 必选：是 - * 默认值：无 -* **sql** - * 描述：待执行的flink sql所在路径 。 - * 必选：是 - * 默认值：无 - -* **localSqlPluginPath** - * 描述：本地插件根目录地址，也就是打包后产生的plugins目录。 - * 必选：是 - * 默认值：无 - -* **remoteSqlPluginPath** - * 描述：flink执行集群上的插件根目录地址（将打包好的插件存放到各个flink节点上,如果是yarn集群需要存放到所有的nodemanager上）。 - * 必选：否 - * 默认值：无 - -* **addjar** - * 描述：扩展jar路径,当前主要是UDF定义的jar； - * 必选：否 - * 默认值：无 - -* **confProp** - * 描述：一些参数设置 - * 必选：否 - * 默认值：{} - * 可选参数: - * sql.ttl.min: 最小过期时间,大于0的整数,如1d、1h(d\D:天,h\H:小时,m\M:分钟,s\s:秒) - * sql.ttl.max: 最大过期时间,大于0的整数,如2d、2h(d\D:天,h\H:小时,m\M:分钟,s\s:秒),需同时设置最小时间,且比最小时间大5分钟 - * state.backend: 任务状态后端，可选为MEMORY,FILESYSTEM,ROCKSDB，默认为flinkconf中的配置。 - * state.checkpoints.dir: FILESYSTEM,ROCKSDB状态后端文件系统存储路径，例如：hdfs://ns1/dtInsight/flink180/checkpoints。 - * state.backend.incremental: ROCKSDB状态后端是否开启增量checkpoint,默认为true。 - * sql.env.parallelism: 默认并行度设置 - * sql.max.env.parallelism: 最大并行度设置 - * time.characteristic: 可选值[ProcessingTime|IngestionTime|EventTime] - * sql.checkpoint.interval: 设置了该参数表明开启checkpoint(ms) - * sql.checkpoint.mode: 可选值[EXACTLY_ONCE|AT_LEAST_ONCE] - * sql.checkpoint.timeout: 生成checkpoint的超时时间(ms) - * sql.max.concurrent.checkpoints: 最大并发生成checkpoint数 - * sql.checkpoint.cleanup.mode: 默认是不会将checkpoint存储到外部存储,[true(任务cancel之后会删除外部存储)|false(外部存储需要手动删除)] - * flinkCheckpointDataURI: 设置checkpoint的外部存储路径,根据实际的需求设定文件路径,hdfs://, file:// - * jobmanager.memory.mb: per_job模式下指定jobmanager的内存大小(单位MB, 默认值:768) - * taskmanager.memory.mb: per_job模式下指定taskmanager的内存大小(单位MB, 默认值:768) - * taskmanager.num: per_job模式下指定taskmanager的实例数(默认1) - * taskmanager.slots：per_job模式下指定每个taskmanager对应的slot数量(默认1) - * savePointPath：任务恢复点的路径（默认无） - * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） - * restore.enable：是否失败重启（默认是true） - * failure.interval：衡量失败率的时间段，单位分钟（默认6m） - * delay.interval：连续两次重启尝试间的间隔，单位是秒（默认10s） - * logLevel: 日志级别动态配置（默认info） - * [prometheus 相关参数](docs/prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 - - -* **flinkconf** - * 描述：flink配置文件所在的目录（单机模式下不需要），如/hadoop/flink-1.10.0/conf - * 必选：否 - * 默认值：无 - -* **yarnconf** - * 描述：Hadoop配置文件（包括hdfs和yarn）所在的目录（单机模式下不需要），如/hadoop/etc/hadoop - * 必选：否 - * 默认值：无 - -* **flinkJarPath** - * 描述：per_job 模式提交需要指定本地的flink jar存放路径 - * 必选：否 - * 默认值：无 - -* **queue** - * 描述：per_job 模式下指定的yarn queue - * 必选：否 - * 默认值：default - -* **pluginLoadMode** - * 描述：per_job 模式下的插件包加载方式。classpath:从每台机器加载插件包，shipfile:将需要插件从提交的节点上传到hdfs,不需要每台安装插件 - * 必选：否 - * 默认值：classpath - -* **yarnSessionConf** - * 描述：yarn session 模式下指定的运行的一些参数，[可参考](https://ci.apache.org/projects/flink/flink-docs-release-1.8/ops/cli.html),目前只支持指定yid - * 必选：否 - * 默认值：{} - -## 1.5 任务样例 +# mode: 任务启动的模式 +# name: 本次任务名称 +# sql: 本次任务执行sql脚本 +# localSqPluginPath: 本地插件包根目录地址 +# remoteSqlPluginPath: flink执行集群上的插件根目录地址 +# flinkconf: flink配置文件所在目录（单机模式下不需要） +# yarnconf: Hadoop配置文件（包括hdfs和yarn）所在目录 +# flinkJarPath: yarnPer模式提交需要指定本地的flink jar存放路径 +# pluginLoadMode:yarnPer模式下的插件包加载方式 +# confProp: 其他额外参数配置 +# yarnSessionConf: yarnSession模式下指定的运行参数，目前只支持指定yid ``` -# 一个 kafka source join all维表 到 kafka sink的样例 -CREATE TABLE MyTable( - id bigint, - name varchar, - address varchar -)WITH( - type = 'kafka10', - bootstrapServers = '172.16.101.224:9092', - zookeeperQuorm = '172.16.100.188:2181/kafka', - offsetReset = 'latest', - topic = 'tiezhu_test_in', - groupId = 'flink_sql', - timezone = 'Asia/Shanghai', - topicIsPattern = 'false', - parallelism = '1' -); - -CREATE TABLE sideTable( - id bigint, - school varchar, - home varchar, - PRIMARY KEY(id), - PERIOD FOR SYSTEM_TIME -)WITH( - type='mysql', - url='jdbc:mysql://172.16.8.109:3306/tiezhu', - userName='dtstack', - password='you-guess', - tableName='stressTest', - cache='ALL', - parallelism='1', - asyncCapacity='100' -); +参数具体细节请看[命令参数说明](./config.md) -CREATE TABLE MyResult( - id bigint, - name varchar, - address varchar, - home varchar, - school varchar -)WITH( - type = 'kafka10', - bootstrapServers = '172.16.101.224:9092', - zookeeperQuorm = '172.16.100.188:2181/kafka', - offsetReset = 'latest', - topic = 'tiezhu_test_out', - parallelism = '1' -); - - -insert -into - MyResult - select - t1.id AS id, - t1.name AS name, - t1.address AS address, - t2.school AS school, - t2.home AS home - from - ( - select - id, - name, - address - from - MyTable - ) t1 - left join sideTable t2 - on t2.id = t2.id; -``` +任务sql详情请看[任务样例](./demo.md) -# 招聘 +### 招聘 1.大数据平台开发工程师，想了解岗位详细信息可以添加本人微信号ysqwhiletrue,注明招聘，如有意者发送简历至sishu@dtstack.com。 From 28085f5e8e4c59c8d5b2d3f0db4bc136956ab919 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 16 Apr 2020 10:06:43 +0800 Subject: [PATCH 258/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9quickStart=E9=83=A8?= =?UTF-8?q?=E5=88=86=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/quickStart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quickStart.md b/docs/quickStart.md index a51f234e3..d7b613092 100644 --- a/docs/quickStart.md +++ b/docs/quickStart.md @@ -65,7 +65,7 @@ sh submit.sh # sql: 本次任务执行sql脚本 # localSqPluginPath: 本地插件包根目录地址 # remoteSqlPluginPath: flink执行集群上的插件根目录地址 -# flinkconf: flink配置文件所在目录（单机模式下不需要） +# flinkconf: flink配置文件所在目录（local模式下不需要） # yarnconf: Hadoop配置文件（包括hdfs和yarn）所在目录 # flinkJarPath: yarnPer模式提交需要指定本地的flink jar存放路径 # pluginLoadMode:yarnPer模式下的插件包加载方式 From 393c98e5cd089f58ac678866d37228e7d2d27fbe Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 16 Apr 2020 10:50:38 +0800 Subject: [PATCH 259/523] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E6=97=A5=E5=BF=97logger=E5=92=8C=E9=87=87=E7=94=A8=E5=85=A8?= =?UTF-8?q?=E5=B1=80=E8=AE=A1=E6=95=B0=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java | 6 ++---- .../dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 8 ++------ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 7f8478183..0e59f4590 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -51,7 +51,6 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; /** * All interfaces inherit naming rules: type + "AsyncReqRow" such as == "MysqlAsyncReqRow @@ -65,7 +64,6 @@ public abstract class BaseAsyncReqRow extends RichAsyncFunction impl private static final Logger LOG = LoggerFactory.getLogger(BaseAsyncReqRow.class); private static final long serialVersionUID = 2098635244857937717L; private RuntimeContext runtimeContext; - private final static AtomicLong FAIL_NUM = new AtomicLong(0); private static int TIMEOUT_LOG_FLUSH_NUM = 10; private int timeOutNum = 0; protected BaseSideInfo sideInfo; @@ -257,9 +255,9 @@ protected void cancelTimerWhenComplete(ResultFuture resultFuture, Schedule } protected void dealFillDataError(CRow input, ResultFuture resultFuture, Throwable e) { - if(FAIL_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ + parseErrorRecords.inc(); + if(parseErrorRecords.getCount() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ LOG.info("dealFillDataError", e); - parseErrorRecords.inc(); resultFuture.completeExceptionally(e); } else { dealMissKey(input, resultFuture); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 9571d7962..b34e8297c 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -76,13 +76,10 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private final static AtomicBoolean CONN_STATUS = new AtomicBoolean(true); - private Logger logger = LoggerFactory.getLogger(getClass()); - public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); } - @Override protected void preInvoke(CRow input, ResultFuture resultFuture){ @@ -93,7 +90,6 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul AtomicLong networkLogCounter = new AtomicLong(0L); while (!CONN_STATUS.get()){//network is unhealth - //todo:统一计数 if(networkLogCounter.getAndIncrement() % 1000 == 0){ LOG.info("network unhealth to block task"); } @@ -122,7 +118,7 @@ private void connectWithRetry(Map inputParams, CRow input, Resul try { if(conn.failed()){ if(failCounter.getAndIncrement() % 1000 == 0){ - logger.error("getConnection error", conn.cause()); + LOG.error("getConnection error", conn.cause()); } if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3L)){ dealFillDataError(input, resultFuture, conn.cause()); @@ -146,7 +142,7 @@ private void connectWithRetry(Map inputParams, CRow input, Resul try { latch.wait(); } catch (InterruptedException e) { - logger.error("", e); + LOG.error("", e); } } From d394f209f7dc44f4880066db0ed56ee0ff6eca8b Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 16 Apr 2020 10:55:18 +0800 Subject: [PATCH 260/523] =?UTF-8?q?quickStart=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=9B=9B=E7=A7=8D=E6=A8=A1=E5=BC=8F=E5=91=BD=E4=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/quickStart.md | 48 +++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/docs/quickStart.md b/docs/quickStart.md index d7b613092..f2dcc39ab 100644 --- a/docs/quickStart.md +++ b/docs/quickStart.md @@ -46,7 +46,27 @@ mvn clean package -Dmaven.test.skip #### 1.4.1 启动命令 ```shell script -# 脚本启动 +# local模式命令 +sh submit.sh + -mode local + -name local_test + -sql F:\dtstack\stressTest\flinkStreamSql\stressTest.sql + -localSqlPluginPath F:\dtstack\project\flinkStreamSQL\plugins + +#standalone模式命令 +sh submit.sh + -mode standalone + -sql F:\dtstack\flinkStreamSql\tiezhu\twodimjoin.sql + -name wtz_standalone_flinkStreamSql + -localSqlPluginPath F:\dtstack\project\flinkStreamSQL\plugins + -remoteSqlPluginPath /home/admin/dtstack/flinkStreamSQL/plugins + -flinkconf F:\dtstack\flinkStreamSql\localhost\flinkConf + -yarnconf F:\dtstack\flinkStreamSql\localhost\hadoop + -flinkJarPath F:\Java\flink-1.8.2-bin-scala_2.12\flink-1.8.2\lib + -pluginLoadMode shipfile + -confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} + +# yarn模式命令 sh submit.sh -mode yarn -name flink1.10_yarnSession @@ -57,20 +77,22 @@ sh submit.sh -yarnconf F:\dtstack\flinkStreamSql\yarnConf_node1 -flinkJarPath F:\dtstack\flink\flink-1.10.0\lib -pluginLoadMode shipfile - -confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} -yarnSessionConf {\"yid\":\"application_1586851105774_0014\"} -# mode: 任务启动的模式 -# name: 本次任务名称 -# sql: 本次任务执行sql脚本 -# localSqPluginPath: 本地插件包根目录地址 -# remoteSqlPluginPath: flink执行集群上的插件根目录地址 -# flinkconf: flink配置文件所在目录（local模式下不需要） -# yarnconf: Hadoop配置文件（包括hdfs和yarn）所在目录 -# flinkJarPath: yarnPer模式提交需要指定本地的flink jar存放路径 -# pluginLoadMode:yarnPer模式下的插件包加载方式 -# confProp: 其他额外参数配置 -# yarnSessionConf: yarnSession模式下指定的运行参数，目前只支持指定yid +# yarnPer模式命令 +sh submit.sh + -mode yarnPer + -sql /home/wen/Desktop/flink_stream_sql_conf/sql/Test01.sql + -name TestAll + -localSqlPluginPath /home/wen/IdeaProjects/flinkStreamSQL/plugins + -remoteSqlPluginPath /home/wen/IdeaProjects/flinkStreamSQL/plugins + -flinkconf /home/wen/Desktop/flink_stream_sql_conf/flinkConf + -yarnconf /home/wen/Desktop/flink_stream_sql_conf/yarnConf_node1 + -flinkJarPath /home/wen/Desktop/dtstack/flink-1.8.1/lib + -pluginLoadMode shipfile + -confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} + -queue c + ``` 参数具体细节请看[命令参数说明](./config.md) From d68751dfe9422d1f659256396663b6eb1ef8f3c0 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 16 Apr 2020 11:00:06 +0800 Subject: [PATCH 261/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9quickStart=E6=8E=92?= =?UTF-8?q?=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/quickStart.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/quickStart.md b/docs/quickStart.md index f2dcc39ab..a5bf1fa9a 100644 --- a/docs/quickStart.md +++ b/docs/quickStart.md @@ -23,6 +23,7 @@ zookeeper.sasl.login-context-name: Client ``` ### 1.3 打包 + 进入项目根目录，使用maven打包： ```shell script mvn clean package -Dmaven.test.skip @@ -41,19 +42,19 @@ mvn clean package -Dmaven.test.skip | |--- xxxsink | |--- xxxside ``` -### 1.4 快速启动 - -#### 1.4.1 启动命令 +### 1.4 快速启动命令 +#### local模式命令 ```shell script -# local模式命令 sh submit.sh -mode local -name local_test -sql F:\dtstack\stressTest\flinkStreamSql\stressTest.sql -localSqlPluginPath F:\dtstack\project\flinkStreamSQL\plugins +``` -#standalone模式命令 +#### standalone模式命令 +```shell script sh submit.sh -mode standalone -sql F:\dtstack\flinkStreamSql\tiezhu\twodimjoin.sql @@ -65,8 +66,10 @@ sh submit.sh -flinkJarPath F:\Java\flink-1.8.2-bin-scala_2.12\flink-1.8.2\lib -pluginLoadMode shipfile -confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} +``` -# yarn模式命令 +#### yarn模式命令 +```shell script sh submit.sh -mode yarn -name flink1.10_yarnSession @@ -78,8 +81,10 @@ sh submit.sh -flinkJarPath F:\dtstack\flink\flink-1.10.0\lib -pluginLoadMode shipfile -yarnSessionConf {\"yid\":\"application_1586851105774_0014\"} +``` -# yarnPer模式命令 +#### yarnPer模式命令 +```shell script sh submit.sh -mode yarnPer -sql /home/wen/Desktop/flink_stream_sql_conf/sql/Test01.sql @@ -92,7 +97,6 @@ sh submit.sh -pluginLoadMode shipfile -confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} -queue c - ``` 参数具体细节请看[命令参数说明](./config.md) From 92baf834cd0a75230e5b5ff11fc5e0456b496070 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 16 Apr 2020 11:56:54 +0800 Subject: [PATCH 262/523] add rdbSideParams.md --- docs/plugin/rdbSideParams.md | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 docs/plugin/rdbSideParams.md diff --git a/docs/plugin/rdbSideParams.md b/docs/plugin/rdbSideParams.md new file mode 100644 index 000000000..26666bd5b --- /dev/null +++ b/docs/plugin/rdbSideParams.md @@ -0,0 +1,41 @@ +## 关系型数据库维表参数 + +适用于`MYSQL`,`ORACLE`,`SQLSERVER`,`POSTGRESQL`,`DB2`,`POLARDB`,`CLICKHOUSE`,`IMPALA`维表插件 + +### 维表参数 + +|参数名称|含义|是否必填|默认值| +|----|---|---|----| +| type | 维表类型， 例如:mysql |是|| +| url | 连接数据库 jdbcUrl |是|| +| userName | 连接用户名 |是|| +| password | 连接密码|是|| +| tableName| 表名称|是|| +| schema| 表空间|否|| +| cache | 维表缓存策略(NONE/LRU/ALL)|否|LRU| +| partitionedJoin | 是否在維表join之前先根据设定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| + +### 缓存策略 + +- NONE：不做内存缓存。每条流数据触发一次维表查询操作。 +- ALL: 任务启动时，一次性加载所有数据到内存，并进行缓存。适用于维表数据量较小的情况。 +- LRU: 任务执行时，根据维表关联条件使用异步算子加载维表数据，并进行缓存。 + +#### ALL全量维表参数 + +|参数名称|含义|默认值| +|----|---|----| +| cacheTTLMs | 缓存周期刷新时间 |60，单位s| + +#### LRU异步维表参数 + +|参数名称|含义|默认值| +|----|---|----| +| cacheTTLMs | LRU缓存写入后超时时间 |60，单位s| +| cacheSize | LRU缓存大小 |10000| +| cacheMode | 异步请求处理有序还是无序，可选：ordered，unordered |ordered| +| asyncCapacity | 异步线程容量 |100| +| asyncTimeout | 异步处理超时时间 |10000，单位毫秒| +| asyncPoolSize | 异步查询DB最大线程池，上限20 |min(20,Runtime.getRuntime().availableProcessors() * 2)| + + From 4566ece8e09a97af3e16e773f8298e018007b062 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 16 Apr 2020 14:27:11 +0800 Subject: [PATCH 263/523] oracle mysql side table doc --- docs/plugin/mysqlSide.md | 149 ++++++++++++++++++++++++----------- docs/plugin/oracleSide.md | 133 +++++++++++++++++++++++-------- docs/plugin/rdbSideParams.md | 1 + 3 files changed, 203 insertions(+), 80 deletions(-) diff --git a/docs/plugin/mysqlSide.md b/docs/plugin/mysqlSide.md index 6a0e0aea9..96e997e9e 100644 --- a/docs/plugin/mysqlSide.md +++ b/docs/plugin/mysqlSide.md @@ -1,11 +1,15 @@ ## 1.格式： + + 通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 + ``` CREATE TABLE tableName( colName cloType, ... PRIMARY KEY(keyInfo), - PERIOD FOR SYSTEM_TIME + PERIOD FOR SYSTEM_TIME )WITH( type='mysql', url='jdbcUrl', @@ -24,62 +28,117 @@ mysql-5.6.35 ## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | mysql表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| + + [关系型数据库维表参数](docs/rdbSideParams.md) -## 4.参数 +## 4.样例 - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型 mysql |是|| - | url | 连接mysql数据库 jdbcUrl |是|| - | userName | mysql连接用户名 |是|| - | password | mysql连接密码|是|| - | tableName | mysql表名称|是|| - | tableName | mysql 的表名称|是|| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - * cacheMode: (unordered|ordered)异步加载是有序还是无序,默认有序。 - * asyncCapacity:异步请求容量，默认1000 - * asyncTimeout：异步请求超时时间，默认10000毫秒 +### ALL全量维表定义 +``` + // 定义全量维表 +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='mysql', + url ='jdbc:mysql://172.16.10.204:3306/mqtest', + userName ='dtstack', + password ='1abc123', + tableName ='test_mysql_10', + cache ='ALL', + cacheTTLMs ='60000', + parallelism ='2' + ); -## 5.样例 ``` -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), +### LRU异步维表定义 + +``` +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , PERIOD FOR SYSTEM_TIME )WITH( - type='mysql', - url='jdbc:mysql://172.16.8.104:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='sidetest', + type ='mysql', + url ='jdbc:mysql://172.16.10.204:3306/mqtest', + userName ='dtstack', + password ='1abc123', + tableName ='yctest_mysql_10', + partitionedJoin ='false', cache ='LRU', cacheSize ='10000', cacheTTLMs ='60000', - cacheMode='unordered', - asyncCapacity='1000', - asyncTimeout='10000' - parallelism ='1', - partitionedJoin='false' + asyncPoolSize ='3', + parallelism ='2' ); +``` + +### MySQL异步维表关联 ``` +CREATE TABLE MyTable( + id int, + name varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='cannan_yctest01', + timezone='Asia/Shanghai', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + +CREATE TABLE MyResult( + id INT, + name VARCHAR + )WITH( + type ='mysql', + url ='jdbc:mysql://172.16.10.204:3306/mqtest', + userName ='dtstack', + password ='1abc123', + tableName ='yctest_mysql_mq', + updateMode ='append', + parallelism ='1', + batchSize ='100', + batchWaitInterval ='1000' + ); + +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='mysql', + url ='jdbc:mysql://172.16.10.204:3306/mqtest', + userName ='dtstack', + password ='1abc123', + tableName ='yctest_mysql_10', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + asyncPoolSize ='3', + parallelism ='1' + ); +insert +into + MyResult + select + m.id, + s.name + from + MyTable m + join + sideTable s + on m.id=s.id; +``` \ No newline at end of file diff --git a/docs/plugin/oracleSide.md b/docs/plugin/oracleSide.md index 47b8e36c7..1cb4e4cce 100644 --- a/docs/plugin/oracleSide.md +++ b/docs/plugin/oracleSide.md @@ -1,5 +1,13 @@ ## 1.格式： + + 通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 + + 此外, 针对oracle char类型自动补齐的特性，我们允许为char指定长度，确保维表查询时能够匹配数据。 + + + 注意：Oracle维表使用的字段大小写，需要和Oracle中定义的保持一致。 ``` CREATE TABLE tableName( colName cloType, @@ -22,43 +30,35 @@ # 2.支持版本 10g 11g -## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | oracle表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| - -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型 oracle |是|| - | url | 连接oracle数据库 jdbcUrl |是|| - | userName | oracle连接用户名 |是|| - | password | oracle连接密码|是|| - | tableName | oracle表名称|是|| - | schema | oracle 的schema|否|当前登录用户| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - * cacheMode: (unordered|ordered)异步加载是有序还是无序,默认有序。 - * asyncCapacity:异步请求容量，默认1000 - * asyncTimeout：异步请求超时时间，默认10000毫秒 - -## 5.样例 +## 3. 表结构定义 + [关系型数据库维表参数](docs/rdbSideParams.md) + +## 4.样例 + +### ALL全量维表定义 +``` +CREATE TABLE sideTable( + ID char(20), // oracle定义了char(20) + NAME varchar, + PRIMARY KEY (ID), + PERIOD FOR SYSTEM_TIME + )WITH( + type='oracle', + url = 'jdbc:oracle:thin:@172.16.8.178:1521:xe', + userName = 'system', + password = 'oracle', + tableName = 'SIDETEST1', + schema = 'dtstack', + cache = 'ALL', + cacheTTLMs ='60000' + ); +``` + +### LRU异步维表定义 + ``` create table sideTable( - channel varchar, + channel char, xccount int, PRIMARY KEY(channel), PERIOD FOR SYSTEM_TIME @@ -79,7 +79,70 @@ create table sideTable( schema = 'MQTEST' ); +``` + +### Oracle异步维表关联 + +```aidl +CREATE TABLE MyTable( + id varchar, + name varchar + --ts timestamp, + --tsDate Date + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest01', + timezone='Asia/Shanghai', + topicIsPattern ='false', + parallelism ='1' + ); + + +CREATE TABLE sideTable( + ID char(20), + NAME varchar, + PRIMARY KEY (ID), + PERIOD FOR SYSTEM_TIME + )WITH( + type='oracle', + url = 'jdbc:oracle:thin:@172.16.8.178:1521:xe', + userName = 'system', + password = 'oracle', + tableName = 'SIDETEST1', + --schema = 'dtstack', + cache = 'LRU', + asyncPoolSize ='3' + ); + + +CREATE TABLE MyResult( + NAME varchar, + ID char(20), + PRIMARY KEY (ID) + )WITH( + --type ='console', + type='oracle', + url = 'jdbc:oracle:thin:@172.16.8.178:1521:xe', + userName = 'system', + password = 'oracle', + tableName = 'SINK_TEST', + batchSize='1' + ); + + +INSERT INTO MyResult +SELECT + s.ID as ID, + m.name as NAME +FROM MyTable m +LEFT JOIN + sideTable s +ON + m.id=s.ID ``` diff --git a/docs/plugin/rdbSideParams.md b/docs/plugin/rdbSideParams.md index 26666bd5b..34d02978e 100644 --- a/docs/plugin/rdbSideParams.md +++ b/docs/plugin/rdbSideParams.md @@ -14,6 +14,7 @@ | schema| 表空间|否|| | cache | 维表缓存策略(NONE/LRU/ALL)|否|LRU| | partitionedJoin | 是否在維表join之前先根据设定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| +| parallelism | 处理后的数据流并行度|否|| ### 缓存策略 From b264114417d7044274972d3b3309abc612a02d30 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 16 Apr 2020 14:30:45 +0800 Subject: [PATCH 264/523] link rdbSideParams.md --- docs/plugin/mysqlSide.md | 2 +- docs/plugin/oracleSide.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugin/mysqlSide.md b/docs/plugin/mysqlSide.md index 96e997e9e..f28f09518 100644 --- a/docs/plugin/mysqlSide.md +++ b/docs/plugin/mysqlSide.md @@ -29,7 +29,7 @@ ## 3.表结构定义 - [关系型数据库维表参数](docs/rdbSideParams.md) + [关系型数据库维表参数](docs/plugin/rdbSideParams.md) ## 4.样例 diff --git a/docs/plugin/oracleSide.md b/docs/plugin/oracleSide.md index 1cb4e4cce..ee7e03331 100644 --- a/docs/plugin/oracleSide.md +++ b/docs/plugin/oracleSide.md @@ -31,7 +31,7 @@ 10g 11g ## 3. 表结构定义 - [关系型数据库维表参数](docs/rdbSideParams.md) + [关系型数据库维表参数](docs/plugin/rdbSideParams.md) ## 4.样例 From 1e2db49880757b2820544e0bf9a91081652fc95a Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 16 Apr 2020 15:52:06 +0800 Subject: [PATCH 265/523] =?UTF-8?q?update=20redis=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/config.md | 2 +- docs/plugin/redisSide.md | 46 +++++++++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/docs/config.md b/docs/config.md index 3fb6ac723..9aa8df994 100644 --- a/docs/config.md +++ b/docs/config.md @@ -73,7 +73,7 @@ sh submit.sh -key1 val1 -key2 val2 * savePointPath：任务恢复点的路径（默认无） * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） * logLevel: 日志级别动态配置（默认info） - * [prometheus 相关参数](docs/prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 + * [prometheus 相关参数](./prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 * **flinkconf** diff --git a/docs/plugin/redisSide.md b/docs/plugin/redisSide.md index f9b5e41f9..6aa335fea 100644 --- a/docs/plugin/redisSide.md +++ b/docs/plugin/redisSide.md @@ -22,29 +22,31 @@ redis5.0 ## 3.表结构定义 +注意：redis中没有表和schema的概念，参数中tableName是指符合命名规则的key，具体规则请看[缓存redis的存储命名规则] |参数名称|含义| |----|---| -| tableName | 注册到flink的表名称(可选填;不填默认和redis对应的表名称相同)| -| colName | 列名称，维表列名格式 表名:主键名:主键值:列名]| +| tableName | 注册到flinkStreamSql的表名称(可选填;不填默认和redis对应的"表"名称相同)| +| colName | 列名称，对应redis对应"表"的field| | colType | 列类型，当前只支持varchar| -| PRIMARY KEY |主键，多个字段做为联合主键时以逗号分隔 -| PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| +| PRIMARY KEY |主键，多个字段做为联合主键时以逗号分隔| +| PERIOD FOR SYSTEM_TIME | 关键字，表明该定义的表为维表信息| ## 4.参数 +参数详细说明请看[参数详细说明]() + |参数名称|含义|是否必填|默认值| |----|---|---|----| | type | 表明维表的类型[hbase|mysql|redis]|是|| | url | redis 的地址;格式ip:port[,ip:port]|是|| -| password | redis 的密码 |是|| +| password | redis 的密码 |否|空| | redisType | redis模式（1 单机，2 哨兵， 3 集群）| 是 | | masterName | 主节点名称（哨兵模式下为必填项） | 否 | -| database | reids 的数据库地址|否|| -| tableName | redis 的表名称|是|| +| database | reids 的数据库地址|否|0| +| tableName | redis 的“表”名称|是|| | cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - -------------- > 缓存策略 * NONE: 不做内存缓存 @@ -55,27 +57,37 @@ ## 5.样例 ``` -create table sideTable( - channel varchar, - xccount varchar, - PRIMARY KEY(channel), +CREATE TABLE MyRedis( + id varchar, + message varchar, + PRIMARY KEY(id), PERIOD FOR SYSTEM_TIME - )WITH( +)WITH( type='redis', url='172.16.10.79:6379', password='abc123', database='0', redisType = '1', - tableName='sidetest', + tableName = 'sideTable', cache = 'LRU', cacheTTLMs='10000' - ); +); ``` -## 6.缓存redis的存储结构规则 -``` +## 6.redis的存储命名规则 + redis使用散列类型 hash 数据结构，key=tableName_primaryKey1_primaryKey2,value={column1=value1, column2=value2} 如果以班级class表为例，id和name作为联合主键，那么redis的结构为 + +在样例中，tableName为sideTable，主键为id，column为id，message，所以对应的redis数据插入语句为 + +数据在redis中对应的数据存储情况为： +``` +192.168.80.105:6379> hgetall sideTable_5 +1) "id" +2) "5" +3) "message" +4) "redis" ``` From bbf37e1df45ee7f589522177dc5f3eabdfe5dbc5 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 16 Apr 2020 16:05:08 +0800 Subject: [PATCH 266/523] oracle mysql side doc --- docs/plugin/mysqlSide.md | 14 +++++++++++++- docs/plugin/oracleSide.md | 13 ++++++++++++- docs/plugin/{rdbSideParams.md => sideParams.md} | 11 ++++------- 3 files changed, 29 insertions(+), 9 deletions(-) rename docs/plugin/{rdbSideParams.md => sideParams.md} (76%) diff --git a/docs/plugin/mysqlSide.md b/docs/plugin/mysqlSide.md index f28f09518..834183ed2 100644 --- a/docs/plugin/mysqlSide.md +++ b/docs/plugin/mysqlSide.md @@ -29,7 +29,19 @@ ## 3.表结构定义 - [关系型数据库维表参数](docs/plugin/rdbSideParams.md) + [维表参数信息](docs/plugin/sideParams.md) + + mysql独有的参数配置： + + +|参数名称|含义|是否必填|默认值| +|----|---|---|----| +| type | 维表类型， mysql |是|| +| url | 连接数据库 jdbcUrl |是|| +| userName | 连接用户名 |是|| +| password | 连接密码|是|| + + ## 4.样例 diff --git a/docs/plugin/oracleSide.md b/docs/plugin/oracleSide.md index ee7e03331..b681536f8 100644 --- a/docs/plugin/oracleSide.md +++ b/docs/plugin/oracleSide.md @@ -31,7 +31,18 @@ 10g 11g ## 3. 表结构定义 - [关系型数据库维表参数](docs/plugin/rdbSideParams.md) + [维表参数](docs/plugin/sideParams.md) + + + oracle独有的参数配置： + + |参数名称|含义|是否必填|默认值| + |----|---|---|----| + | type | 维表类型， oracle |是|| + | url | 连接数据库 jdbcUrl |是|| + | userName | 连接用户名 |是|| + | password | 连接密码|是|| + | schema| 表空间|否|| ## 4.样例 diff --git a/docs/plugin/rdbSideParams.md b/docs/plugin/sideParams.md similarity index 76% rename from docs/plugin/rdbSideParams.md rename to docs/plugin/sideParams.md index 34d02978e..6059dcab8 100644 --- a/docs/plugin/rdbSideParams.md +++ b/docs/plugin/sideParams.md @@ -1,17 +1,14 @@ -## 关系型数据库维表参数 +## 维表参数 -适用于`MYSQL`,`ORACLE`,`SQLSERVER`,`POSTGRESQL`,`DB2`,`POLARDB`,`CLICKHOUSE`,`IMPALA`维表插件 ### 维表参数 +维表需要的基本属性，每个插件还需要提供连接所需的基本信息。 + |参数名称|含义|是否必填|默认值| |----|---|---|----| | type | 维表类型， 例如:mysql |是|| -| url | 连接数据库 jdbcUrl |是|| -| userName | 连接用户名 |是|| -| password | 连接密码|是|| | tableName| 表名称|是|| -| schema| 表空间|否|| | cache | 维表缓存策略(NONE/LRU/ALL)|否|LRU| | partitionedJoin | 是否在維表join之前先根据设定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| | parallelism | 处理后的数据流并行度|否|| @@ -37,6 +34,6 @@ | cacheMode | 异步请求处理有序还是无序，可选：ordered，unordered |ordered| | asyncCapacity | 异步线程容量 |100| | asyncTimeout | 异步处理超时时间 |10000，单位毫秒| -| asyncPoolSize | 异步查询DB最大线程池，上限20 |min(20,Runtime.getRuntime().availableProcessors() * 2)| +| asyncPoolSize | 异步查询DB最大线程池，上限20。适用于MYSQL,ORACLE,SQLSERVER,POSTGRESQL,DB2,POLARDB,CLICKHOUSE,IMPALA维表插件|min(20,Runtime.getRuntime().availableProcessors() * 2)| From dd7987e420d5f4d52008491acd8edfb3c63ab4e5 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 16 Apr 2020 16:21:14 +0800 Subject: [PATCH 267/523] =?UTF-8?q?redisSide=20=E6=B7=BB=E5=8A=A0=E7=A4=BA?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/redisSide.md | 99 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 89 insertions(+), 10 deletions(-) diff --git a/docs/plugin/redisSide.md b/docs/plugin/redisSide.md index 6aa335fea..3f4483e65 100644 --- a/docs/plugin/redisSide.md +++ b/docs/plugin/redisSide.md @@ -1,5 +1,5 @@ -## 1.格式： +## 1.格式 ``` CREATE TABLE tableName( colName cloType, @@ -15,7 +15,9 @@ redisType = '1', cache ='LRU', cacheSize ='10000', - cacheTTLMs ='60000' + cacheTTLMs ='60000', + parallelism ='1', + partitionedJoin='false' ); ``` ## 2.支持版本 @@ -34,7 +36,7 @@ ## 4.参数 -参数详细说明请看[参数详细说明]() +参数详细说明请看[参数详细说明](./sideParams.md) |参数名称|含义|是否必填|默认值| |----|---|---|----| @@ -48,14 +50,9 @@ | cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| -------------- -> 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - * ALL: 缓存全量表数据 ## 5.样例 +### LRU维表示例 ``` CREATE TABLE MyRedis( id varchar, @@ -70,10 +67,92 @@ CREATE TABLE MyRedis( redisType = '1', tableName = 'sideTable', cache = 'LRU', - cacheTTLMs='10000' + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1', + partitionedJoin='false' +); +``` +### ALL维表示例 +``` +CREATE TABLE MyRedis( + id varchar, + message varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME +)WITH( + type='redis', + url='172.16.10.79:6379', + password='abc123', + database='0', + redisType = '1', + tableName = 'sideTable', + cache = 'ALL', + cacheTTLMs ='60000', + parallelism ='1', + partitionedJoin='false' +); +``` +### redis异步维表关联完整案例 +``` +CREATE TABLE MyTable( + id varchar, + name varchar, + address varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_in2', + groupId = 'flink_sql', + timezone = 'Asia/Shanghai', + topicIsPattern = 'false', + parallelism = '1' ); +CREATE TABLE MyRedis( + id varchar, + message varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME +)WITH( + type='redis', + url='172.16.10.79:6379', + password='abc123', + database='0', + redisType = '1', + tableName = 'sideTable', + cache = 'LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1', + partitionedJoin='false' +); + +CREATE TABLE MyResult( + id varchar, + name varchar, + address varchar, + message varchar +)WITH( + type = 'console' +); + +insert into MyResult + select + t1.id AS id, + t1.name AS name, + t1.address AS address, + t2.message AS message + from + ( + select id, name, address + from MyTable + ) t1 join MyRedis t2 + on t1.id = t2.id; ``` + ## 6.redis的存储命名规则 redis使用散列类型 hash 数据结构，key=tableName_primaryKey1_primaryKey2,value={column1=value1, column2=value2} From 38e34dc546ee421530915d5c2b387304ac6b8fdf Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 17 Apr 2020 11:06:19 +0800 Subject: [PATCH 268/523] =?UTF-8?q?update=20hbase=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E9=83=A8=E5=88=86=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/hbaseSide.md | 140 ++++++++++++++++++++++++++++++++------- 1 file changed, 117 insertions(+), 23 deletions(-) diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index 0c4e545f9..3d08084d0 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -1,5 +1,5 @@ -## 1.格式： +## 1.格式 ``` CREATE TABLE tableName( columnFamily:columnName type as alias, @@ -31,7 +31,9 @@ | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| | PRIMARY KEY(keyInfo) | 维表主键定义;hbase 维表rowkey的构造方式;可选择的构造包括 md5(alias + alias), '常量',也包括上述方式的自由组合 | -## 3.参数 +## 4.参数 + +参数详细说明请看[参数详细说明](./sideParams.md) |参数名称|含义|是否必填|默认值| |----|---|---|----| @@ -43,31 +45,123 @@ | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| -------------- -> 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) -## 4.样例 +## 5.样例 +### LRU维表示例 +``` +CREATE TABLE sideTable ( + wtz:message varchar as message, + wtz:info varchar as info , + PRIMARY KEY (rowkey), + PERIOD FOR SYSTEM_TIME +) WITH ( + type = 'hbase', + zookeeperQuorum = '192.168.80.105:2181,192.168.80.106:2181,192.168.80.107:2181', + zookeeperParent = '/hbase', + tableName = 'testFlinkStreamSql', + parallelism = '1', + cache = 'LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1', + partitionedJoin='false' +); +``` + +### ALL维表示例 +``` +CREATE TABLE sideTable ( + wtz:message varchar as message, + wtz:info varchar as info , + PRIMARY KEY (rowkey), + PERIOD FOR SYSTEM_TIME +) WITH ( + type = 'hbase', + zookeeperQuorum = '192.168.80.105:2181,192.168.80.106:2181,192.168.80.107:2181', + zookeeperParent = '/hbase', + tableName = 'testFlinkStreamSql', + parallelism = '1', + cache = 'ALL', + cacheTTLMs ='60000', + parallelism ='1', + partitionedJoin='false' +); ``` -CREATE TABLE sideTable( - cf:name varchar as name, - cf:info int as info, - PRIMARY KEY(md5(name) + 'test'), - PERIOD FOR SYSTEM_TIME - )WITH( - type ='hbase', - zookeeperQuorum ='rdos1:2181', - zookeeperParent ='/hbase', - tableName ='workerinfo', - cache ='LRU', - cacheSize ='10000', - cacheTTLMs ='60000', - parallelism ='1', - partitionedJoin='true' + +### hbase异步维表关联完整案例 +``` +CREATE TABLE MyTable( + id varchar, + name varchar, + address varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_in', + groupId = 'flink_sql', + timezone = 'Asia/Shanghai', + topicIsPattern = 'false', + parallelism = '1' +); + +CREATE TABLE MyResult( + id varchar, + name varchar, + address varchar, + message varchar, + info varchar +)WITH( + type = 'console' ); + + CREATE TABLE sideTable ( + wtz:message varchar as message, + wtz:info varchar as info , + PRIMARY KEY (rowkey), + PERIOD FOR SYSTEM_TIME +) WITH ( + type = 'hbase', + zookeeperQuorum = '192.168.80.105:2181,192.168.80.106:2181,192.168.80.107:2181', + zookeeperParent = '/hbase', + tableName = 'testFlinkStreamSql', + parallelism = '1', + cache = 'LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1', + partitionedJoin='false' +); +insert +into + MyResult + select + a.name, + a.id, + a.address, + b.message, + b.info + from + MyTable a + left join + sideTable b + on a.id=b.rowkey; ``` +## 6.hbase中数据存储形式 + +在hbase中，数据是以列簇的形式存储，其中rowKey作为主键，按字典排序。 +在样例中，wtz为列族名，message, info为列名，数据在hbase中的存储情况为： +``` +hbase(main):002:0> scan 'testFlinkStreamSql' +ROW COLUMN+CELL + 0 column=wtz:info, timestamp=1587089266719, value=hadoop + 0 column=wtz:message, timestamp=1587089245780, value=hbase + 1 column=wtz:info, timestamp=1587088818432, value=flink + 1 column=wtz:message, timestamp=1587088796633, value=dtstack + 2 column=wtz:info, timestamp=1587088858564, value=sql + 2 column=wtz:message, timestamp=1587088840507, value=stream +``` From 774da5c35d3abc07abf90aa9494d44531013b98b Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 17 Apr 2020 11:07:57 +0800 Subject: [PATCH 269/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9hbase=E7=BB=B4?= =?UTF-8?q?=E8=A1=A8=E6=8E=92=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/hbaseSide.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index 3d08084d0..2ab50f914 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -107,13 +107,13 @@ CREATE TABLE MyTable( ); CREATE TABLE MyResult( - id varchar, - name varchar, - address varchar, - message varchar, - info varchar + id varchar, + name varchar, + address varchar, + message varchar, + info varchar )WITH( - type = 'console' + type = 'console' ); CREATE TABLE sideTable ( From 4bbe75178dc71115df4236078078b9d7526199f8 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 17 Apr 2020 11:36:25 +0800 Subject: [PATCH 270/523] =?UTF-8?q?update=20hbase,=20redis=E7=BB=B4?= =?UTF-8?q?=E8=A1=A8=E6=8E=92=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/hbaseSide.md | 33 +++++++++-------- docs/plugin/redisSide.md | 77 ++++++++++++++++++++-------------------- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index 2ab50f914..cc5fa319c 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -88,6 +88,24 @@ CREATE TABLE sideTable ( ); ``` +## 6.hbase完整样例 + +### 数据说明 + +在hbase中，数据是以列簇的形式存储，其中rowKey作为主键，按字典排序。 + +在样例中，wtz为列族名，message, info为列名，数据在hbase中的存储情况为： +``` +hbase(main):002:0> scan 'testFlinkStreamSql' +ROW COLUMN+CELL + 0 column=wtz:info, timestamp=1587089266719, value=hadoop + 0 column=wtz:message, timestamp=1587089245780, value=hbase + 1 column=wtz:info, timestamp=1587088818432, value=flink + 1 column=wtz:message, timestamp=1587088796633, value=dtstack + 2 column=wtz:info, timestamp=1587088858564, value=sql + 2 column=wtz:message, timestamp=1587088840507, value=stream +``` + ### hbase异步维表关联完整案例 ``` CREATE TABLE MyTable( @@ -150,18 +168,3 @@ into on a.id=b.rowkey; ``` -## 6.hbase中数据存储形式 - -在hbase中，数据是以列簇的形式存储，其中rowKey作为主键，按字典排序。 - -在样例中，wtz为列族名，message, info为列名，数据在hbase中的存储情况为： -``` -hbase(main):002:0> scan 'testFlinkStreamSql' -ROW COLUMN+CELL - 0 column=wtz:info, timestamp=1587089266719, value=hadoop - 0 column=wtz:message, timestamp=1587089245780, value=hbase - 1 column=wtz:info, timestamp=1587088818432, value=flink - 1 column=wtz:message, timestamp=1587088796633, value=dtstack - 2 column=wtz:info, timestamp=1587088858564, value=sql - 2 column=wtz:message, timestamp=1587088840507, value=stream -``` diff --git a/docs/plugin/redisSide.md b/docs/plugin/redisSide.md index 3f4483e65..c63c542cd 100644 --- a/docs/plugin/redisSide.md +++ b/docs/plugin/redisSide.md @@ -93,6 +93,25 @@ CREATE TABLE MyRedis( partitionedJoin='false' ); ``` + +## 6.redis完整样例 + +### redis数据说明 + +redis使用散列类型 hash 数据结构，key=tableName_primaryKey1_primaryKey2,value={column1=value1, column2=value2} +如果以班级class表为例，id和name作为联合主键，那么redis的结构为 + +在样例中，tableName为sideTable，主键为id，column为id，message，所以对应的redis数据插入语句为 + +数据在redis中对应的数据存储情况为： +``` +192.168.80.105:6379> hgetall sideTable_5 +1) "id" +2) "5" +3) "message" +4) "redis" +``` + ### redis异步维表关联完整案例 ``` CREATE TABLE MyTable( @@ -112,12 +131,12 @@ CREATE TABLE MyTable( ); CREATE TABLE MyRedis( - id varchar, - message varchar, - PRIMARY KEY(id), - PERIOD FOR SYSTEM_TIME + id varchar, + message varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME )WITH( - type='redis', + type='redis', url='172.16.10.79:6379', password='abc123', database='0', @@ -131,42 +150,24 @@ CREATE TABLE MyRedis( ); CREATE TABLE MyResult( - id varchar, - name varchar, - address varchar, - message varchar + id varchar, + name varchar, + address varchar, + message varchar )WITH( - type = 'console' + type = 'console' ); insert into MyResult - select - t1.id AS id, - t1.name AS name, - t1.address AS address, - t2.message AS message - from - ( - select id, name, address - from MyTable - ) t1 join MyRedis t2 - on t1.id = t2.id; -``` - -## 6.redis的存储命名规则 - -redis使用散列类型 hash 数据结构，key=tableName_primaryKey1_primaryKey2,value={column1=value1, column2=value2} -如果以班级class表为例，id和name作为联合主键，那么redis的结构为 - -在样例中，tableName为sideTable，主键为id，column为id，message，所以对应的redis数据插入语句为 - -数据在redis中对应的数据存储情况为： + select + t1.id AS id, + t1.name AS name, + t1.address AS address, + t2.message AS message + from( + select id, name, address + from MyTable + ) t1 join MyRedis t2 + on t1.id = t2.id; ``` -192.168.80.105:6379> hgetall sideTable_5 -1) "id" -2) "5" -3) "message" -4) "redis" -``` - From f919e35185846ee4330882d0f97e3375f2055503 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 9 Apr 2020 11:27:10 +0800 Subject: [PATCH 271/523] =?UTF-8?q?add=20isHighAvailability=20=E5=88=A4?= =?UTF-8?q?=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/launcher/ClusterClientFactory.java | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 14cd847b5..eb06b39ba 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -24,15 +24,12 @@ import org.apache.commons.io.Charsets; import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.client.program.MiniClusterClient; -import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.client.program.rest.RestClusterClient; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.runtime.akka.AkkaUtils; -import org.apache.flink.runtime.minicluster.MiniCluster; -import org.apache.flink.runtime.minicluster.MiniClusterConfiguration; import org.apache.flink.runtime.util.LeaderConnectionInfo; import org.apache.flink.yarn.AbstractYarnClusterDescriptor; import org.apache.flink.yarn.YarnClusterDescriptor; @@ -57,6 +54,18 @@ */ public class ClusterClientFactory { + private static final Logger LOG = LoggerFactory.getLogger(ClusterClientFactory.class); + + private static final String HA_CLUSTER_ID = "high-availability.cluster-id"; + + private static final String HIGH_AVAILABILITY = "high-availability"; + + private static final String NODE = "NONE"; + + private static final String ZOOKEEPER = "zookeeper"; + + private static final String HADOOP_CONF = "fs.hdfs.hadoopconf"; + public static ClusterClient createClusterClient(Options launcherOptions) throws Exception { String mode = launcherOptions.getMode(); if (mode.equals(ClusterMode.standalone.name())) { @@ -70,10 +79,12 @@ public static ClusterClient createClusterClient(Options launcherOptions) throws public static ClusterClient createStandaloneClient(Options launcherOptions) throws Exception { String flinkConfDir = launcherOptions.getFlinkconf(); Configuration config = GlobalConfiguration.loadConfiguration(flinkConfDir); - MiniClusterConfiguration.Builder configBuilder = new MiniClusterConfiguration.Builder(); - configBuilder.setConfiguration(config); - MiniCluster miniCluster = new MiniCluster(configBuilder.build()); - MiniClusterClient clusterClient = new MiniClusterClient(config, miniCluster); + + LOG.info("------------config params-------------------------"); + config.toMap().forEach((key, value) -> LOG.info("{}: {}", key, value)); + LOG.info("-------------------------------------------"); + + RestClusterClient clusterClient = new RestClusterClient<>(config, "clusterClient"); LeaderConnectionInfo connectionInfo = clusterClient.getClusterConnectionInfo(); InetSocketAddress address = AkkaUtils.getInetSocketAddressFromAkkaURL(connectionInfo.getAddress()); config.setString(JobManagerOptions.ADDRESS, address.getAddress().getHostName()); @@ -89,6 +100,8 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { if (StringUtils.isNotBlank(yarnConfDir)) { try { + boolean isHighAvailability; + config.setString(ConfigConstants.PATH_HADOOP_CONFIG, yarnConfDir); FileSystem.initialize(config); @@ -101,6 +114,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { String yarnSessionConf = launcherOptions.getYarnSessionConf(); yarnSessionConf = URLDecoder.decode(yarnSessionConf, Charsets.UTF_8.toString()); Properties yarnSessionConfProperties = PluginUtil.jsonStrToObject(yarnSessionConf, Properties.class); + Object yid = yarnSessionConfProperties.get("yid"); if (null != yid) { @@ -109,12 +123,22 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { applicationId = getYarnClusterApplicationId(yarnClient); } - System.out.println("applicationId=" + applicationId.toString()); + LOG.info("current applicationId = {}", applicationId.toString()); if (StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); } + isHighAvailability = config.getString(HIGH_AVAILABILITY, NODE).equals(ZOOKEEPER); + + if (isHighAvailability && config.getString(HA_CLUSTER_ID, null) == null) { + config.setString(HA_CLUSTER_ID, applicationId.toString()); + } + + LOG.info("------------config params-------------------------"); + config.toMap().forEach((key, value) -> LOG.info("{}: {}", key, value)); + LOG.info("-------------------------------------------"); + AbstractYarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(config, yarnConf, flinkConfDir, yarnClient, false); ClusterClient clusterClient = clusterDescriptor.retrieve(applicationId); clusterClient.setDetached(true); @@ -158,7 +182,7 @@ private static ApplicationId getYarnClusterApplicationId(YarnClient yarnClient) } - if (StringUtils.isEmpty(applicationId.toString())) { + if (applicationId == null || StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); } return applicationId; @@ -166,7 +190,7 @@ private static ApplicationId getYarnClusterApplicationId(YarnClient yarnClient) private static ApplicationId toApplicationId(String appIdStr) { Iterator it = StringHelper._split(appIdStr).iterator(); - if (!(it.next()).equals("application")) { + if (!"application".equals(it.next())) { throw new IllegalArgumentException("Invalid ApplicationId prefix: " + appIdStr + ". The valid ApplicationId should start with prefix " + "application"); } else { try { From e438f544bf0a82de74ad665abca61b95c8fddfcf Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 10 Apr 2020 09:34:55 +0800 Subject: [PATCH 272/523] fix --- .../flink/sql/launcher/LauncherMain.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index c098bc27e..cb1fffe01 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -16,7 +16,7 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.launcher; @@ -66,8 +66,7 @@ public class LauncherMain { private static String getLocalCoreJarPath(String localSqlRootJar) throws Exception { String jarPath = PluginUtil.getCoreJarFileName(localSqlRootJar, CORE_JAR); - String corePath = localSqlRootJar + SP + jarPath; - return corePath; + return localSqlRootJar + SP + jarPath; } public static void main(String[] args) throws Exception { @@ -104,6 +103,7 @@ public static void main(String[] args) throws Exception { String flinkConfDir = launcherOptions.getFlinkconf(); Configuration config = StringUtils.isEmpty(flinkConfDir) ? new Configuration() : GlobalConfiguration.loadConfiguration(flinkConfDir); JobGraph jobGraph = PackagedProgramUtils.createJobGraph(program, config, 1); + PerJobSubmitter.submit(launcherOptions, jobGraph, config); } else { ClusterClient clusterClient = ClusterClientFactory.createClusterClient(launcherOptions); @@ -115,20 +115,20 @@ public static void main(String[] args) throws Exception { private static String[] parseJson(String[] args) { BufferedReader reader = null; - String lastStr = ""; - try{ + StringBuilder lastStr = new StringBuilder(); + try { FileInputStream fileInputStream = new FileInputStream(args[0]); - InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, "UTF-8"); + InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, Charsets.UTF_8); reader = new BufferedReader(inputStreamReader); - String tempString = null; - while((tempString = reader.readLine()) != null){ - lastStr += tempString; + String tempString; + while ((tempString = reader.readLine()) != null) { + lastStr.append(tempString); } reader.close(); - }catch(IOException e){ + } catch (IOException e) { e.printStackTrace(); - }finally{ - if(reader != null){ + } finally { + if (reader != null) { try { reader.close(); } catch (IOException e) { @@ -136,14 +136,14 @@ private static String[] parseJson(String[] args) { } } } - Map map = JSON.parseObject(lastStr, new TypeReference>(){} ); + Map map = JSON.parseObject(lastStr.toString(), new TypeReference>() { + }); List list = new LinkedList<>(); for (Map.Entry entry : map.entrySet()) { list.add("-" + entry.getKey()); list.add(entry.getValue().toString()); } - String[] array = list.toArray(new String[list.size()]); - return array; + return list.toArray(new String[0]); } } From 83429b26199feac0f30a5ec15c2e230c670a4b78 Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 20 Apr 2020 19:50:17 +0800 Subject: [PATCH 273/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E5=90=88=E5=B9=B6=E8=BF=87=E7=A8=8B=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/launcher/ClusterClientFactory.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index eb06b39ba..cfbab1023 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -25,6 +25,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.rest.RestClusterClient; +import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; @@ -39,6 +40,8 @@ import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.StringHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; import java.net.URLDecoder; From 38b9b388b6b517631080d1f17135a175528bfe54 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 21 Apr 2020 11:21:46 +0800 Subject: [PATCH 274/523] =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E8=A1=A8=E5=90=8D?= =?UTF-8?q?=E6=9B=BF=E6=8D=A2=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/util/FieldReplaceUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java index f92643d5f..69bdc97b2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java @@ -181,12 +181,12 @@ public static SqlIdentifier createNewIdentify(SqlIdentifier sqlIdentifier, return sqlIdentifier; } + sqlIdentifier = sqlIdentifier.setName(0, newTbName); + String mappingFieldName = mappingField.get(fieldName); if(mappingFieldName == null){ return sqlIdentifier; } - - sqlIdentifier = sqlIdentifier.setName(0, newTbName); sqlIdentifier = sqlIdentifier.setName(1, mappingFieldName); return sqlIdentifier; } From ca8d08de02594d5abfe8efa6cc687219e0a274ad Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 21 Apr 2020 12:53:24 +0800 Subject: [PATCH 275/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcase=20when=20?= =?UTF-8?q?=E4=B8=AD=E6=B2=A1=E6=9C=89=E8=A7=A3=E6=9E=90else=E5=AD=97?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/JoinNodeDealer.java | 2 +- .../java/com/dtstack/flink/sql/util/FieldReplaceUtil.java | 4 ++++ .../main/java/com/dtstack/flink/sql/util/TableUtils.java | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index f072e2591..544e8ebb0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -233,7 +233,6 @@ public Set extractField(SqlNode sqlNode, Set fieldFromJoinCondition = extractSelectFieldFromJoinCondition(joinFieldSet, fromTableNameSet, tableRef); Set extractGroupByField = extractFieldFromGroupByList(parentGroupByList, fromTableNameSet, tableRef); - extractSelectField.addAll(extractCondition); extractSelectField.addAll(fieldFromJoinCondition); extractSelectField.addAll(extractGroupByField); @@ -575,6 +574,7 @@ private void extractSelectField(SqlNode selectNode, || selectNode.getKind() == TIMESTAMP_ADD || selectNode.getKind() == TIMESTAMP_DIFF || selectNode.getKind() == LIKE + || selectNode.getKind() == COALESCE ){ SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java index 69bdc97b2..775e0a985 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java @@ -160,6 +160,10 @@ private static SqlNode replaceNodeInfo(SqlNode groupNode, SqlNode replaceNode = replaceSelectFieldName(sqlNode, oldTbName, newTbName, mappingField); sqlCase.getThenOperands().set(i,replaceNode); } + if(sqlCase.getElseOperand() != null){ + SqlNode replaceNode = replaceSelectFieldName(sqlCase.getElseOperand(), oldTbName, newTbName, mappingField); + sqlCase.setOperand(3, replaceNode); + } return sqlCase; } else { return groupNode; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 579beaf63..4a4973901 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -151,6 +151,7 @@ public static void extractSelectFieldToFieldInfo(SqlNode fieldNode, String fromN || fieldNode.getKind() == TIMESTAMP_ADD || fieldNode.getKind() == TIMESTAMP_DIFF || fieldNode.getKind() == LIKE + || fieldNode.getKind() == COALESCE ) { SqlBasicCall sqlBasicCall = (SqlBasicCall) fieldNode; for (int i = 0; i < sqlBasicCall.getOperands().length; i++) { @@ -363,6 +364,7 @@ public static void replaceSelectFieldTable(SqlNode selectNode, || selectNode.getKind() == TIMESTAMP_ADD || selectNode.getKind() == TIMESTAMP_DIFF || selectNode.getKind() == LIKE + || selectNode.getKind() == COALESCE ){ SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; @@ -548,6 +550,7 @@ private static void replaceConditionNode(SqlNode selectNode, String oldTbName, S || selectNode.getKind() == TIMESTAMP_ADD || selectNode.getKind() == TIMESTAMP_DIFF || selectNode.getKind() == LIKE + || selectNode.getKind() == COALESCE ){ SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; @@ -573,6 +576,7 @@ private static void replaceConditionNode(SqlNode selectNode, String oldTbName, S sqlCase.getWhenOperands().getList().forEach(sqlNode -> replaceConditionNode(sqlNode, oldTbName, newTbName, fieldReplaceRef)); sqlCase.getThenOperands().getList().forEach(sqlNode -> replaceConditionNode(sqlNode, oldTbName, newTbName, fieldReplaceRef)); + replaceConditionNode(sqlCase.getElseOperand(), oldTbName, newTbName, fieldReplaceRef); } else { throw new RuntimeException(String.format("not support node kind of %s to replace name now.", selectNode.getKind())); } @@ -620,6 +624,7 @@ public static void getConditionRefTable(SqlNode selectNode, Set fieldInf || selectNode.getKind() == TIMESTAMP_ADD || selectNode.getKind() == TIMESTAMP_DIFF || selectNode.getKind() == LIKE + || selectNode.getKind() == COALESCE ) { SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; @@ -645,6 +650,7 @@ public static void getConditionRefTable(SqlNode selectNode, Set fieldInf sqlCase.getWhenOperands().getList().forEach(sqlNode -> getConditionRefTable(sqlNode, fieldInfos)); sqlCase.getThenOperands().getList().forEach(sqlNode -> getConditionRefTable(sqlNode, fieldInfos)); + getConditionRefTable(sqlCase.getElseOperand(), fieldInfos); } else { throw new RuntimeException(String.format("not support node kind of %s to replace name now.", selectNode.getKind())); } From b231974e816aafa5d8d685921e1404b7a1211249 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 21 Apr 2020 16:06:35 +0800 Subject: [PATCH 276/523] fix --- docs/plugin/mysqlSink.md | 73 +++++++++++++++++++++++++++++++++------- docs/plugin/redisSink.md | 5 +++ 2 files changed, 66 insertions(+), 12 deletions(-) diff --git a/docs/plugin/mysqlSink.md b/docs/plugin/mysqlSink.md index 300086e21..8dabea047 100644 --- a/docs/plugin/mysqlSink.md +++ b/docs/plugin/mysqlSink.md @@ -22,7 +22,7 @@ CREATE TABLE tableName( |参数名称|含义| |----|---| -| tableName| mysql表名称| +| tableName| 结果表名称| | colName | 列名称| | colType | 列类型 [colType支持的类型](docs/colType.md)| @@ -33,21 +33,70 @@ CREATE TABLE tableName( |type |表名 输出表类型[mysq|hbase|elasticsearch]|是|| |url | 连接mysql数据库 jdbcUrl |是|| |userName | mysql连接用户名 |是|| -| password | mysql连接密码|是|| -| tableName | mysql表名称|是|| -| parallelism | 并行度设置|否|1| - -## 5.样例： +|password | mysql连接密码|是|| +|tableName | mysql表名称|是|| +|parallelism | 并行度设置|否|1| +|batchNum | flush的大小|否|100| +|batchWaitInterval | flush的时间间隔，单位ms|否|1000| +|allReplace| true:新值替换旧值|否|false| +|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|| + +## 5.完整样例： ``` +CREATE TABLE MyTable( + id int, + channel varchar, + pv varchar, + xctime varchar, + name varchar + )WITH( + type ='kafka10', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='es_test', + timezone='Asia/Shanghai', + topicIsPattern ='false', + parallelism ='1' + ); + CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR + pv VARCHAR, + channel VARCHAR )WITH( type ='mysql', - url ='jdbc:mysql://172.16.8.104:3306/test?charset=utf8', + url ='jdbc:mysql://172.16.10.134:3306/test', userName ='dtstack', password ='abc123', - tableName ='pv2', + tableName ='myresult', parallelism ='1' - ) - ``` \ No newline at end of file + ); + + +insert +into + MyResult + select + channel, + pv + from + MyTable + ``` + +## 6.结果表数据示例： +``` +mysql> desc myresult; ++---------+--------------+------+-----+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++---------+--------------+------+-----+---------+-------+ +| channel | varchar(255) | YES | | NULL | | +| pv | varchar(11) | YES | | NULL | | ++---------+--------------+------+-----+---------+-------+ +2 rows in set (0.00 sec) + +mysql> select * from myresult limit 1; ++---------+------+ +| channel | pv | ++---------+------+ +| aa | mq6 | +``` \ No newline at end of file diff --git a/docs/plugin/redisSink.md b/docs/plugin/redisSink.md index 6a754e5c6..b43cb7377 100644 --- a/docs/plugin/redisSink.md +++ b/docs/plugin/redisSink.md @@ -41,6 +41,11 @@ redis5.0 | database | reids 的数据库地址|否|| | tableName | redis 的表名称|是|| | parallelism | 并行度设置|否|1| +|timeout| 连接超时时间|否|10000| +|maxTotal|最大连接数|否|8| +|maxIdle|最大空闲连接数|否|8| +|minIdle|最小空闲连接数|否||0| +|masterName| 哨兵模式下的masterName|否|| ## 5.样例： From 6f8c9aa6c560c47b42bfd51684481e1054abfc20 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 21 Apr 2020 19:53:54 +0800 Subject: [PATCH 277/523] =?UTF-8?q?fix=20=E5=88=AB=E5=90=8Djoin=E5=BC=82?= =?UTF-8?q?=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/side/SideInfo.java | 2 +- .../flink/sql/side/rdb/async/RdbAsyncSideInfo.java | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java index 029c86e25..4b53f77f7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java @@ -93,7 +93,7 @@ public void parseSelectFields(JoinInfo joinInfo){ if(fieldInfo.getTable().equalsIgnoreCase(sideTableName)){ String sideFieldName = sideTableInfo.getPhysicalFields().getOrDefault(fieldInfo.getFieldName(), fieldInfo.getFieldName()); fields.add(sideFieldName); - sideSelectFieldsType.put(sideTableFieldIndex, getTargetFieldType(sideFieldName)); + sideSelectFieldsType.put(sideTableFieldIndex, getTargetFieldType(fieldInfo.getFieldName())); sideFieldIndex.put(i, sideTableFieldIndex); sideFieldNameIndex.put(i, sideFieldName); sideTableFieldIndex++; diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java index 9e8c13080..f87b27ae4 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java @@ -18,8 +18,6 @@ package com.dtstack.flink.sql.side.rdb.async; -import org.apache.flink.api.java.typeutils.RowTypeInfo; - import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; @@ -33,9 +31,11 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; @@ -86,6 +86,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { SqlIdentifier left = (SqlIdentifier) ((SqlBasicCall) sqlNode).getOperands()[0]; SqlIdentifier right = (SqlIdentifier) ((SqlBasicCall) sqlNode).getOperands()[1]; + Map physicalFields = sideTableInfo.getPhysicalFields(); String leftTableName = left.getComponent(0).getSimple(); String leftField = left.getComponent(1).getSimple(); @@ -94,7 +95,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { String rightField = right.getComponent(1).getSimple(); if (leftTableName.equalsIgnoreCase(sideTableName)) { - equalFieldList.add(leftField); + equalFieldList.add(physicalFields.get(leftField)); int equalFieldIndex = -1; for (int i = 0; i < rowTypeInfo.getFieldNames().length; i++) { String fieldName = rowTypeInfo.getFieldNames()[i]; @@ -110,7 +111,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { } else if (rightTableName.equalsIgnoreCase(sideTableName)) { - equalFieldList.add(rightField); + equalFieldList.add(physicalFields.get(rightField)); int equalFieldIndex = -1; for (int i = 0; i < rowTypeInfo.getFieldNames().length; i++) { String fieldName = rowTypeInfo.getFieldNames()[i]; From ccbc0a1207e4554021d214e9e40caeb156898e92 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 22 Apr 2020 16:04:18 +0800 Subject: [PATCH 278/523] =?UTF-8?q?update=20hbaseSide=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/hbaseSide.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index cc5fa319c..25af638d6 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -29,7 +29,7 @@ | columnFamily:columnName | hbase中的列族名称和列名称 | | alias | hbase 中的列对应到flink中注册的列名称 | | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| -| PRIMARY KEY(keyInfo) | 维表主键定义;hbase 维表rowkey的构造方式;可选择的构造包括 md5(alias + alias), '常量',也包括上述方式的自由组合 | +| PRIMARY KEY(keyInfo) | 维表主键定义;hbase 维表rowkey的构造方式:可选择的构造包括 md5(alias + alias), '常量',也包括上述方式的自由组合 | ## 4.参数 @@ -105,7 +105,8 @@ ROW COLUMN+CELL 2 column=wtz:info, timestamp=1587088858564, value=sql 2 column=wtz:message, timestamp=1587088840507, value=stream ``` - +在hbase中，rowKey是一个二进制码流，可以为任意字符串，flinkStreamSql读取rowKey并通过rowKey唯一确定数据，对rowKey没有任何限制,对rowKey可选择的构造包括 md5(alias + alias), '常量',也可以它们的自由组合。 +在本次案例中，rowKey为了简单，设置成了"0,1,2"这样的数值型字符，若有必要，也可以设计得更为复杂。 ### hbase异步维表关联完整案例 ``` CREATE TABLE MyTable( From 2b0af472c7d46e792c43db9070a7691fa23d1512 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 23 Apr 2020 11:02:30 +0800 Subject: [PATCH 279/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=94=99=E4=B9=B1=E9=97=AE=E9=A2=98http://re?= =?UTF-8?q?dmine.prod.dtstack.cn/issues/24996?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 971285e52..52a27fcc3 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -88,7 +88,7 @@ protected void init(SideInfo sideInfo) { @Override public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { - CRow copyCrow = new CRow(input.row(), input.change()); + CRow copyCrow = new CRow(Row.copy(input.row()), input.change()); JsonArray inputParams = new JsonArray(); for (Integer conValIndex : sideInfo.getEqualValIndex()) { Object equalObj = copyCrow.row().getField(conValIndex); From 73ed644f80fef243dc97c5bbde86edc114e2b57a Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 23 Apr 2020 16:09:26 +0800 Subject: [PATCH 280/523] =?UTF-8?q?=E4=B9=B1=E5=BA=8F=EF=BC=8C=E5=88=AB?= =?UTF-8?q?=E5=90=8D=E5=92=8Chase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/dtstack/flink/sql/side/SideInfo.java | 5 ++--- .../com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java | 2 +- .../flink/sql/side/hbase/table/HbaseSideParser.java | 1 + .../dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java | 7 +++---- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java index d8a22e022..4b53f77f7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java @@ -132,13 +132,12 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName){ String leftTableName = left.getComponent(0).getSimple(); String leftField = left.getComponent(1).getSimple(); - Map physicalFields = sideTableInfo.getPhysicalFields(); String rightTableName = right.getComponent(0).getSimple(); String rightField = right.getComponent(1).getSimple(); if(leftTableName.equalsIgnoreCase(sideTableName)){ - equalFieldList.add(physicalFields.get(leftField)); + equalFieldList.add(leftField); int equalFieldIndex = -1; for(int i=0; i resultFuture) throws Exception { - CRow inputCopy = new CRow(input.row(), input.change()); + CRow inputCopy = new CRow(Row.copy(input.row()), input.change()); Map refData = Maps.newHashMap(); for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 7b627da4d..3e9ab7f45 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -100,6 +100,7 @@ private void dealField(Matcher matcher, TableInfo tableInfo){ sideTableInfo.addFieldClass(fieldClass); sideTableInfo.addFieldType(fieldType); sideTableInfo.putAliasNameRef(aliasStr, fieldName); + sideTableInfo.addPhysicalMappings(aliasStr, fieldName); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java index f3830793e..7aa83b3c8 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java @@ -87,7 +87,6 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { SqlIdentifier left = (SqlIdentifier) ((SqlBasicCall) sqlNode).getOperands()[0]; SqlIdentifier right = (SqlIdentifier) ((SqlBasicCall) sqlNode).getOperands()[1]; - Map physicalFields = sideTableInfo.getPhysicalFields(); String leftTableName = left.getComponent(0).getSimple(); String leftField = left.getComponent(1).getSimple(); @@ -96,7 +95,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { String rightField = right.getComponent(1).getSimple(); if (leftTableName.equalsIgnoreCase(sideTableName)) { - equalFieldList.add(physicalFields.get(leftField)); + equalFieldList.add(leftField); int equalFieldIndex = -1; for (int i = 0; i < rowTypeInfo.getFieldNames().length; i++) { String fieldName = rowTypeInfo.getFieldNames()[i]; @@ -112,7 +111,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { } else if (rightTableName.equalsIgnoreCase(sideTableName)) { - equalFieldList.add(physicalFields.get(rightField)); + equalFieldList.add(rightField); int equalFieldIndex = -1; for (int i = 0; i < rowTypeInfo.getFieldNames().length; i++) { String fieldName = rowTypeInfo.getFieldNames()[i]; @@ -144,7 +143,7 @@ public String getSelectFromStatement(String tableName, List selectFields .collect(Collectors.joining(", ")); String whereClause = conditionFields.stream() - .map(f -> quoteIdentifier(f) + sqlJoinCompareOperate.get(conditionFields.indexOf(f)) + wrapperPlaceholder(f)) + .map(f -> quoteIdentifier(sideTableInfo.getPhysicalFields().getOrDefault(f, f)) + sqlJoinCompareOperate.get(conditionFields.indexOf(f)) + wrapperPlaceholder(f)) .collect(Collectors.joining(" AND ")); String predicateClause = predicateInfoes.stream() From 93844cff640be6676305ceeeddef3ea3ea655f79 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 23 Apr 2020 16:29:57 +0800 Subject: [PATCH 281/523] pgsink sqlserversink doc --- docs/plugin/mysqlSink.md | 2 +- docs/plugin/postgresqlSink.md | 101 ++++++++++++++++++++++++++++------ docs/plugin/sqlserverSink.md | 96 ++++++++++++++++++++++++++------ 3 files changed, 164 insertions(+), 35 deletions(-) diff --git a/docs/plugin/mysqlSink.md b/docs/plugin/mysqlSink.md index 8dabea047..254f11f2a 100644 --- a/docs/plugin/mysqlSink.md +++ b/docs/plugin/mysqlSink.md @@ -36,7 +36,7 @@ CREATE TABLE tableName( |password | mysql连接密码|是|| |tableName | mysql表名称|是|| |parallelism | 并行度设置|否|1| -|batchNum | flush的大小|否|100| +|batchSize | flush的大小|否|100| |batchWaitInterval | flush的时间间隔，单位ms|否|1000| |allReplace| true:新值替换旧值|否|false| |updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|| diff --git a/docs/plugin/postgresqlSink.md b/docs/plugin/postgresqlSink.md index 6d7d87834..c04ff0ec0 100644 --- a/docs/plugin/postgresqlSink.md +++ b/docs/plugin/postgresqlSink.md @@ -3,7 +3,8 @@ CREATE TABLE tableName( colName colType, ... - colNameX colType + colNameX colType， + primary key (colName) )WITH( type ='postgresql', url ='jdbcUrl', @@ -25,29 +26,95 @@ CREATE TABLE tableName( | tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| | colType | 列类型 [colType支持的类型](docs/colType.md)| +| primary key | updateMode为UPSERT时，需要指定的主键信息| + ## 4.参数： |参数名称|含义|是否必填|默认值| |----|----|----|----| -| type |表明 输出表类型[postgresql]|是|| -| url | 连接postgresql数据库 jdbcUrl |是|| -| userName | postgresql连接用户名 |是|| -| password | postgresql连接密码|是|| -| tableName | postgresqll表名称|是|| -| parallelism | 并行度设置|否|1| +|type |结果表插件类型，必须为postgresql|是|| +|url | 连接postgresql数据库 jdbcUrl |是|| +|userName |postgresql连接用户名 |是|| +|password | postgresql连接密码|是|| +|tableName | postgresql表名称|是|| +|schema | postgresql表空间|否|| +|parallelism | 并行度设置|否|1| +|batchSize | flush的大小|否|100| +|batchWaitInterval | flush的时间间隔，单位ms|否|1000| +|allReplace| true:新值替换旧值|否|false| +|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|| ## 5.样例： + +回溯流删除 + ``` + +CREATE TABLE source1 ( + id int, + name VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest03', + timezone='Asia/Shanghai', + topicIsPattern ='false' + ); + + + +CREATE TABLE source2( + id int, + address VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest04', + timezone='Asia/Shanghai', + topicIsPattern ='false' +); + + CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR - )WITH( - type ='postgresql', - url ='jdbc:postgresql://localhost:9001/test?sslmode=disable', - userName ='dtstack', - password ='abc123', - tableName ='pv2', - parallelism ='1' - ) + id int, + name VARCHAR, + address VARCHAR, + primary key (id) +)WITH( + type='postgresql', + url='jdbc:postgresql://172.16.8.193:5432/DTstack', + userName='root', + password='123456', + tableName='userInfo', + schema = 'aaa', + updateMode = 'upsert', + batchSize = '1' +); + +insert into MyResult +select + s1.id, + s1.name, + s2.address +from + source1 s1 +left join + source2 s2 +on + s1.id = s2.id + + + ``` + + +数据结果： + +向Topic mqTest03 发送数据 {"name":"maqi","id":1001} 插入 (1001,"maqi",null) + +向Topic mqTest04 发送数据 {"address":"hz","id":1001} 删除 (1001,"maqi",null) 插入 (1001,"maqi","hz") diff --git a/docs/plugin/sqlserverSink.md b/docs/plugin/sqlserverSink.md index 8ecb14308..0ab9431e9 100644 --- a/docs/plugin/sqlserverSink.md +++ b/docs/plugin/sqlserverSink.md @@ -3,7 +3,8 @@ CREATE TABLE tableName( colName colType, ... - colNameX colType + colNameX colType, + primary key (colName) )WITH( type ='sqlserver', url ='jdbcUrl', @@ -25,29 +26,90 @@ CREATE TABLE tableName( | tableName| sqlserver表名称| | colName | 列名称| | colType | 列类型 [colType支持的类型](docs/colType.md)| +| primary key | updateMode为UPSERT时，需要指定的主键信息| ## 4.参数： |参数名称|含义|是否必填|默认值| |----|----|----|----| -|type |表名 输出表类型[mysq|hbase|elasticsearch]|是|| +|type |结果表插件类型，必须为sqlserver|是|| |url | 连接sqlserver数据库 jdbcUrl |是|| -|userName | sqlserver连接用户名 |是|| -| password | sqlserver连接密码|是|| -| tableName | sqlserver表名称|是|| -| parallelism | 并行度设置|否|1| +|userName |sqlserver连接用户名 |是|| +|password | sqlserver连接密码|是|| +|tableName | sqlserver表名称|是|| +|schema | sqlserver表空间|否|| +|parallelism | 并行度设置|否|1| +|batchSize | flush的大小|否|100| +|batchWaitInterval | flush的时间间隔，单位ms|否|1000| +|allReplace| true:新值替换旧值|否|false| +|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|| + ## 5.样例： + +回溯流删除 ``` + +CREATE TABLE source1 ( + id int, + name VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest03', + timezone='Asia/Shanghai', + topicIsPattern ='false' + ); + + +CREATE TABLE source2( + id int, + address VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest04', + timezone='Asia/Shanghai', + topicIsPattern ='false' +); + + CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR - )WITH( - type ='sqlserver', - url ='jdbc:jtds:sqlserver://172.16.8.104:1433;DatabaseName=mytest', - userName ='dtstack', - password ='abc123', - tableName ='pv2', - parallelism ='1' - ) - ``` \ No newline at end of file + id int, + name VARCHAR, + address VARCHAR, + primary key (id) +)WITH( + type='sqlserver', + url='jdbc:jtds:sqlserver://172.16.8.149:1433;DatabaseName=DTstack', + userName='sa', + password='Dtstack2018', + tableName='user', + schema = 'aaa', + updateMode = 'upsert', + batchSize = '1' +); + +insert into MyResult +select + s1.id, + s1.name, + s2.address +from + source1 s1 +left join + source2 s2 +on + s1.id = s2.id + + ``` + +数据结果： + +向Topic mqTest03 发送数据 {"name":"maqi","id":1001} 插入 (1001,"maqi",null) + +向Topic mqTest04 发送数据 {"address":"hz","id":1001} 删除 (1001,"maqi",null) 插入 (1001,"maqi","hz") From 83571b419945d3540c4ca3d7104318673cbb8d5a Mon Sep 17 00:00:00 2001 From: gkd Date: Thu, 23 Apr 2020 20:23:52 +0800 Subject: [PATCH 282/523] postgresql-side clickhouse-sise --- docs/plugin/clickhouseSide.md | 170 ++++++++++++++++++++++++---------- docs/plugin/postgresqlSide.md | 162 +++++++++++++++++++++++--------- 2 files changed, 239 insertions(+), 93 deletions(-) diff --git a/docs/plugin/clickhouseSide.md b/docs/plugin/clickhouseSide.md index 57e4eef6b..df050ca8b 100644 --- a/docs/plugin/clickhouseSide.md +++ b/docs/plugin/clickhouseSide.md @@ -1,6 +1,9 @@ - ## 1.格式： -``` + +通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 + +```sql CREATE TABLE tableName( colName cloType, ... @@ -21,65 +24,132 @@ ``` # 2.支持版本 + 19.14.x、19.15.x、19.16.x - + ## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | clickhouse表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| - -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型 clickhouse |是|| - | url | 连接clickhouse数据库 jdbcUrl |是|| - | userName | clickhouse连接用户名 |是|| - | password | clickhouse连接密码|是|| - | tableName | clickhouse表名称|是|| - | tableName | clickhouse 的表名称|是|| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - * cacheMode: (unordered|ordered)异步加载是有序还是无序,默认有序。 - * asyncCapacity:异步请求容量，默认1000 - * asyncTimeout：异步请求超时时间，默认10000毫秒 - -## 5.样例 + + [维表参数信息](docs/plugin/sideParams.md) + +clinkhose独有的参数信息 + +| 参数名称 | 含义 | 是否必填 | 默认值 | +| -------- | --------------------- | -------- | ------ | +| type | 维表类型， clinkhouse | 是 | | +| url | 连接数据库 jdbcUrl | 是 | | +| userName | 连接用户名 | 是 | | +| password | 连接密码 | 是 | | + +## 4.样例 + +------ + +## ALL全量维表定义 + +```sql +// 定义全量维表 +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME + )WITH( + type ='clickhouse', + url ='jdbc:clickhouse://172.16.8.104:3306/zftest', + userName ='dtstack', + password ='abc123', + tableName ='all_test_clinkhousesql', + cache ='ALL', + cacheTTLMs ='60000', + parallelism ='1' + ); ``` -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), + +### LRU异步维表定义 + +``` +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , PERIOD FOR SYSTEM_TIME )WITH( - type='clickhouse', - url='jdbc:clickhouse://172.16.8.104:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='sidetest', + type ='clickhousesql', + url ='jdbc:clickhousesql://172.16.8.104:3306/zftest', + userName ='dtstack', + password ='abc123', + tableName ='lru_test_clickhousesql', + partitionedJoin ='false', cache ='LRU', cacheSize ='10000', cacheTTLMs ='60000', - cacheMode='unordered', - asyncCapacity='1000', - asyncTimeout='10000' + asyncPoolSize ='3', + parallelism ='1' + ); +``` + +### ClickHouseSQL异步维表关联 + +```sql +CREATE TABLE MyTable( + id int, + name varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='cannan_zftest01', + timezone='Asia/Shanghai', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + +CREATE TABLE MyResult( + id INT, + name VARCHAR + )WITH( + type ='clickhousesql', + url ='jdbc:clickhousesql://172.16.8.104:3306/zftest', + userName ='dtstack', + password ='abc123', + tableName ='test_clickhouse_zf', + updateMode ='append', parallelism ='1', - partitionedJoin='false' + batchSize ='100', + batchWaitInterval ='1000' ); +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='clickhousesql', + url ='jdbc:clickhousesql://172.16.8.104:3306/zftest', + userName ='dtstack', + password ='abc123', + tableName ='test_clickhouse_10', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + asyncPoolSize ='3', + parallelism ='1' + ); +insert +into + MyResult + select + m.id, + s.name + from + MyTable m + join + sideTable s + on m.id=s.id; ``` - diff --git a/docs/plugin/postgresqlSide.md b/docs/plugin/postgresqlSide.md index 6928042f6..b50a2de52 100644 --- a/docs/plugin/postgresqlSide.md +++ b/docs/plugin/postgresqlSide.md @@ -1,5 +1,8 @@ - ## 1.格式： + +通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 + ``` CREATE TABLE tableName( colName cloType, @@ -21,59 +24,132 @@ ``` # 2.支持版本 + postgresql-8.2+ - + ## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | 注册到flink的表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| - -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型[postgresql] |是|| - | url | 连接postgresql数据库 jdbcUrl |是|| - | userName | postgresql连接用户名 |是|| - | password | postgresql连接密码|是|| - | tableName | postgresql表名称|是|| - | cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - - -## 5.样例 + + [维表参数信息](docs/plugin/sideParams.md) + +postgresql独有的参数配置 + +| 参数名称 | 含义 | 是否必填 | 默认值 | +| -------- | ------------------ | -------- | ------ | +| type | 维表类型， mysql | 是 | | +| url | 连接数据库 jdbcUrl | 是 | | +| userName | 连接用户名 | 是 | | +| password | 连接密码 | 是 | | + +## 4.样例 + +### ALL全量维表定义 + +```sql + // 定义全量维表 +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='postgresql', + url ='jdbc:postgresql://172.16.10.194:5432/zftest', + userName ='dtstack', + password ='abc123', + tableName ='all_test_postgresql', + cache ='ALL', + cacheTTLMs ='60000', + parallelism ='1' + ); ``` -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), + +### LRU异步维表定义 + +```sql +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , PERIOD FOR SYSTEM_TIME )WITH( - type='postgresql', - url='jdbc:postgresql://localhost:9001/test?sslmode=disable', - userName='dtstack', - password='abc123', - tableName='sidetest', + type ='postgresql', + url ='jdbc:postgresql://172.16.10.194:5432/zftest', + userName ='dtstack', + password ='abc123', + tableName ='lru_test_postgresql', + partitionedJoin ='false', cache ='LRU', cacheSize ='10000', cacheTTLMs ='60000', + asyncPoolSize ='3', + parallelism ='1' + ); + +``` + +### PostgreSQL异步维表关联 + +```sql +CREATE TABLE MyTable( + id int, + name varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='cannan_zftest01', + timezone='Asia/Shanghai', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + +CREATE TABLE MyResult( + id INT, + name VARCHAR + )WITH( + type ='postgresql', + url ='jdbc:postgresql://172.16.10.194:5432/zftest', + userName ='dtstack', + password ='abc123', + tableName ='test_postgresql_zf', + updateMode ='append', parallelism ='1', - partitionedJoin='false' + batchSize ='100', + batchWaitInterval ='1000' ); +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='postgresql', + url ='jdbc:postgresql://172.16.10.194:5432/zftest', + userName ='dtstack', + password ='abc123', + tableName ='test_postgresql_10', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + asyncPoolSize ='3', + parallelism ='1' + ); -``` +insert +into + MyResult + select + m.id, + s.name + from + MyTable m + join + sideTable s + on m.id=s.id; +``` From e4bedb2dac3b135d331900a64ed0ef7759322e46 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 24 Apr 2020 15:41:47 +0800 Subject: [PATCH 283/523] ck impala sink doc --- docs/plugin/clickhouseSink.md | 93 ++++++++++++++++++++++++++++++----- docs/plugin/impalaColType.md | 14 ++++++ docs/plugin/impalaSink.md | 4 +- 3 files changed, 98 insertions(+), 13 deletions(-) create mode 100644 docs/plugin/impalaColType.md diff --git a/docs/plugin/clickhouseSink.md b/docs/plugin/clickhouseSink.md index 780ac77b0..6b4a724be 100644 --- a/docs/plugin/clickhouseSink.md +++ b/docs/plugin/clickhouseSink.md @@ -24,7 +24,7 @@ CREATE TABLE tableName( |----|---| | tableName| clickhouse表名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | clickhouse基本数据类型，不包括Array，Tuple,Nested等| ## 4.参数： @@ -36,18 +36,87 @@ CREATE TABLE tableName( | password | clickhouse 连接密码|是|| | tableName | clickhouse 表名称|是|| | parallelism | 并行度设置|否|1| +|updateMode| 只支持APPEND模式，过滤掉回撤数据||| + ## 5.样例： + + ``` + +CREATE TABLE source1 ( + id int, + name VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest03', + timezone='Asia/Shanghai', + topicIsPattern ='false' + ); + + + +CREATE TABLE source2( + id int, + address VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest04', + timezone='Asia/Shanghai', + topicIsPattern ='false' +); + + CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR - )WITH( - type ='clickhouse', - url ='jdbc:clickhouse://172.16.8.104:3306/test?charset=utf8', - userName ='dtstack', - password ='abc123', - tableName ='pv2', - parallelism ='1' - ) - ``` \ No newline at end of file + id int, + name VARCHAR, + address VARCHAR +)WITH( + type='clickhouse', + url='jdbc:clickhouse://172.16.10.168:8123/tudou?charset=utf8', + userName='dtstack', + password='abc123', + tableName='MyResult', + updateMode = 'append' +); + +insert into MyResult +select + s1.id, + s1.name, + s2.address +from + source1 s1 +left join + source2 s2 +on + s1.id = s2.id + + + ``` + + +**Clickhouse建表语句** + + + ```aidl +CREATE TABLE tudou.MyResult (`id` Int64, `name` String, `address` String) + ENGINE = MergeTree PARTITION BY address ORDER BY id SETTINGS index_granularity = 8192 +``` + + + + +数据结果： + +向Topic mqTest03 发送数据 {"name":"maqi","id":1001} 插入 (1001,"maqi",null) + +向Topic mqTest04 发送数据 {"address":"hz","id":1001} 插入 (1001,"maqi","hz") + + \ No newline at end of file diff --git a/docs/plugin/impalaColType.md b/docs/plugin/impalaColType.md new file mode 100644 index 000000000..4b84b3aa3 --- /dev/null +++ b/docs/plugin/impalaColType.md @@ -0,0 +1,14 @@ +| 支持的类型 | java对应类型 | +| ------ | ----- | +| boolean | Boolean | +| char | Character | +| double | Double| +| float | Float| +| tinyint | Byte | +| smallint | Short| +| int | Integer | +| bigint | Long | +| decimal |BigDecimal| +| string | String | +| varchar | String | +| timestamp | Timestamp | \ No newline at end of file diff --git a/docs/plugin/impalaSink.md b/docs/plugin/impalaSink.md index af3baa282..398ec6ef2 100644 --- a/docs/plugin/impalaSink.md +++ b/docs/plugin/impalaSink.md @@ -24,7 +24,7 @@ CREATE TABLE tableName( |----|---| | tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](docs/plugin/impalaColType.md)| ## 4.参数： @@ -45,6 +45,8 @@ CREATE TABLE tableName( | enablePartition | 是否支持分区 |否|false| | partitionFields | 分区字段名|否，enablePartition='true'时为必填|| | parallelism | 并行度设置|否|1| +| parallelism | 并行度设置|否|1| +|updateMode| 只支持APPEND模式，过滤掉回撤数据||| ## 5.样例： From f1e1d28f6caff24952e8dc747104bd5c20cce147 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 24 Apr 2020 16:02:51 +0800 Subject: [PATCH 284/523] =?UTF-8?q?where=20=E6=9D=A1=E4=BB=B6=E5=9C=A8?= =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E4=B8=AD=E4=B8=8B=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/SidePredicatesParser.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java index 4d6112aa8..7b89b25fd 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java @@ -101,6 +101,8 @@ private void parseSql(SqlNode sqlNode, Map sideTableMap, parseSql(unionLeft, sideTableMap, tabMapping); parseSql(unionRight, sideTableMap, tabMapping); break; + default: + break; } } @@ -128,10 +130,10 @@ private void extractPredicateInfo(SqlNode whereNode, List predica // 跳过函数 if ((((SqlBasicCall) whereNode).getOperands()[0] instanceof SqlIdentifier) - && (((SqlBasicCall) whereNode).getOperands()[1].getKind() != SqlKind.OTHER_FUNCTION)) { + && (((SqlBasicCall) whereNode).getOperands()[1].getKind() == SqlKind.LITERAL)) { fillPredicateInfoToList((SqlBasicCall) whereNode, predicatesInfoList, operatorName, operatorKind, 0, 1); } else if ((((SqlBasicCall) whereNode).getOperands()[1] instanceof SqlIdentifier) - && (((SqlBasicCall) whereNode).getOperands()[0].getKind() != SqlKind.OTHER_FUNCTION)) { + && (((SqlBasicCall) whereNode).getOperands()[0].getKind() == LITERAL)) { fillPredicateInfoToList((SqlBasicCall) whereNode, predicatesInfoList, operatorName, operatorKind, 1, 0); } } From 9a2a8915b67ca3a0b0c992420fa82a42bd6a8c41 Mon Sep 17 00:00:00 2001 From: maqi Date: Sun, 26 Apr 2020 16:52:05 +0800 Subject: [PATCH 285/523] db2 upsert --- .../dtstack/flink/sql/sink/db/DbDialect.java | 80 +++++++++++++++++++ .../com/dtstack/flink/sql/sink/db/DbSink.java | 1 + kafka09/kafka09-sink/pom.xml | 1 + kafka09/kafka09-source/pom.xml | 1 + 4 files changed, 83 insertions(+) diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbDialect.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbDialect.java index 9c7985d8f..49a3613bc 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbDialect.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbDialect.java @@ -19,8 +19,13 @@ package com.dtstack.flink.sql.sink.db; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import com.dtstack.flink.sql.util.DtStringUtil; +import org.apache.commons.lang3.StringUtils; +import java.util.Arrays; +import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; /** * Date: 2020/1/19 @@ -43,4 +48,79 @@ public String quoteIdentifier(String identifier) { return identifier; } + @Override + public Optional getUpsertStatement(String schema, String tableName, String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + tableName = DtStringUtil.getTableFullPath(schema, tableName); + StringBuilder sb = new StringBuilder(); + sb.append("MERGE INTO " + tableName + " T1 USING " + + "(" + buildValuesStatement(fieldNames) + ") T2 (" + + buildFiledNameStatement(fieldNames) + + ") ON (" + + buildConnectionConditions(uniqueKeyFields) + ") "); + + String updateSql = buildUpdateConnection(fieldNames, uniqueKeyFields, allReplace); + + if (StringUtils.isNotEmpty(updateSql)) { + sb.append(" WHEN MATCHED THEN UPDATE SET "); + sb.append(updateSql); + } + + sb.append(" WHEN NOT MATCHED THEN " + + "INSERT (" + Arrays.stream(fieldNames).map(this::quoteIdentifier).collect(Collectors.joining(",")) + ") VALUES (" + + Arrays.stream(fieldNames).map(col -> "T2." + quoteIdentifier(col)).collect(Collectors.joining(",")) + ")"); + return Optional.of(sb.toString()); + } + + /** + * build T1."A"=T2."A" or T1."A"=nvl(T2."A",T1."A") + * @param fieldNames + * @param uniqueKeyFields + * @param allReplace + * @return + */ + private String buildUpdateConnection(String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + List uniqueKeyList = Arrays.asList(uniqueKeyFields); + return Arrays.stream(fieldNames) + .filter(col -> !uniqueKeyList.contains(col)) + .map(col -> buildConnectString(allReplace, col)) + .collect(Collectors.joining(",")); + } + + private String buildConnectString(boolean allReplace, String col) { + return allReplace ? quoteIdentifier("T1") + "." + quoteIdentifier(col) + " = " + quoteIdentifier("T2") + "." + quoteIdentifier(col) : + quoteIdentifier("T1") + "." + quoteIdentifier(col) + " =NVL(" + quoteIdentifier("T2") + "." + quoteIdentifier(col) + "," + + quoteIdentifier("T1") + "." + quoteIdentifier(col) + ")"; + } + + + private String buildConnectionConditions(String[] uniqueKeyFields) { + return Arrays.stream(uniqueKeyFields).map(col -> "T1." + quoteIdentifier(col) + "=T2." + quoteIdentifier(col)).collect(Collectors.joining(",")); + } + + /** + * build sql part e.g: VALUES('1001','zs','sss') + * + * @param column destination column + * @return + */ + public String buildValuesStatement(String[] column) { + StringBuilder sb = new StringBuilder("VALUES("); + String collect = Arrays.stream(column) + .map(col -> " ? ") + .collect(Collectors.joining(", ")); + + return sb.append(collect).append(")").toString(); + } + + /** + * build sql part e.g: id, name, address + * @param column + * @return + */ + public String buildFiledNameStatement(String[] column) { + return Arrays.stream(column) + .collect(Collectors.joining(", ")); + } + + } diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java index 9942d4438..573a0896a 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java @@ -19,6 +19,7 @@ public JDBCUpsertOutputFormat getOutputFormat() { .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) + .setSchema(schema) .setTableName(tableName) .build(); diff --git a/kafka09/kafka09-sink/pom.xml b/kafka09/kafka09-sink/pom.xml index 8f52d7bc8..e01c63cc3 100644 --- a/kafka09/kafka09-sink/pom.xml +++ b/kafka09/kafka09-sink/pom.xml @@ -37,6 +37,7 @@ shade + false org.slf4j diff --git a/kafka09/kafka09-source/pom.xml b/kafka09/kafka09-source/pom.xml index deef866c9..dde2352a1 100644 --- a/kafka09/kafka09-source/pom.xml +++ b/kafka09/kafka09-source/pom.xml @@ -35,6 +35,7 @@ shade + false org.slf4j From 2e4721b6770c731015214633a181559a93bb655d Mon Sep 17 00:00:00 2001 From: maqi Date: Sun, 26 Apr 2020 17:35:58 +0800 Subject: [PATCH 286/523] db2 doc --- docs/plugin/db2Side.md | 162 ++++++++++++++++++++++++++-------- docs/plugin/db2Sink.md | 120 +++++++++++++++++++++---- docs/plugin/mysqlSink.md | 5 +- docs/plugin/postgresqlSink.md | 2 +- 4 files changed, 233 insertions(+), 56 deletions(-) diff --git a/docs/plugin/db2Side.md b/docs/plugin/db2Side.md index 8381d6206..af0c29c02 100644 --- a/docs/plugin/db2Side.md +++ b/docs/plugin/db2Side.md @@ -1,5 +1,7 @@ ## 1.格式： + 通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 ``` CREATE TABLE tableName( colName cloType, @@ -25,55 +27,145 @@ ## 3.表结构定义 + + [维表参数信息](docs/plugin/sideParams.md) + db2独有的参数配置： + |参数名称|含义| |----|---| - | tableName | db2表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| +| type | 维表类型， db2 |是|| +| url | 连接数据库 jdbcUrl |是|| +| userName | 连接用户名 |是|| +| password | 连接密码|是|| +| schema | 表所属scheam|否|| -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型 db2 |是|| - | url | 连接mysql数据库 jdbcUrl |是|| - | userName | db2连接用户名 |是|| - | password | db2连接密码|是|| - | tableName | db2表名称|是|| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - - -## 5.样例 + + +## 4.样例 + +### ALL全量维表定义 ``` -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), + // 定义全量维表 +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , PERIOD FOR SYSTEM_TIME )WITH( type='db2', - url='jdbc:db2://172.16.8.104:50000/test?charset=utf8', - userName='dtstack', + url='jdbc:db2://172.16.10.251:50000/mqTest', + userName='DB2INST1', password='abc123', - tableName='sidetest', + tableName='USER_INFO2', + schema = 'DTSTACK' + cache ='ALL', + cacheTTLMs ='60000', + parallelism ='2' + ); + +``` +### LRU异步维表定义 + +``` +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type='db2', + url='jdbc:db2://172.16.10.251:50000/mqTest', + userName='DB2INST1', + password='abc123', + tableName='USER_INFO2', + schema = 'DTSTACK' + partitionedJoin ='false', cache ='LRU', cacheSize ='10000', cacheTTLMs ='60000', - parallelism ='1', - partitionedJoin='false' + asyncPoolSize ='3', + parallelism ='2' + ); + +``` + + +### DB2异步维表关联输出到Console +``` + +CREATE TABLE source1 ( + id int, + name VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest03', + timezone='Asia/Shanghai', + topicIsPattern ='false' ); + +CREATE TABLE source2( + id int, + address VARCHAR, + PERIOD FOR SYSTEM_TIME +)WITH( + type='db2', + url='jdbc:db2://172.16.10.251:50000/mqTest', + userName='DB2INST1', + password='abc123', + tableName='USER_INFO2', + schema = 'DTSTACK', + batchSize = '1' +); + + +CREATE TABLE MyResult( + id int, + name VARCHAR, + address VARCHAR, + primary key (id) +)WITH( + type='console' +); + +insert into MyResult +select + s1.id, + s1.name, + s2.address +from + source1 s1 +left join + source2 s2 +on + s1.id = s2.id + ``` +DB2维表字段信息 + +```aidl +-- DTSTACK.USER_INFO2 definition + +CREATE TABLE "DTSTACK "."USER_INFO2" ( + "ID" INTEGER , + "NAME" VARCHAR(50 OCTETS) , + "ADDRESS" VARCHAR(50 OCTETS) ) + IN "USERSPACE1" + ORGANIZE BY ROW + ; + +GRANT CONTROL ON TABLE "DTSTACK "."USER_INFO2" TO USER "DB2INST1" +; +``` + +维表数据：(1001,maqi,hz) + +源表数据：{"name":"maqi","id":1001} + +输出结果： (1001,maqi,hz) \ No newline at end of file diff --git a/docs/plugin/db2Sink.md b/docs/plugin/db2Sink.md index 0fcbdd3d3..9544e44f1 100644 --- a/docs/plugin/db2Sink.md +++ b/docs/plugin/db2Sink.md @@ -3,7 +3,8 @@ CREATE TABLE tableName( colName colType, ... - colNameX colType + colNameX colType, + [primary key (colName)] )WITH( type ='db2', url ='jdbcUrl', @@ -25,29 +26,112 @@ CREATE TABLE tableName( | tableName| db2表名称| | colName | 列名称| | colType | 列类型 [colType支持的类型](docs/colType.md)| +| primary key | updateMode为UPSERT时，需要指定的主键信息，不需要和数据库一致| + ## 4.参数： |参数名称|含义|是否必填|默认值| |----|----|----|----| -| type |表名 输出表类型[mysq|hbase|elasticsearch]|是|| -| url | 连接db2数据库 jdbcUrl |是|| -| userName | db2连接用户名 |是|| -| password | db2连接密码|是|| -| tableName | db2表名称|是|| -| parallelism | 并行度设置|否|1| +|type |结果表插件类型，必须为db2|是|| +|url | 连接db2数据库 jdbcUrl |是|| +|userName |db2连接用户名 |是|| +|password | db2连接密码|是|| +|tableName | db2表名称|是|| +|schema | db2表空间|否|| +|parallelism | 并行度设置|否|1| +|batchSize | flush的大小|否|100| +|batchWaitInterval | flush的时间间隔，单位ms|否|1000| +|allReplace| true:新值替换旧值|否|false| +|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|结果表设置主键则为UPSERT| ## 5.样例： + +回溯流删除 + ``` + +CREATE TABLE source1 ( + id int, + name VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest03', + timezone='Asia/Shanghai', + topicIsPattern ='false' + ); + + + +CREATE TABLE source2( + id int, + address VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest04', + timezone='Asia/Shanghai', + topicIsPattern ='false' +); + + CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR - )WITH( - type ='db2', - url ='jdbc:db2://172.16.8.104:50000/test?charset=utf8', - userName ='dtstack', - password ='abc123', - tableName ='pv2', - parallelism ='1' - ) - ``` \ No newline at end of file + id int, + name VARCHAR, + address VARCHAR, + primary key (id) +)WITH( + type='db2', + url='jdbc:db2://172.16.10.251:50000/mqTest', + userName='DB2INST1', + password='abc123', + tableName='USER_INFO2', + schema = 'DTSTACK', + updateMode = 'upsert', // 设置 primary key则默认为upsert + batchSize = '1' +); + +insert into MyResult +select + s1.id, + s1.name, + s2.address +from + source1 s1 +left join + source2 s2 +on + s1.id = s2.id + + + + ``` + + + DB2结果表建表语句： + + ```aidl +CREATE TABLE "DTSTACK "."USER_INFO2" ( + "ID" INTEGER , + "NAME" VARCHAR(50 OCTETS) , + "ADDRESS" VARCHAR(50 OCTETS) ) + IN "USERSPACE1" + ORGANIZE BY ROW + ; + +GRANT CONTROL ON TABLE "DTSTACK "."USER_INFO2" TO USER "DB2INST1" +; +``` + + + +数据结果： + +向Topic mqTest03 发送数据 {"name":"maqi","id":1001} 插入 (1001,"maqi",null) + +向Topic mqTest04 发送数据 {"address":"hz","id":1001} 删除 (1001,"maqi",null) 插入 (1001,"maqi","hz") \ No newline at end of file diff --git a/docs/plugin/mysqlSink.md b/docs/plugin/mysqlSink.md index 254f11f2a..ffc62792c 100644 --- a/docs/plugin/mysqlSink.md +++ b/docs/plugin/mysqlSink.md @@ -3,7 +3,8 @@ CREATE TABLE tableName( colName colType, ... - colNameX colType + colNameX colType, + [primary key (colName)] )WITH( type ='mysql', url ='jdbcUrl', @@ -39,7 +40,7 @@ CREATE TABLE tableName( |batchSize | flush的大小|否|100| |batchWaitInterval | flush的时间间隔，单位ms|否|1000| |allReplace| true:新值替换旧值|否|false| -|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|| +|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|结果表设置主键则为UPSERT| ## 5.完整样例： ``` diff --git a/docs/plugin/postgresqlSink.md b/docs/plugin/postgresqlSink.md index c04ff0ec0..eb055cc04 100644 --- a/docs/plugin/postgresqlSink.md +++ b/docs/plugin/postgresqlSink.md @@ -43,7 +43,7 @@ CREATE TABLE tableName( |batchSize | flush的大小|否|100| |batchWaitInterval | flush的时间间隔，单位ms|否|1000| |allReplace| true:新值替换旧值|否|false| -|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|| +|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|结果表设置主键则为UPSERT| ## 5.样例： From ed3aa19336fbfd1ca68d5e3f9db25cfcc7df83a2 Mon Sep 17 00:00:00 2001 From: maqi Date: Sun, 26 Apr 2020 17:57:54 +0800 Subject: [PATCH 287/523] db2 side fix --- .../com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java | 7 +++++++ .../com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java | 7 ++----- .../com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java | 8 ++++++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java index 282ee440c..b40cfd772 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java @@ -22,6 +22,8 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -38,4 +40,9 @@ public class Db2AllSideInfo extends RdbAllSideInfo { public Db2AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } + + @Override + public String getTableName(RdbSideTableInfo rdbSideTableInfo) { + return DtStringUtil.getTableFullPath(rdbSideTableInfo.getSchema(), rdbSideTableInfo.getTableName()); + } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index cda2ed128..ed9fb025b 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -29,8 +29,6 @@ import io.vertx.ext.jdbc.JDBCClient; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; @@ -43,8 +41,7 @@ */ public class Db2AsyncReqRow extends RdbAsyncReqRow { - - private static final Logger LOG = LoggerFactory.getLogger(Db2AsyncReqRow.class); + private final static String DB2_PREFERRED_TEST_QUERY_SQL = "select 1 from sysibm.dual"; private final static String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; @@ -63,7 +60,7 @@ public void open(Configuration parameters) throws Exception { .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) - .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) + .put("preferred_test_query", DB2_PREFERRED_TEST_QUERY_SQL) .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java index e53dd7f27..dbb8230e7 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java @@ -22,6 +22,8 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -38,4 +40,10 @@ public class Db2AsyncSideInfo extends RdbAsyncSideInfo { public Db2AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } + + @Override + public String getTableName(RdbSideTableInfo rdbSideTableInfo) { + return DtStringUtil.getTableFullPath(rdbSideTableInfo.getSchema(), rdbSideTableInfo.getTableName()); + } + } From 588071175fc04ea8daa9d183de0ebfbd339ad194 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 27 Apr 2020 13:53:53 +0800 Subject: [PATCH 288/523] =?UTF-8?q?=E6=9B=B4=E6=96=B0cassandra-side?= =?UTF-8?q?=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/cassandraSide.md | 139 +++++++++++++++++++++++++++++------ 1 file changed, 115 insertions(+), 24 deletions(-) diff --git a/docs/plugin/cassandraSide.md b/docs/plugin/cassandraSide.md index f5210a900..5b8c955d5 100644 --- a/docs/plugin/cassandraSide.md +++ b/docs/plugin/cassandraSide.md @@ -1,5 +1,8 @@ ## 1.格式： + + 通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 ``` CREATE TABLE tableName( colName cloType, @@ -22,15 +25,15 @@ ``` # 2.支持版本 - cassandra-3.6.x + cassandra-3.x ## 3.表结构定义 |参数名称|含义| |----|---| - | tableName | 注册到flink的表名称(可选填;不填默认和hbase对应的表名称相同)| + | tableName | 注册到flink的表名称(可选填;不填默认和cassandra对应的表名称相同)| | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| + | colType | 列类型| | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| @@ -46,40 +49,128 @@ | database | cassandra表名称|是|| | cache | 维表缓存策略(NONE/LRU)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - | maxRequestsPerConnection | 每个连接最多允许64个并发请求|否|NONE| - | coreConnectionsPerHost | 和Cassandra集群里的每个机器都至少有2个连接|否|NONE| - | maxConnectionsPerHost | 和Cassandra集群里的每个机器都最多有6个连接|否|NONE| - | maxQueueSize | Cassandra队列大小|否|NONE| - | readTimeoutMillis | Cassandra读超时|否|NONE| - | connectTimeoutMillis | Cassandra连接超时|否|NONE| - | poolTimeoutMillis | Cassandra线程池超时|否|NONE| + | maxRequestsPerConnection | 每个连接最多允许64个并发请求|否|1| + | coreConnectionsPerHost | Cassandra集群里的每个机器都最少连接数|否|8| + | maxConnectionsPerHost | Cassandra集群里的每个机器都最多连接数|否|32768| + | maxQueueSize | Cassandra队列大小|否|100000| + | readTimeoutMillis | Cassandra读超时|否|60000| + | connectTimeoutMillis | Cassandra连接超时|否|60000| + | poolTimeoutMillis | Cassandra线程池超时|否|60000| ---------- > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) +- NONE：不做内存缓存。每条流数据触发一次维表查询操作。 +- ALL: 任务启动时，一次性加载所有数据到内存，并进行缓存。适用于维表数据量较小的情况。 +- LRU: 任务执行时，根据维表关联条件使用异步算子加载维表数据，并进行缓存。 -## 5.样例 +## 5.维表定义样例 + +### ALL全量维表定义 ``` -create table sideTable( - CHANNEL varchar, - XCCOUNT int, - PRIMARY KEY(channel), +CREATE TABLE sideTable( + id bigint, + school varchar, + home varchar, + PRIMARY KEY(id), PERIOD FOR SYSTEM_TIME - )WITH( +)WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/tiezhu', + userName='dtstack', + password='abc123', + tableName='stressTest', + cache='ALL', + parallelism='1' +); +``` +### LRU异步维表定义 +``` +CREATE TABLE sideTable( + id bigint, + message varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME +)WITH( type ='cassandra', - address ='172.21.32.1:9042,172.21.32.1:9042', - database ='test', - tableName ='sidetest', + address ='192.168.80.106:9042, 192.168.80.107:9042', + database ='tiezhu', + tableName ='stu', + userName='cassandra', + password='cassandra', cache ='LRU', parallelism ='1', partitionedJoin='false' - ); +); +``` +## 6.完整样例 +``` +CREATE TABLE MyTable( + id bigint, + name varchar, + address varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_in2', + timezone = 'Asia/Shanghai', + topicIsPattern = 'false', + parallelism = '1' +); +CREATE TABLE sideTable( + id bigint, + message varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME +)WITH( + type ='cassandra', + address ='192.168.80.106:9042, 192.168.80.107:9042', + database ='tiezhu', + tableName ='stu', + userName='cassandra', + password='cassandra', + cache ='LRU', + parallelism ='1', + partitionedJoin='false' +); + +CREATE TABLE MyResult( + id bigint, + name varchar, + address varchar, + message varchar + )WITH( + type ='cassandra', + address ='192.168.80.106:9042,192.168.80.107:9042', + userName='cassandra', + password='cassandra', + database ='tiezhu', + tableName ='stu_out', + parallelism ='1' +); +insert +into + MyResult + select + t1.id AS id, + t1.name AS name, + t1.address AS address, + t2.message AS message + from + ( + select + id, + name, + address + from + MyTable + ) t1 + join sideTable t2 + on t1.id = t2.id; ``` From ab67cfb197b762dc9f3740abc4dd28e546bd4875 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 27 Apr 2020 13:56:25 +0800 Subject: [PATCH 289/523] =?UTF-8?q?=E6=9B=B4=E6=96=B0cassandra-side?= =?UTF-8?q?=E6=96=87=E6=A1=A3=E4=B8=AD=E5=8F=82=E6=95=B0=E8=AF=B4=E6=98=8E?= =?UTF-8?q?=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/cassandraSide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugin/cassandraSide.md b/docs/plugin/cassandraSide.md index 5b8c955d5..048caed62 100644 --- a/docs/plugin/cassandraSide.md +++ b/docs/plugin/cassandraSide.md @@ -49,8 +49,8 @@ | database | cassandra表名称|是|| | cache | 维表缓存策略(NONE/LRU)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - | maxRequestsPerConnection | 每个连接最多允许64个并发请求|否|1| - | coreConnectionsPerHost | Cassandra集群里的每个机器都最少连接数|否|8| + | maxRequestsPerConnection | 每个连接允许的并发请求数|否|1| + | coreConnectionsPerHost | 每台主机连接的核心数|否|8| | maxConnectionsPerHost | Cassandra集群里的每个机器都最多连接数|否|32768| | maxQueueSize | Cassandra队列大小|否|100000| | readTimeoutMillis | Cassandra读超时|否|60000| From cdc91af4738687d173ba715a2a6e835e7b4bd20f Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 27 Apr 2020 14:03:25 +0800 Subject: [PATCH 290/523] =?UTF-8?q?=E6=9B=B4=E6=96=B0cassandra-sink?= =?UTF-8?q?=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/cassandraSink.md | 125 +++++++++++++++++++++++++++++------ 1 file changed, 106 insertions(+), 19 deletions(-) diff --git a/docs/plugin/cassandraSink.md b/docs/plugin/cassandraSink.md index 6d6dd2301..8702acb7a 100644 --- a/docs/plugin/cassandraSink.md +++ b/docs/plugin/cassandraSink.md @@ -17,7 +17,7 @@ CREATE TABLE tableName( ``` ## 2.支持版本 - cassandra-3.6.x + cassandra-3.x ## 3.表结构定义 @@ -27,7 +27,7 @@ CREATE TABLE tableName( | colName | 列名称| | colType | 列类型 [colType支持的类型](docs/colType.md)| -## 4.参数： +## 4.参数 |参数名称|含义|是否必填|默认值| |----|----|----|----| @@ -38,26 +38,113 @@ CREATE TABLE tableName( |tableName | cassandra表名称|是|| |database | cassandra表名称|是|| |parallelism | 并行度设置|否|1| -|maxRequestsPerConnection | 每个连接最多允许64个并发请求|否|NONE| -|coreConnectionsPerHost | 和Cassandra集群里的每个机器都至少有2个连接|否|NONE| -|maxConnectionsPerHost | 和Cassandra集群里的每个机器都最多有6个连接|否|NONE| -|maxQueueSize | Cassandra队列大小|否|NONE| -|readTimeoutMillis | Cassandra读超时|否|NONE| -|connectTimeoutMillis | Cassandra连接超时|否|NONE| -|poolTimeoutMillis | Cassandra线程池超时|否|NONE| +| maxRequestsPerConnection | 每个连接允许的并发请求数|否|1| +| coreConnectionsPerHost | 每台主机连接的核心数|否|8| +| maxConnectionsPerHost | Cassandra集群里的每个机器都最多连接数|否|32768| +| maxQueueSize | Cassandra队列大小|否|100000| +| readTimeoutMillis | Cassandra读超时|否|60000| +| connectTimeoutMillis | Cassandra连接超时|否|60000| +| poolTimeoutMillis | Cassandra线程池超时|否|60000| -## 5.样例： +## 5.完整样例： ``` +CREATE TABLE MyTable( + id bigint, + name varchar, + address varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_in2', + timezone = 'Asia/Shanghai', + topicIsPattern = 'false', + parallelism = '1' +); + +CREATE TABLE sideTable( + id bigint, + message varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME +)WITH( + type ='cassandra', + address ='192.168.80.106:9042, 192.168.80.107:9042', + database ='tiezhu', + tableName ='stu', + userName='cassandra', + password='cassandra', + cache ='LRU', + parallelism ='1', + partitionedJoin='false' +); + CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR + id bigint, + name varchar, + address varchar, + message varchar )WITH( type ='cassandra', - address ='172.21.32.1:9042,172.21.32.1:9042', - userName ='dtstack', - password ='abc123', - database ='test', - tableName ='pv', + address ='192.168.80.106:9042,192.168.80.107:9042', + userName='cassandra', + password='cassandra', + database ='tiezhu', + tableName ='stu_out', parallelism ='1' - ) - ``` \ No newline at end of file +); + +insert +into + MyResult + select + t1.id AS id, + t1.name AS name, + t1.address AS address, + t2.message AS message + from + ( + select + id, + name, + address + from + MyTable + ) t1 + join sideTable t2 + on t1.id = t2.id; + ``` +### 6.结果表数据展示 +``` +cqlsh:tiezhu> desc stu_out + +CREATE TABLE tiezhu.stu_out ( + id int PRIMARY KEY, + address text, + message text, + name text +) WITH bloom_filter_fp_chance = 0.01 + AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} + AND comment = '' + AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'} + AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND dclocal_read_repair_chance = 0.1 + AND default_time_to_live = 0 + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair_chance = 0.0 + AND speculative_retry = '99PERCENTILE'; + + +cqlsh:tiezhu> select * from stu_out limit 1; + + id | address | message | name +----+------------+------------------+---------- + 23 | hangzhou23 | flinkStreamSql23 | tiezhu23 + +(1 rows) +``` \ No newline at end of file From ccd5edefa8ba01b640159201dd833cf8ae78fcf2 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 27 Apr 2020 14:39:06 +0800 Subject: [PATCH 291/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-sink=E6=96=87?= =?UTF-8?q?=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/kafkaSink.md | 152 +++++++++++++++++++++++++++++++++++++++ pom.xml | 32 ++++----- 2 files changed, 168 insertions(+), 16 deletions(-) create mode 100644 docs/plugin/kafkaSink.md diff --git a/docs/plugin/kafkaSink.md b/docs/plugin/kafkaSink.md new file mode 100644 index 000000000..8b50c9e3f --- /dev/null +++ b/docs/plugin/kafkaSink.md @@ -0,0 +1,152 @@ +## 1.格式： +``` +CREATE TABLE tableName( + colName colType, + ... + colNameX colType, + [primary key (colName)] + )WITH( + type ='kafka09', + bootstrapServers ='ip:port,ip:port...', + zookeeperQuorum ='ip:port,ip:port/zkparent', + offsetReset ='latest', + topic ='topicName', + groupId='test', + parallelism ='parllNum', + timezone='Asia/Shanghai', + sourcedatatype ='json' #可不设置 + ); +``` + +## 2.支持版本 + kafka09,kafka10,kafka11及以上版本 + +## 3.表结构定义 + +|参数名称|含义| +|----|---| +| tableName| 结果表名称| +| colName | 列名称| +| colType | 列类型 [colType支持的类型](docs/colType.md)| + +## 4.参数： + +|参数名称|含义|是否必填|默认值| +|----|----|----|----| +|type |表名的输出表类型[kafka09|kafka10|kafka11|kafka]|是|| +|groupId | 需要读取的 groupId 名称|否|| +|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| +|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| +|topic | 需要读取的 topic 名称|是|| +|topicIsPattern | topic是否是正则表达式格式(true|false) |否| false +|offsetReset | 读取的topic 的offset初始位置[latest|earliest|指定offset值({"0":12312,"1":12321,"2":12312},{"partition_no":offset_value})]|否|latest| +|parallelism | 并行度设置|否|1| +|sourcedatatype | 数据类型|否|json| +|timezone|时区设置[timezone支持的参数](../timeZone.md)|否|'Asia/Shanghai' +**kafka相关参数可以自定义，使用kafka.开头即可。** +``` +kafka.consumer.id +kafka.socket.timeout.ms +kafka.fetch.message.max.bytes +kafka.num.consumer.fetchers +kafka.auto.commit.enable +kafka.auto.commit.interval.ms +kafka.queued.max.message.chunks +kafka.rebalance.max.retries +kafka.fetch.min.bytes +kafka.fetch.wait.max.ms +kafka.rebalance.backoff.ms +kafka.refresh.leader.backoff.ms +kafka.consumer.timeout.ms +kafka.exclude.internal.topics +kafka.partition.assignment.strategy +kafka.client.id +kafka.zookeeper.session.timeout.ms +kafka.zookeeper.connection.timeout.ms +kafka.zookeeper.sync.time.ms +kafka.offsets.storage +kafka.offsets.channel.backoff.ms +kafka.offsets.channel.socket.timeout.ms +kafka.offsets.commit.max.retries +kafka.dual.commit.enabled +kafka.partition.assignment.strategy +kafka.socket.receive.buffer.bytes +kafka.fetch.min.bytes +``` + +## 5.完整样例： +``` +CREATE TABLE MyTable( + id bigint, + name varchar, + address varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_in2', + groupId = 'flink_sql', + timezone = 'Asia/Shanghai', + topicIsPattern = 'false', + parallelism = '1' +); + +CREATE TABLE sideTable( + id bigint, + school varchar, + home varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME +)WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/tiezhu', + userName='dtstack', + password='abc123', + tableName='stressTest', + cache='ALL', + parallelism='1' +); + +CREATE TABLE MyResult( + id bigint, + name varchar, + address varchar, + home varchar, + school varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_out2', + parallelism = '1' +); + +insert +into + MyResult + select + t1.id AS id, + t1.name AS name, + t1.address AS address, + t2.school AS school, + t2.home AS home + from( + select + id, + name, + address + from + MyTable + ) t1 + join sideTable t2 + on t1.id = t2.id; + ``` + +## 6.结果表数据示例： +``` +[root@node002 bin]# ./kafka-console-consumer.sh --bootstrap-server 172.16.101.224:9092 --topic tiezhu_test_out2 +{"id":122,"name":"tiezhu122","address":"hangzhou122","home":"ganzhou122","school":" ecjtu122"} +{"id":123,"name":"tiezhu123","address":"hangzhou123","home":"ganzhou123","school":" ecjtu123"} +``` \ No newline at end of file diff --git a/pom.xml b/pom.xml index 2794f9004..b5b674379 100644 --- a/pom.xml +++ b/pom.xml @@ -16,24 +16,24 @@ kafka11 kafka mysql - hbase - elasticsearch5 - mongo - redis5 + + + + launcher rdb - sqlserver - oracle - cassandra - kudu - postgresql - serversocket - console - clickhouse - impala - db2 - polardb - elasticsearch6 + + + + + + + + + + + + From eb7502a01b1c5c2a8d9633e851f7008afbb0e743 Mon Sep 17 00:00:00 2001 From: gkd Date: Mon, 27 Apr 2020 16:18:02 +0800 Subject: [PATCH 292/523] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=96=87=E6=A1=A3db2?= =?UTF-8?q?side=20impalaside=20sqlserverside?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/db2Side.md | 141 +++++++++++++++++++++------- docs/plugin/impalaSide.md | 173 +++++++++++++++++++++++++---------- docs/plugin/sqlserverSide.md | 146 ++++++++++++++++++++++------- 3 files changed, 344 insertions(+), 116 deletions(-) diff --git a/docs/plugin/db2Side.md b/docs/plugin/db2Side.md index 8381d6206..88f0bfc0e 100644 --- a/docs/plugin/db2Side.md +++ b/docs/plugin/db2Side.md @@ -1,6 +1,9 @@ - ## 1.格式： -``` + +通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 + +```sql CREATE TABLE tableName( colName cloType, ... @@ -20,41 +23,48 @@ ); ``` -# 2.支持版本 +## 2.支持版本 + db2 9.X - + ## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | db2表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| - + + [维表参数信息](docs/plugin/sideParams.md) + +db2独有的参数 + +| 参数名称 | 含义 | 是否必填 | 默认值 | +| -------- | ------------------ | -------- | ------ | +| type | 维表类型， db2 | 是 | | +| url | 连接数据库 jdbcUrl | 是 | | +| userName | 连接用户名 | 是 | | +| password | 连接密码 | 是 | | + ## 4.参数 - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型 db2 |是|| - | url | 连接mysql数据库 jdbcUrl |是|| - | userName | db2连接用户名 |是|| - | password | db2连接密码|是|| - | tableName | db2表名称|是|| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - - -## 5.样例 +### ALL全量维表定义 + +```sql +// 定义全量维表 +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='db2', + url ='jdbc:db2://172.16.8.104:50000/test?charset=utf8', + userName ='dtstack', + password ='abc123', + tableName ='all_test_db2', + cache ='ALL', + cacheTTLMs ='60000', + parallelism ='1' + ); ``` +### LRU异步维表定义 + +```sql create table sideTable( channel varchar, xccount int, @@ -65,15 +75,78 @@ create table sideTable( url='jdbc:db2://172.16.8.104:50000/test?charset=utf8', userName='dtstack', password='abc123', - tableName='sidetest', + tableName='lru_test_db2', cache ='LRU', cacheSize ='10000', cacheTTLMs ='60000', parallelism ='1', partitionedJoin='false' ); +``` +### Db2异步维表关联 -``` +```sql +CREATE TABLE MyTable( + id int, + name varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='cannan_yctest01', + timezone='Asia/Shanghai', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); +CREATE TABLE MyResult( + id INT, + name VARCHAR + )WITH( + type='db2', + url='jdbc:db2://172.16.8.104:50000/test?charset=utf8', + userName='dtstack', + password='abc123', + tableName ='test_db2_zf', + updateMode ='append', + parallelism ='1', + batchSize ='100', + batchWaitInterval ='1000' + ); + +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type='db2', + url='jdbc:db2://172.16.8.104:50000/test?charset=utf8', + userName='dtstack', + password='abc123', + tableName ='test_db2_10', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + asyncPoolSize ='3', + parallelism ='1' + ); + +insert +into + MyResult + select + m.id, + s.name + from + MyTable m + join + sideTable s + on m.id=s.id; + +``` diff --git a/docs/plugin/impalaSide.md b/docs/plugin/impalaSide.md index 5c7479b04..500219b7f 100644 --- a/docs/plugin/impalaSide.md +++ b/docs/plugin/impalaSide.md @@ -1,6 +1,6 @@ - ## 1.格式： -``` + +```sql CREATE TABLE tableName( colName cloType, ... @@ -20,51 +20,65 @@ ); ``` -# 2.支持版本 +## 2.支持版本 + 2.10.0-cdh5.13.0 - + ## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | 注册到flink的表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| - -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型[impala] |是|| - | url | 连接postgresql数据库 jdbcUrl |是|| - | userName | postgresql连接用户名 |是|| - | password | postgresql连接密码|是|| - | tableName | postgresql表名称|是|| - | authMech | 身份验证机制 (0, 1, 2, 3), 暂不支持kerberos |是|0| - | principal | kerberos用于登录的principal（authMech=1时独有） |authMech=1为必填| - | keyTabFilePath | keytab文件的路径（authMech=1时独有） |authMech=1为必填 || - | krb5FilePath | krb5.conf文件路径（authMech=1时独有） |authMech=1为必填|| - | krbServiceName | Impala服务器的Kerberos principal名称（authMech=1时独有） |authMech=1为必填|| - | krbRealm | Kerberos的域名（authMech=1时独有） |否| HADOOP.COM | - | enablePartition | 是否支持分区|否|false| - | partitionfields | 分区字段名|否,enablePartition='true'时为必填|| - | partitionFieldTypes | 分区字段类型 |否,enablePartition='true'时为必填|| - | partitionValues | 分区值|否,enablePartition='true'时为必填|| - | cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - - -## 5.样例 + + [维表参数信息](docs/plugin/sideParams.md) + +impala独有的参数配置 + +| 参数名称 | 含义 | 是否必填 | 默认值 | +| ------------------- | ------------------------------------------------------------ | --------------------------------- | ---------- | +| type | 表明维表的类型[impala] | 是 | | +| url | 连接postgresql数据库 jdbcUrl | 是 | | +| userName | postgresql连接用户名 | 是 | | +| password | postgresql连接密码 | 是 | | +| tableName | postgresql表名称 | 是 | | +| authMech | 身份验证机制 (0, 1, 2, 3), 暂不支持kerberos | 是 | 0 | +| principal | kerberos用于登录的principal（authMech=1时独有） | authMech=1为必填 | | +| keyTabFilePath | keytab文件的路径（authMech=1时独有） | authMech=1为必填 | | +| krb5FilePath | krb5.conf文件路径（authMech=1时独有） | authMech=1为必填 | | +| krbServiceName | Impala服务器的Kerberos principal名称（authMech=1时独有） | authMech=1为必填 | | +| krbRealm | Kerberos的域名（authMech=1时独有） | 否 | HADOOP.COM | +| enablePartition | 是否支持分区 | 否 | false | +| partitionfields | 分区字段名 | 否,enablePartition='true'时为必填 | | +| partitionFieldTypes | 分区字段类型 | 否,enablePartition='true'时为必填 | | +| partitionValues | 分区值 | 否,enablePartition='true'时为必填 | | +| cache | 维表缓存策略(NONE/LRU/ALL) | 否 | NONE | +| partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量) | 否 | false | + +## 4.样例 + +### ALL全量维表定义 + +```sql + // 定义全量维表 +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='mysql', + url ='jdbc:impala://localhost:21050/mqtest', + userName ='dtstack', + password ='1abc123', + tableName ='test_impala_all', + authMech='3', + cache ='ALL', + cacheTTLMs ='60000', + parallelism ='2', + partitionedJoin='false' + ); + ``` + +### LRU异步维表定义 + +```sql create table sideTable( channel varchar, xccount int, @@ -87,11 +101,77 @@ create table sideTable( ``` -## 6.分区样例 +### MySQL异步维表关联 -注：分区字段放在最后面，如下，name是分区字段，放在channel，xccount字段的后面 +```sql +CREATE TABLE MyTable( + id int, + name varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='cannan_yctest01', + timezone='Asia/Shanghai', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + +CREATE TABLE MyResult( + id INT, + name VARCHAR + )WITH( + type='impala', + url='jdbc:impala://localhost:21050/mytest', + userName='dtstack', + password='abc123', + tableName ='test_impala_zf', + updateMode ='append', + parallelism ='1', + batchSize ='100', + batchWaitInterval ='1000' + ); + +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type='impala', + url='jdbc:impala://localhost:21050/mytest', + userName='dtstack', + password='abc123', + tableName ='test_impala_10', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + asyncPoolSize ='3', + parallelism ='1' + ); + +insert +into + MyResult + select + m.id, + s.name + from + MyTable m + join + sideTable s + on m.id=s.id; ``` + +### 分区样例 + +注：分区字段放在最后面，如下，name是分区字段，放在channel，xccount字段的后面 + +```sql create table sideTable( channel varchar, xccount int, @@ -118,4 +198,3 @@ create table sideTable( ``` - diff --git a/docs/plugin/sqlserverSide.md b/docs/plugin/sqlserverSide.md index 1cab43cbd..4d50ed619 100644 --- a/docs/plugin/sqlserverSide.md +++ b/docs/plugin/sqlserverSide.md @@ -1,6 +1,9 @@ - ## 1.格式： -``` + +通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 + +```sql CREATE TABLE tableName( colName cloType, ... @@ -20,41 +23,49 @@ ); ``` -# 2.支持版本 +## 2.支持版本 + mysql-5.6.35 - + ## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | sqlserver表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| - -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型 sqlserver |是|| - | url | 连接mysql数据库 jdbcUrl |是|| - | userName | sqlserver连接用户名 |是|| - | password | sqlserver连接密码|是|| - | tableName | sqlserver表名称|是|| - | tableName | sqlserver 的表名称|是|| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - -## 5.样例 + + [维表参数信息](docs/plugin/sideParams.md) + +sqlserver独有的参数配置 + +| 参数名称 | 含义 | 是否必填 | 默认值 | +| -------- | -------------------- | -------- | ------ | +| type | 维表类型， sqlserver | 是 | | +| url | 连接数据库 jdbcUrl | 是 | | +| userName | 连接用户名 | 是 | | +| password | 连接密码 | 是 | | + +## 4.样例 + +### ALL全量维表定义 + +```sql + // 定义全量维表 +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='sqlserver', + url ='jdbc:jtds:sqlserver://172.16.8.104:1433;DatabaseName=mytest', + userName ='dtstack', + password ='abc123', + tableName ='all_test_sqlserver', + cache ='ALL', + cacheTTLMs ='60000', + parallelism ='1' + ); ``` + +### LRU异步维表定义 + +```sql create table sideTable( channel varchar, xccount int, @@ -65,7 +76,7 @@ create table sideTable( url='jdbc:jtds:sqlserver://172.16.8.104:1433;DatabaseName=mytest', userName='dtstack', password='abc123', - tableName='sidetest', + tableName=',ru_test_sqlserver', cache ='LRU', cacheSize ='10000', cacheTTLMs ='60000', @@ -76,4 +87,69 @@ create table sideTable( ``` +### PostSQL异步维表关联 + +```sql +CREATE TABLE MyTable( + id int, + name varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='cannan_yctest01', + timezone='Asia/Shanghai', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + +CREATE TABLE MyResult( + id INT, + name VARCHAR + )WITH( + type='sqlserver', + url='jdbc:jtds:sqlserver://172.16.8.104:1433;DatabaseName=mytest', + userName='dtstack', + password='abc123', + tableName ='test_sqlserver_zf', + updateMode ='append', + parallelism ='1', + batchSize ='100', + batchWaitInterval ='1000' + ); + +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type='sqlserver', + url='jdbc:jtds:sqlserver://172.16.8.104:1433;DatabaseName=mytest', + userName='dtstack', + password='abc123', + tableName ='test_sqlserver_10', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + asyncPoolSize ='3', + parallelism ='1' + ); + +insert +into + MyResult + select + m.id, + s.name + from + MyTable m + join + sideTable s + on m.id=s.id; + +``` From 4e34155b0cdecccec1e5ff9a6fcbe88163b6e38d Mon Sep 17 00:00:00 2001 From: gkd Date: Mon, 27 Apr 2020 16:28:04 +0800 Subject: [PATCH 293/523] =?UTF-8?q?=E9=87=8D=E6=9E=84=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/db2Side.md | 115 +---------------------------------- docs/plugin/impalaSide.md | 1 - docs/plugin/sqlserverSide.md | 1 + 3 files changed, 4 insertions(+), 113 deletions(-) diff --git a/docs/plugin/db2Side.md b/docs/plugin/db2Side.md index 4fc7541ec..af0c29c02 100644 --- a/docs/plugin/db2Side.md +++ b/docs/plugin/db2Side.md @@ -1,9 +1,5 @@ -## 1.格式： - -通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， - 维表JOIN的条件必须与`keyInfo`字段一致。 -```sql +## 1.格式： 通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， 维表JOIN的条件必须与`keyInfo`字段一致。 ``` @@ -26,24 +22,10 @@ ); ``` -## 2.支持版本 - +# 2.支持版本 db2 9.X - + ## 3.表结构定义 - - [维表参数信息](docs/plugin/sideParams.md) - -db2独有的参数 - -| 参数名称 | 含义 | 是否必填 | 默认值 | -| -------- | ------------------ | -------- | ------ | -| type | 维表类型， db2 | 是 | | -| url | 连接数据库 jdbcUrl | 是 | | -| userName | 连接用户名 | 是 | | -| password | 连接密码 | 是 | | - -## 4.参数 [维表参数信息](docs/plugin/sideParams.md) @@ -58,35 +40,9 @@ db2独有的参数 | schema | 表所属scheam|否|| -### ALL全量维表定义 - -```sql -// 定义全量维表 -CREATE TABLE sideTable( - id INT, - name VARCHAR, - PRIMARY KEY(id) , - PERIOD FOR SYSTEM_TIME - )WITH( - type ='db2', - url ='jdbc:db2://172.16.8.104:50000/test?charset=utf8', - userName ='dtstack', - password ='abc123', - tableName ='all_test_db2', - cache ='ALL', - cacheTTLMs ='60000', - parallelism ='1' - ); -``` -### LRU异步维表定义 ## 4.样例 -```sql -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), ### ALL全量维表定义 ``` // 定义全量维表 @@ -100,7 +56,6 @@ CREATE TABLE sideTable( url='jdbc:db2://172.16.10.251:50000/mqTest', userName='DB2INST1', password='abc123', - tableName='lru_test_db2', tableName='USER_INFO2', schema = 'DTSTACK' cache ='ALL', @@ -131,7 +86,6 @@ CREATE TABLE sideTable( asyncPoolSize ='3', parallelism ='2' ); -``` ``` @@ -168,57 +122,7 @@ CREATE TABLE source2( batchSize = '1' ); -### Db2异步维表关联 - -```sql -CREATE TABLE MyTable( - id int, - name varchar - )WITH( - type ='kafka11', - bootstrapServers ='172.16.8.107:9092', - zookeeperQuorum ='172.16.8.107:2181/kafka', - offsetReset ='latest', - topic ='cannan_yctest01', - timezone='Asia/Shanghai', - enableKeyPartitions ='false', - topicIsPattern ='false', - parallelism ='1' - ); - -CREATE TABLE MyResult( - id INT, - name VARCHAR - )WITH( - type='db2', - url='jdbc:db2://172.16.8.104:50000/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName ='test_db2_zf', - updateMode ='append', - parallelism ='1', - batchSize ='100', - batchWaitInterval ='1000' - ); -CREATE TABLE sideTable( - id INT, - name VARCHAR, - PRIMARY KEY(id) , - PERIOD FOR SYSTEM_TIME - )WITH( - type='db2', - url='jdbc:db2://172.16.8.104:50000/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName ='test_db2_10', - partitionedJoin ='false', - cache ='LRU', - cacheSize ='10000', - cacheTTLMs ='60000', - asyncPoolSize ='3', - parallelism ='1' - ); CREATE TABLE MyResult( id int, name VARCHAR, @@ -259,19 +163,6 @@ GRANT CONTROL ON TABLE "DTSTACK "."USER_INFO2" TO USER "DB2INST1" ; ``` -insert -into - MyResult - select - m.id, - s.name - from - MyTable m - join - sideTable s - on m.id=s.id; - -``` 维表数据：(1001,maqi,hz) 源表数据：{"name":"maqi","id":1001} diff --git a/docs/plugin/impalaSide.md b/docs/plugin/impalaSide.md index 500219b7f..dce959388 100644 --- a/docs/plugin/impalaSide.md +++ b/docs/plugin/impalaSide.md @@ -98,7 +98,6 @@ create table sideTable( partitionedJoin='false' ); - ``` ### MySQL异步维表关联 diff --git a/docs/plugin/sqlserverSide.md b/docs/plugin/sqlserverSide.md index 4d50ed619..f95d8bba5 100644 --- a/docs/plugin/sqlserverSide.md +++ b/docs/plugin/sqlserverSide.md @@ -63,6 +63,7 @@ CREATE TABLE sideTable( ); ``` + ### LRU异步维表定义 ```sql From d6f72c937960e0047ea03fcdbc07b5d7fc459aa1 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 27 Apr 2020 16:51:49 +0800 Subject: [PATCH 294/523] mongo doc --- docs/plugin/mongoSide.md | 142 +++++++++++++++++++++++++++++---------- docs/plugin/mongoSink.md | 75 ++++++++++++++++++--- 2 files changed, 170 insertions(+), 47 deletions(-) diff --git a/docs/plugin/mongoSide.md b/docs/plugin/mongoSide.md index 90ffa2582..92362a752 100644 --- a/docs/plugin/mongoSide.md +++ b/docs/plugin/mongoSide.md @@ -26,53 +26,123 @@ ## 3.表结构定义 - |参数名称|含义| - |----|---| - | tableName | 注册到flink的表名称(可选填;不填默认和hbase对应的表名称相同)| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| + [通用维表参数信息](docs/plugin/sideParams.md) -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type |表明 输出表类型 mongo|是|| - | address | 连接mongo数据库 jdbcUrl |是|| - | userName | mongo连接用户名|否|| - | password | mongo连接密码|否|| - | tableName | mongo表名称|是|| - | database | mongo表名称|是|| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - + + mongo相关参数配置： + +|参数名称|含义|是否必填|默认值| +|----|---|---|----| +| type |表明 输出表类型 mongo|是|| +| address | 连接mongo数据库 jdbcUrl |是|| +| userName | mongo连接用户名|否|| +| password | mongo连接密码|否|| +| tableName | mongo表名称|是|| +| database | mongo表名称|是|| + +## 4.样例 -## 5.样例 + +### 全量维表结构 + +```aidl +CREATE TABLE source2( + id int, + address VARCHAR, + PERIOD FOR SYSTEM_TIME +)WITH( + type ='mongo', + address ='172.16.8.193:27017', + database ='dtstack', + tableName ='userInfo', + cache ='ALL', + parallelism ='1', + partitionedJoin='false' +); ``` -create table sideTable( - CHANNEL varchar, - XCCOUNT int, - PRIMARY KEY(channel), + +### 异步维表结构 + +```aidl +CREATE TABLE source2( + id int, + address VARCHAR, PERIOD FOR SYSTEM_TIME - )WITH( +)WITH( type ='mongo', - address ='172.21.32.1:27017,172.21.32.1:27017', - database ='test', - tableName ='sidetest', + address ='172.16.8.193:27017', + database ='dtstack', + tableName ='userInfo', cache ='LRU', parallelism ='1', partitionedJoin='false' +); + +``` + +### 异步维表关联样例 + +``` + +CREATE TABLE source1 ( + id int, + name VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest03', + timezone='Asia/Shanghai', + topicIsPattern ='false' ); +CREATE TABLE source2( + id int, + address VARCHAR, + PERIOD FOR SYSTEM_TIME +)WITH( + type ='mongo', + address ='172.16.8.193:27017', + database ='dtstack', + tableName ='userInfo', + cache ='ALL', + parallelism ='1', + partitionedJoin='false' +); + + +CREATE TABLE MyResult( + id int, + name VARCHAR, + address VARCHAR, + primary key (id) +)WITH( + type='console' +); + +insert into MyResult +select + s1.id, + s1.name, + s2.address +from + source1 s1 +left join + source2 s2 +on + s1.id = s2.id + + ``` +维表数据：{"id": 1001,"address":"hz""} + +源表数据：{"name":"maqi","id":1001} + + +输出结果： (1001,maqi,hz) + + diff --git a/docs/plugin/mongoSink.md b/docs/plugin/mongoSink.md index 9900dea12..8c32dc95e 100644 --- a/docs/plugin/mongoSink.md +++ b/docs/plugin/mongoSink.md @@ -40,17 +40,70 @@ CREATE TABLE tableName( |parallelism | 并行度设置|否|1| ## 5.样例： + +双流join并插入mongo + ``` + +CREATE TABLE source1 ( + id int, + name VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest03', + timezone='Asia/Shanghai', + topicIsPattern ='false' + ); + + + +CREATE TABLE source2( + id int, + address VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest04', + timezone='Asia/Shanghai', + topicIsPattern ='false' +); + + CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR - )WITH( + id int, + name VARCHAR, + address VARCHAR, + primary key (id) +)WITH( type ='mongo', - address ='172.21.32.1:27017,172.21.32.1:27017', - userName ='dtstack', - password ='abc123', - database ='test', - tableName ='pv', - parallelism ='1' - ) - ``` \ No newline at end of file + address ='172.16.8.193:27017', + database ='dtstack', + tableName ='userInfo' +); + +insert into MyResult +select + s1.id, + s1.name, + s2.address +from + source1 s1 +left join + source2 s2 +on + s1.id = s2.id + + + ``` + + + 数据结果： + + 向Topic mqTest03 发送数据 {"name":"maqi","id":1001} 插入 (1001,"maqi",null) + + 向Topic mqTest04 发送数据 {"address":"hz","id":1001} 插入 (1001,"maqi","hz") \ No newline at end of file From f17dc3fbf526196fb60366c0a996a7361f834ff3 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 28 Apr 2020 09:41:17 +0800 Subject: [PATCH 295/523] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/launcher/PluginLoadModeTest.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java b/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java index 0a153d018..3916d8543 100644 --- a/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java +++ b/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java @@ -48,8 +48,22 @@ public static void testClasspathMode() throws Exception { LauncherMain.main(sql); } + + public static void testRocSql() throws Exception{ + String[] sql = new String[]{"-mode", "local", "-sql", "/Users/roc/Documents/flink_sql/sql/zy_sql/hbase_side.sql", "-name", "roc", + "-localSqlPluginPath", "/Users/roc/workspace/git_code/flinkStreamSQL/plugins", + "-remoteSqlPluginPath", "/Users/roc/workspace/git_code/flinkStreamSQL/plugins", + "-flinkconf", "/Users/roc/Documents/flink_sql/flinkconf", + "-confProp", "{\"sql.checkpoint.cleanup.mode\":\"false\",\"sql.checkpoint.interval\":10000,\"time.characteristic\":\"EventTime\"}", + "-yarnconf", "/Users/roc/Documents/flink_sql/yarnconf", + "-flinkJarPath", "/Users/roc/Documents/flink_sql/flinkJarPath", "-queue", "c", "-pluginLoadMode", "classpath"}; + System.setProperty("HADOOP_USER_NAME", "admin"); + LauncherMain.main(sql); + } + public static void main(String[] args) throws Exception { - testShipfileMode(); + testRocSql(); +// testShipfileMode(); // testClasspathMode(); } } From 9957cfd80fd4ad9d3f5e0668c717ed090327d17a Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 28 Apr 2020 14:06:55 +0800 Subject: [PATCH 296/523] =?UTF-8?q?=E6=94=AF=E6=8C=81decimal=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/side/hbase/utils/HbaseUtils.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java index 39df3c913..5a08b278a 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java @@ -66,9 +66,12 @@ public static Object convertByte(byte[] hbaseData, String type){ case "double": return Bytes.toDouble(hbaseData); - + case "decimal": + return Bytes.toBigDecimal(hbaseData); + default: + throw new RuntimeException("not support type of " + type); } - - throw new RuntimeException("not support type of " + type); } + + } From dfea94ba8db66547bba0ca734880b21afcaed676 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 28 Apr 2020 14:42:21 +0800 Subject: [PATCH 297/523] =?UTF-8?q?hbase=20=E7=88=B6=E7=B1=BB=E6=B2=A1?= =?UTF-8?q?=E6=9C=89open=E5=92=8C=E7=BC=93=E5=AD=98=E4=B8=AD=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E7=B1=BB=E5=9E=8B=E4=B8=8D=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java | 1 + .../sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 251b88034..0daf2bea5 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -89,6 +89,7 @@ public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List Date: Tue, 28 Apr 2020 15:05:46 +0800 Subject: [PATCH 298/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9kafka=20json=20?= =?UTF-8?q?=E5=AF=B9=E7=B1=BB=E5=9E=8B=E6=AF=94=E8=BE=83=E7=9A=84=E9=83=A8?= =?UTF-8?q?=E5=88=86=E3=80=82=E4=BF=AE=E6=94=B9=20'=3D=3D'=20=E4=B8=BA=20e?= =?UTF-8?q?quals;?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kafka/AbstractKafkaProducerFactory.java | 6 +- .../json/DTJsonRowSerializationSchema.java | 225 +++++++++++++++++ .../kafka/AbstractKafkaConsumerFactory.java | 6 +- .../json/DTJsonRowDeserializationSchema.java | 233 ++++++++++++++++++ 4 files changed, 464 insertions(+), 6 deletions(-) create mode 100644 kafka-base/kafka-base-sink/src/main/java/org/apache/flink/formats/json/DTJsonRowSerializationSchema.java create mode 100644 kafka-base/kafka-base-source/src/main/java/org/apache/flink/formats/json/DTJsonRowDeserializationSchema.java diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java index 88c2ca939..9bdc0d64f 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java @@ -25,7 +25,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.formats.avro.AvroRowSerializationSchema; import org.apache.flink.formats.csv.CsvRowSerializationSchema; -import org.apache.flink.formats.json.JsonRowSerializationSchema; +import org.apache.flink.formats.json.DTJsonRowSerializationSchema; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.types.Row; @@ -62,9 +62,9 @@ private SerializationSchema createSerializationSchema(KafkaSinkTableInfo ka if (FormatType.JSON.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { if (StringUtils.isNotBlank(kafkaSinkTableInfo.getSchemaString())) { - serializationSchema = new JsonRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); + serializationSchema = new DTJsonRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); } else if (typeInformation != null && typeInformation.getArity() != 0) { - serializationSchema = new JsonRowSerializationSchema(typeInformation); + serializationSchema = new DTJsonRowSerializationSchema(typeInformation); } else { throw new IllegalArgumentException("sinkDataType:" + FormatType.JSON.name() + " must set schemaString（JSON Schema）or TypeInformation"); } diff --git a/kafka-base/kafka-base-sink/src/main/java/org/apache/flink/formats/json/DTJsonRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/org/apache/flink/formats/json/DTJsonRowSerializationSchema.java new file mode 100644 index 000000000..f67d2353d --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/org/apache/flink/formats/json/DTJsonRowSerializationSchema.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.json; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Time; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.util.Objects; + +/** + * Serialization schema that serializes an object of Flink types into a JSON bytes. + * + *

Serializes the input Flink object into a JSON string and + * converts it into byte[]. + * + *

Result byte[] messages can be deserialized using {@link DTJsonRowDeserializationSchema}. + */ +@PublicEvolving +public class DTJsonRowSerializationSchema implements SerializationSchema { + + private static final long serialVersionUID = -2885556750743978636L; + + /** Type information describing the input type. */ + private final TypeInformation typeInfo; + + /** Object mapper that is used to create output JSON objects. */ + private final ObjectMapper mapper = new ObjectMapper(); + + /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone, without milliseconds). */ + private SimpleDateFormat timeFormat = new SimpleDateFormat("HH:mm:ss'Z'"); + + /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone). */ + private SimpleDateFormat timeFormatWithMillis = new SimpleDateFormat("HH:mm:ss.SSS'Z'"); + + /** Formatter for RFC 3339-compliant string representation of a timestamp value (with UTC timezone). */ + private SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + + /** Reusable object node. */ + private transient ObjectNode node; + + /** + * Creates a JSON serialization schema for the given type information. + * + * @param typeInfo The field names of {@link Row} are used to map to JSON properties. + */ + public DTJsonRowSerializationSchema(TypeInformation typeInfo) { + Preconditions.checkNotNull(typeInfo, "Type information"); + this.typeInfo = typeInfo; + } + + /** + * Creates a JSON serialization schema for the given JSON schema. + * + * @param jsonSchema JSON schema describing the result type + * + * @see http://json-schema.org/ + */ + public DTJsonRowSerializationSchema(String jsonSchema) { + this(JsonRowSchemaConverter.convert(jsonSchema)); + } + + @Override + public byte[] serialize(Row row) { + if (node == null) { + node = mapper.createObjectNode(); + } + + try { + convertRow(node, (RowTypeInfo) typeInfo, row); + return mapper.writeValueAsBytes(node); + } catch (Throwable t) { + throw new RuntimeException("Could not serialize row '" + row + "'. " + + "Make sure that the schema matches the input.", t); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DTJsonRowSerializationSchema that = (DTJsonRowSerializationSchema) o; + return Objects.equals(typeInfo, that.typeInfo); + } + + @Override + public int hashCode() { + return Objects.hash(typeInfo); + } + + // -------------------------------------------------------------------------------------------- + + private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) { + if (reuse == null) { + reuse = mapper.createObjectNode(); + } + final String[] fieldNames = info.getFieldNames(); + final TypeInformation[] fieldTypes = info.getFieldTypes(); + + // validate the row + if (row.getArity() != fieldNames.length) { + throw new IllegalStateException(String.format( + "Number of elements in the row '%s' is different from number of field names: %d", row, fieldNames.length)); + } + + for (int i = 0; i < fieldNames.length; i++) { + final String name = fieldNames[i]; + + final JsonNode fieldConverted = convert(reuse, reuse.get(name), fieldTypes[i], row.getField(i)); + reuse.set(name, fieldConverted); + } + + return reuse; + } + + private JsonNode convert(ContainerNode container, JsonNode reuse, TypeInformation info, Object object) { + if (info.equals(Types.VOID) || object == null) { + return container.nullNode(); + } else if (info.equals(Types.BOOLEAN)) { + return container.booleanNode((Boolean) object); + } else if (info.equals(Types.STRING)) { + return container.textNode((String) object); + } else if (info.equals(Types.BIG_DEC)) { + // convert decimal if necessary + if (object instanceof BigDecimal) { + return container.numberNode((BigDecimal) object); + } + return container.numberNode(BigDecimal.valueOf(((Number) object).doubleValue())); + } else if (info.equals(Types.BIG_INT)) { + // convert integer if necessary + if (object instanceof BigInteger) { + return container.numberNode((BigInteger) object); + } + return container.numberNode(BigInteger.valueOf(((Number) object).longValue())); + } else if (info.equals(Types.SQL_DATE)) { + return container.textNode(object.toString()); + } else if (info.equals(Types.SQL_TIME)) { + final Time time = (Time) object; + // strip milliseconds if possible + if (time.getTime() % 1000 > 0) { + return container.textNode(timeFormatWithMillis.format(time)); + } + return container.textNode(timeFormat.format(time)); + } else if (info.equals(Types.SQL_TIMESTAMP)) { + return container.textNode(timestampFormat.format((Timestamp) object)); + } else if (info instanceof RowTypeInfo) { + if (reuse != null && reuse instanceof ObjectNode) { + return convertRow((ObjectNode) reuse, (RowTypeInfo) info, (Row) object); + } else { + return convertRow(null, (RowTypeInfo) info, (Row) object); + } + } else if (info instanceof ObjectArrayTypeInfo) { + if (reuse != null && reuse instanceof ArrayNode) { + return convertObjectArray((ArrayNode) reuse, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } else { + return convertObjectArray(null, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } + } else if (info instanceof BasicArrayTypeInfo) { + if (reuse != null && reuse instanceof ArrayNode) { + return convertObjectArray((ArrayNode) reuse, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } else { + return convertObjectArray(null, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } + } else if (info instanceof PrimitiveArrayTypeInfo && ((PrimitiveArrayTypeInfo) info).getComponentType().equals(Types.BYTE)) { + return container.binaryNode((byte[]) object); + } else { + // for types that were specified without JSON schema + // e.g. POJOs + try { + return mapper.valueToTree(object); + } catch (IllegalArgumentException e) { + throw new IllegalStateException("Unsupported type information '" + info + "' for object: " + object, e); + } + } + } + + private ArrayNode convertObjectArray(ArrayNode reuse, TypeInformation info, Object[] array) { + if (reuse == null) { + reuse = mapper.createArrayNode(); + } else { + reuse.removeAll(); + } + + for (Object object : array) { + reuse.add(convert(reuse, null, info, object)); + } + return reuse; + } +} diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java index 2cae221cf..29cf3193b 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java @@ -27,7 +27,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.formats.avro.AvroRowDeserializationSchema; import org.apache.flink.formats.csv.CsvRowDeserializationSchema; -import org.apache.flink.formats.json.JsonRowDeserializationSchema; +import org.apache.flink.formats.json.DTJsonRowDeserializationSchema; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase; import org.apache.flink.types.Row; @@ -61,9 +61,9 @@ private DeserializationSchema createDeserializationSchema(KafkaSourceTableI } else if (FormatType.JSON.name().equalsIgnoreCase(kafkaSourceTableInfo.getSourceDataType())) { if (StringUtils.isNotBlank(kafkaSourceTableInfo.getSchemaString())) { - deserializationSchema = new JsonRowDeserializationSchema(kafkaSourceTableInfo.getSchemaString()); + deserializationSchema = new DTJsonRowDeserializationSchema(kafkaSourceTableInfo.getSchemaString()); } else if (typeInformation != null && typeInformation.getArity() != 0) { - deserializationSchema = new JsonRowDeserializationSchema(typeInformation); + deserializationSchema = new DTJsonRowDeserializationSchema(typeInformation); } else { throw new IllegalArgumentException("sourceDataType:" + FormatType.JSON.name() + " must set schemaString（JSON Schema）or TypeInformation"); } diff --git a/kafka-base/kafka-base-source/src/main/java/org/apache/flink/formats/json/DTJsonRowDeserializationSchema.java b/kafka-base/kafka-base-source/src/main/java/org/apache/flink/formats/json/DTJsonRowDeserializationSchema.java new file mode 100644 index 000000000..6204058fe --- /dev/null +++ b/kafka-base/kafka-base-source/src/main/java/org/apache/flink/formats/json/DTJsonRowDeserializationSchema.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.json; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.io.IOException; +import java.lang.reflect.Array; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.Objects; + +/** + * Deserialization schema from JSON to Flink types. + * + *

Deserializes a byte[] message as a JSON object and reads + * the specified fields. + * + *

Failures during deserialization are forwarded as wrapped IOExceptions. + */ +@PublicEvolving +public class DTJsonRowDeserializationSchema implements DeserializationSchema { + + private static final long serialVersionUID = -228294330688809195L; + + /** Type information describing the result type. */ + private final TypeInformation typeInfo; + + /** Object mapper for parsing the JSON. */ + private final ObjectMapper objectMapper = new ObjectMapper(); + + /** Flag indicating whether to fail on a missing field. */ + private boolean failOnMissingField; + + /** + * Creates a JSON deserialization schema for the given type information. + * + * @param typeInfo Type information describing the result type. The field names of {@link Row} + * are used to parse the JSON properties. + */ + public DTJsonRowDeserializationSchema(TypeInformation typeInfo) { + Preconditions.checkNotNull(typeInfo, "Type information"); + this.typeInfo = typeInfo; + + if (!(typeInfo instanceof RowTypeInfo)) { + throw new IllegalArgumentException("Row type information expected."); + } + } + + /** + * Creates a JSON deserialization schema for the given JSON schema. + * + * @param jsonSchema JSON schema describing the result type + * + * @see http://json-schema.org/ + */ + public DTJsonRowDeserializationSchema(String jsonSchema) { + this(JsonRowSchemaConverter.convert(jsonSchema)); + } + + @Override + public Row deserialize(byte[] message) throws IOException { + try { + final JsonNode root = objectMapper.readTree(message); + return convertRow(root, (RowTypeInfo) typeInfo); + } catch (Throwable t) { + throw new IOException("Failed to deserialize JSON object.", t); + } + } + + @Override + public boolean isEndOfStream(Row nextElement) { + return false; + } + + @Override + public TypeInformation getProducedType() { + return typeInfo; + } + + /** + * Configures the failure behaviour if a JSON field is missing. + * + *

By default, a missing field is ignored and the field is set to null. + * + * @param failOnMissingField Flag indicating whether to fail or not on a missing field. + */ + public void setFailOnMissingField(boolean failOnMissingField) { + this.failOnMissingField = failOnMissingField; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DTJsonRowDeserializationSchema that = (DTJsonRowDeserializationSchema) o; + return failOnMissingField == that.failOnMissingField && Objects.equals(typeInfo, that.typeInfo); + } + + @Override + public int hashCode() { + return Objects.hash(typeInfo, failOnMissingField); + } + + // -------------------------------------------------------------------------------------------- + + private Object convert(JsonNode node, TypeInformation info) { + if (info.equals(Types.VOID) || node.isNull()) { + return null; + } else if (info.equals(Types.BOOLEAN)) { + return node.asBoolean(); + } else if (info.equals(Types.STRING)) { + return node.asText(); + } else if (info.equals(Types.BIG_DEC)) { + return node.decimalValue(); + } else if (info.equals(Types.BIG_INT)) { + return node.bigIntegerValue(); + } else if (info.equals(Types.SQL_DATE)) { + return Date.valueOf(node.asText()); + } else if (info.equals(Types.SQL_TIME)) { + // according to RFC 3339 every full-time must have a timezone; + // until we have full timezone support, we only support UTC; + // users can parse their time as string as a workaround + final String time = node.asText(); + if (time.indexOf('Z') < 0 || time.indexOf('.') >= 0) { + throw new IllegalStateException( + "Invalid time format. Only a time in UTC timezone without milliseconds is supported yet. " + + "Format: HH:mm:ss'Z'"); + } + return Time.valueOf(time.substring(0, time.length() - 1)); + } else if (info.equals(Types.SQL_TIMESTAMP)) { + // according to RFC 3339 every date-time must have a timezone; + // until we have full timezone support, we only support UTC; + // users can parse their time as string as a workaround + final String timestamp = node.asText(); + if (timestamp.indexOf('Z') < 0) { + throw new IllegalStateException( + "Invalid timestamp format. Only a timestamp in UTC timezone is supported yet. " + + "Format: yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + } + return Timestamp.valueOf(timestamp.substring(0, timestamp.length() - 1).replace('T', ' ')); + } else if (info instanceof RowTypeInfo) { + return convertRow(node, (RowTypeInfo) info); + } else if (info instanceof ObjectArrayTypeInfo) { + return convertObjectArray(node, ((ObjectArrayTypeInfo) info).getComponentInfo()); + } else if (info instanceof BasicArrayTypeInfo) { + return convertObjectArray(node, ((BasicArrayTypeInfo) info).getComponentInfo()); + } else if (info instanceof PrimitiveArrayTypeInfo && + ((PrimitiveArrayTypeInfo) info).getComponentType() == Types.BYTE) { + return convertByteArray(node); + } else { + // for types that were specified without JSON schema + // e.g. POJOs + try { + return objectMapper.treeToValue(node, info.getTypeClass()); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Unsupported type information '" + info + "' for node: " + node); + } + } + } + + private Row convertRow(JsonNode node, RowTypeInfo info) { + final String[] names = info.getFieldNames(); + final TypeInformation[] types = info.getFieldTypes(); + + final Row row = new Row(names.length); + for (int i = 0; i < names.length; i++) { + final String name = names[i]; + final JsonNode subNode = node.get(name); + if (subNode == null) { + if (failOnMissingField) { + throw new IllegalStateException( + "Could not find field with name '" + name + "'."); + } else { + row.setField(i, null); + } + } else { + row.setField(i, convert(subNode, types[i])); + } + } + + return row; + } + + private Object convertObjectArray(JsonNode node, TypeInformation elementType) { + final Object[] array = (Object[]) Array.newInstance(elementType.getTypeClass(), node.size()); + for (int i = 0; i < node.size(); i++) { + array[i] = convert(node.get(i), elementType); + } + return array; + } + + private Object convertByteArray(JsonNode node) { + try { + return node.binaryValue(); + } catch (IOException e) { + throw new RuntimeException("Unable to deserialize byte array.", e); + } + } +} From ee766b80005bb4dee464fb67d8d48e6ef39deeaa Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 28 Apr 2020 20:25:29 +0800 Subject: [PATCH 299/523] =?UTF-8?q?fix=20=E7=BC=93=E5=AD=98=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E6=B2=A1=E6=9C=89=E8=A7=A3=E6=9E=90=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java | 2 +- .../com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 0daf2bea5..47adc88b9 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -148,7 +148,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce return; } else if (ECacheContentType.SingleLine == val.getType()) { try { - Row row = fillData(inputCopy.row(), val); + Row row = fillData(inputCopy.row(), val.getContent()); resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); } catch (Exception e) { dealFillDataError(resultFuture, e, inputCopy); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 7b627da4d..901c000f5 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -61,6 +61,7 @@ public HbaseSideParser() { public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseSideTableInfo hbaseTableInfo = new HbaseSideTableInfo(); hbaseTableInfo.setName(tableName); + parseCacheProp(hbaseTableInfo, props); parseFieldsInfo(fieldsInfo, hbaseTableInfo); hbaseTableInfo.setTableName((String) props.get(TABLE_NAME_KEY.toLowerCase())); hbaseTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(PARALLELISM_KEY.toLowerCase()))); From bbb4e9c6b8652f7c8e34412df6b12df002f8ead5 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 29 Apr 2020 15:26:39 +0800 Subject: [PATCH 300/523] =?UTF-8?q?update=20kudu-side=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/kuduSide.md | 151 +++++++++++++++++++++++++--------------- pom.xml | 4 +- 2 files changed, 98 insertions(+), 57 deletions(-) diff --git a/docs/plugin/kuduSide.md b/docs/plugin/kuduSide.md index 718d43117..a45300deb 100644 --- a/docs/plugin/kuduSide.md +++ b/docs/plugin/kuduSide.md @@ -1,53 +1,34 @@ - -## 1.格式： -All: -``` -create table sideTable( - id int, - tablename1 VARCHAR, - PRIMARY KEY(id), - PERIOD FOR SYSTEM_TIME - )WITH( - type='kudu', - kuduMasters ='ip1,ip2,ip3', - tableName ='impala::default.testSide', - cache ='ALL', - primaryKey='id，xx', - lowerBoundPrimaryKey='10,xx', - upperBoundPrimaryKey='15,xx', - workerCount='1', - defaultOperationTimeoutMs='600000', - defaultSocketReadTimeoutMs='6000000', - batchSizeBytes='100000000', - limitNum='1000', - isFaultTolerant='false', - partitionedJoin='false' - ); -``` -LRU: +## 1.格式 + 通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 ``` -create table sideTable( - id int, - tablename1 VARCHAR, - PRIMARY KEY(id), + CREATE TABLE tableName( + colName cloType, + ... + PRIMARY KEY(colName1,colName2) , PERIOD FOR SYSTEM_TIME - )WITH( - type='kudu', + )WITH( + type ='kudu', kuduMasters ='ip1,ip2,ip3', tableName ='impala::default.testSide', - cache ='LRU', + primaryKey='id，xx', + lowerBoundPrimaryKey='10,xx', + upperBoundPrimaryKey='15,xx', workerCount='1', defaultOperationTimeoutMs='600000', defaultSocketReadTimeoutMs='6000000', batchSizeBytes='100000000', limitNum='1000', isFaultTolerant='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1', partitionedJoin='false' - ); - ``` - + ); +``` ## 2.支持版本 -kudu 1.10.0+cdh6.2.0 + kudu 1.10.0+cdh6.2.0 ## 3.表结构定义 @@ -55,12 +36,13 @@ kudu 1.10.0+cdh6.2.0 |----|---| | tableName | 注册到flink的表名称(可选填;不填默认和hbase对应的表名称相同)| | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| + | colType | 列类型 [colType支持的类型](colType.md)| | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| -## 3.参数 +## 4.参数 +参数详细说明请看[参数详细说明](./sideParams.md) |参数名称|含义|是否必填|默认值| |----|---|---|-----| @@ -78,17 +60,25 @@ kudu 1.10.0+cdh6.2.0 | isFaultTolerant |查询是否容错 查询失败是否扫描第二个副本 默认false 容错 | 否|| | cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - -------------- -> 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) -## 4.样例 -All: +## 5.样例 +### LRU维表示例 +``` +create table sideTable( + id int, + tablename1 VARCHAR, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME + )WITH( + type='kudu', + kuduMasters ='ip1,ip2,ip3', + tableName ='impala::default.testSide', + cache ='LRU', + partitionedJoin='false' + ); +``` +### ALL维表示例 ``` create table sideTable( id int, @@ -106,19 +96,70 @@ create table sideTable( partitionedJoin='false' ); ``` -LRU: + +## 6.kudu异步关联完整样例 + ``` -create table sideTable( +CREATE TABLE MyTable( + id bigint, + name varchar, + address varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_in2', + timezone = 'Asia/Shanghai', + topicIsPattern = 'false', + parallelism = '1' +); + +CREATE TABLE sideTable( id int, - tablename1 VARCHAR, + message varchar, PRIMARY KEY(id), PERIOD FOR SYSTEM_TIME - )WITH( +)WITH( type='kudu', kuduMasters ='ip1,ip2,ip3', tableName ='impala::default.testSide', cache ='LRU', partitionedJoin='false' - ); - ``` +); + +CREATE TABLE MyResult( + id bigint, + name varchar, + address varchar, + message varchar +)WITH( + type ='console', + address ='192.168.80.106:9042,192.168.80.107:9042', + userName='cassandra', + password='cassandra', + database ='tiezhu', + tableName ='stu_out', + parallelism ='1' +); + +insert +into + MyResult + select + t1.id AS id, + t1.name AS name, + t1.address AS address, + t2.message AS message + from( + select + id, + name, + address + from + MyTable + ) t1 + join sideTable t2 + on t1.id = t2.id; +``` diff --git a/pom.xml b/pom.xml index b5b674379..2b2d0a986 100644 --- a/pom.xml +++ b/pom.xml @@ -25,10 +25,10 @@ - + kudu - + console From 1a361e1e3c86a0e09b3460ab05b43540f002cd01 Mon Sep 17 00:00:00 2001 From: xuchao Date: Thu, 30 Apr 2020 19:19:43 +0800 Subject: [PATCH 301/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=A1=A8=E5=90=8D?= =?UTF-8?q?=E7=A7=B0=E5=92=8C=20schema=20=E6=B7=BB=E5=8A=A0=E5=BC=95?= =?UTF-8?q?=E5=8F=B7=E7=9A=84=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/util/DtStringUtil.java | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index b2486b1ab..f84cc9ae4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -261,14 +261,43 @@ public static String firstUpperCase(String str) { } public static String getTableFullPath(String schema, String tableName) { + String[] tableInfoSplit = StringUtils.split(tableName, "."); + //表明表信息带了schema + if(tableInfoSplit.length == 2){ + schema = tableInfoSplit[0]; + tableName = tableInfoSplit[1]; + } + + //清理首个字符" 和最后字符 " + schema = rmStrQuote(schema); + tableName = rmStrQuote(tableName); + if (StringUtils.isEmpty(schema)){ return addQuoteForStr(tableName); } + String schemaAndTabName = addQuoteForStr(schema) + "." + addQuoteForStr(tableName); return schemaAndTabName; } + /** + * 清理首个字符" 和最后字符 " + */ + public static String rmStrQuote(String str){ + if(StringUtils.isEmpty(str)){ + return str; + } + + if(str.startsWith("\"")){ + str = str.substring(1); + } + + if(str.endsWith("\"")){ + str = str.substring(0, str.length()-1); + } + return str; + } public static String addQuoteForStr(String column) { return getStartQuote() + column + getEndQuote(); From cc228def0670a8b6c62add4e4be267c437f3e4ed Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 6 May 2020 19:13:48 +0800 Subject: [PATCH 302/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=AD=E5=8E=9Fhba?= =?UTF-8?q?se=E7=BB=B4=E8=A1=A8=E5=8F=96=E5=87=BA=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=9C=89=E8=AF=AF=E7=9A=84=E9=97=AE=E9=A2=98=E5=90=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 664fe6abd..9b79c5c3d 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -33,6 +33,7 @@ import com.dtstack.flink.sql.factory.DTThreadFactory; import com.google.common.collect.Maps; import com.stumbleupon.async.Deferred; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -83,7 +84,7 @@ public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List Date: Mon, 11 May 2020 15:06:00 +0800 Subject: [PATCH 303/523] =?UTF-8?q?redis=20=E7=BB=93=E6=9E=9C=E8=A1=A8?= =?UTF-8?q?=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/launcher/PluginLoadModeTest.java | 15 +++++++++- pom.xml | 28 +++++++++---------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java b/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java index 0a153d018..923c74116 100644 --- a/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java +++ b/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java @@ -48,8 +48,21 @@ public static void testClasspathMode() throws Exception { LauncherMain.main(sql); } + public static void testroc() throws Exception { + String[] sql = new String[]{"-mode", "local", "-sql", "/Users/roc/Documents/flink_sql/sql/result/redis_result.sql", "-name", "roc", + "-localSqlPluginPath", "/Users/roc/workspace/git_code/flinkStreamSQL/plugins", + "-remoteSqlPluginPath", "/Users/roc/workspace/git_code/flinkStreamSQL/plugins", + "-flinkconf", "//Users/roc/Documents/flink_sql/flinkconf", + "-confProp", "{\"sql.checkpoint.cleanup.mode\":\"false\",\"sql.checkpoint.interval\":10000,\"time.characteristic\":\"EventTime\"}", + "-yarnconf", "/Users/roc/Documents/flink_sql/yarnconf", + "-flinkJarPath", "/Users/roc/Documents/flink_sql/flinkJarPath", "-queue", "c", "-pluginLoadMode", "classpath"}; + System.setProperty("HADOOP_USER_NAME", "admin"); + LauncherMain.main(sql); + } + public static void main(String[] args) throws Exception { - testShipfileMode(); + testroc(); +// testShipfileMode(); // testClasspathMode(); } } diff --git a/pom.xml b/pom.xml index 2b2d0a986..2794f9004 100644 --- a/pom.xml +++ b/pom.xml @@ -16,24 +16,24 @@ kafka11 kafka mysql - - - - + hbase + elasticsearch5 + mongo + redis5 launcher rdb - - - + sqlserver + oracle + cassandra kudu - - + postgresql + serversocket console - - - - - + clickhouse + impala + db2 + polardb + elasticsearch6 From 966b17136fb06ad52394908c0f20dd440ebe7911 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 11 May 2020 15:06:17 +0800 Subject: [PATCH 304/523] =?UTF-8?q?=E8=A1=A5=E5=85=85redis=E7=BB=93?= =?UTF-8?q?=E6=9E=9C=E8=A1=A8=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/redisSink.md | 73 ++++++++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 14 deletions(-) diff --git a/docs/plugin/redisSink.md b/docs/plugin/redisSink.md index b43cb7377..224c26b2f 100644 --- a/docs/plugin/redisSink.md +++ b/docs/plugin/redisSink.md @@ -13,8 +13,6 @@ CREATE TABLE tableName( tableName ='tableName', parallelism ='parllNum' ); - - ``` ## 2.支持版本 @@ -50,17 +48,64 @@ redis5.0 ## 5.样例： ``` + CREATE TABLE MyTable( + name varchar, + channel varchar + )WITH( + type ='kafka10', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest01', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + CREATE TABLE MyResult( - channel varchar, - pv varchar, - PRIMARY KEY(channel) - )WITH( - type='redis', - url='172.16.10.79:6379', - password='abc123', - database='0', - redisType='1', - tableName='sinktoredis' - ); + channel VARCHAR, + pv VARCHAR + )WITH( + type ='redis', + redisType ='1', + url ='172.16.8.109:6379', + tableName ='resultTable', + partitionedJoin ='false', + parallelism ='1', + database ='0', + timeout ='10000', + maxTotal ='60000', + maxIdle='8', + minIdle='0' + ); + + insert + into + MyResult + select + channel, + name as pv + from + MyTable a + ``` + +## 6.redis完整样例 +### redis数据说明 +redis使用k-v格式存储，key的构建格式为tableName:privateKey:privateKeyValue:columnName, value=columnValue - ``` \ No newline at end of file +### 源表数据内容 +``` +{"name":"roc","channel":"daishu","age":2} +``` +### redis实际数据内容 +``` +127.0.0.1:6379> keys * +1) "resultTable:name:roc:name" +2) "resultTable:name:roc:channel" +127.0.0.1:6379> get "resultTable:name:roc:name" +"roc" +127.0.0.1:6379> get "resultTable:name:roc:channel" +"daishu" +``` \ No newline at end of file From 85aae4721ccbb27a0319fd8c1d40327a67545ef5 Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 11 May 2020 15:34:35 +0800 Subject: [PATCH 305/523] =?UTF-8?q?=E5=9B=9E=E6=BB=9A=205f30a74b=20?= =?UTF-8?q?=EF=BC=8C=E8=A7=A3=E5=86=B3=20'=E5=9B=9E=E6=BB=9A=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=9A=84test=E5=88=86=E6=94=AF=E5=8F=98=E6=9B=B4?= =?UTF-8?q?=E5=86=85=E5=AE=B9'=E6=97=A0=E6=B3=95=E6=8F=90=E4=BA=A4?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 10 + .../side/cassandra/CassandraAllReqRow.java | 10 +- .../side/cassandra/CassandraAllSideInfo.java | 10 +- .../side/cassandra/CassandraAsyncReqRow.java | 11 +- .../cassandra/CassandraAsyncSideInfo.java | 18 +- .../cassandra/table/CassandraSideParser.java | 17 +- .../table/CassandraSideTableInfo.java | 4 +- .../sink/cassandra/CassandraOutputFormat.java | 4 +- .../sql/sink/cassandra/CassandraSink.java | 4 +- .../cassandra/table/CassandraSinkParser.java | 10 +- .../cassandra/table/CassandraTableInfo.java | 4 +- ci/sonar_notify.sh | 14 + .../side/clickhouse/ClickhouseAllReqRow.java | 17 +- .../clickhouse/ClickhouseAllSideInfo.java | 4 +- .../clickhouse/ClickhouseAsyncReqRow.java | 6 +- .../clickhouse/ClickhouseAsyncSideInfo.java | 4 +- .../table/ClickhouseSideParser.java | 6 +- .../sql/sink/clickhouse/ClickhouseSink.java | 9 +- .../table/ClickhouseSinkParser.java | 6 +- .../sql/sink/console/ConsoleOutputFormat.java | 6 +- .../flink/sql/sink/console/ConsoleSink.java | 4 +- .../sink/console/table/ConsoleSinkParser.java | 10 +- .../sink/console/table/ConsoleTableInfo.java | 4 +- .../sink/console/table/TablePrintUtil.java | 23 +- core/pom.xml | 10 +- .../main/java/com/dtstack/flink/sql/Main.java | 5 +- .../sql/classloader/ClassLoaderManager.java | 4 +- .../flink/sql/constrant/ConfigConstrant.java | 13 +- .../dtstack/flink/sql/enums/ColumnType.java | 63 ++- .../flink/sql/enums/ECacheContentType.java | 9 + .../dtstack/flink/sql/enums/ECacheType.java | 13 +- .../flink/sql/enums/EDatabaseType.java | 9 + .../flink/sql/enums/EPluginLoadMode.java | 6 + .../flink/sql/enums/EStateBackend.java | 9 + .../environment/StreamEnvConfigManager.java | 59 ++- .../flink/sql/exec/ExecuteProcessHelper.java | 76 ++-- .../dtstack/flink/sql/exec/ParamsInfo.java | 1 - .../format/SerializationMetricWrapper.java | 11 +- .../DtNestRowDeserializationSchema.java | 11 +- .../flink/sql/option/OptionParser.java | 21 +- ...t.java => AbstractDtRichOutputFormat.java} | 2 +- .../flink/sql/parser/CreateFuncParser.java | 8 +- .../dtstack/flink/sql/parser/SqlParser.java | 12 +- .../com/dtstack/flink/sql/parser/SqlTree.java | 8 +- ...leInfo.java => AbstractSideTableInfo.java} | 4 +- .../{AllReqRow.java => BaseAllReqRow.java} | 18 +- ...{AsyncReqRow.java => BaseAsyncReqRow.java} | 18 +- .../side/{SideInfo.java => BaseSideInfo.java} | 22 +- .../flink/sql/side/JoinNodeDealer.java | 10 +- .../flink/sql/side/ParserJoinField.java | 2 + .../flink/sql/side/SidePredicatesParser.java | 8 +- .../dtstack/flink/sql/side/SideSQLParser.java | 7 +- .../dtstack/flink/sql/side/SideSqlExec.java | 41 +- .../flink/sql/side/StreamSideFactory.java | 10 +- ...sSideCache.java => AbstractSideCache.java} | 8 +- .../flink/sql/side/cache/LRUSideCache.java | 6 +- .../sql/side/operator/SideAsyncOperator.java | 16 +- .../operator/SideWithAllCacheOperator.java | 18 +- .../flink/sql/sink/IStreamSinkGener.java | 4 +- .../flink/sql/sink/StreamSinkFactory.java | 12 +- .../flink/sql/source/IStreamSourceGener.java | 4 +- .../flink/sql/source/StreamSourceFactory.java | 12 +- ...rser.java => AbstractSideTableParser.java} | 49 +-- ...eParser.java => AbstractSourceParser.java} | 16 +- ...Info.java => AbstractSourceTableInfo.java} | 2 +- ...{TableInfo.java => AbstractTableInfo.java} | 2 +- ...rser.java => AbstractTableInfoParser.java} | 20 +- ...leParser.java => AbstractTableParser.java} | 20 +- ...Info.java => AbstractTargetTableInfo.java} | 2 +- .../sql/table/ITableFieldDealHandler.java | 2 +- .../com/dtstack/flink/sql/util/ClassUtil.java | 3 +- .../com/dtstack/flink/sql/util/DateUtil.java | 408 ++++++++---------- .../dtstack/flink/sql/util/DtStringUtil.java | 8 +- .../com/dtstack/flink/sql/util/JDBCUtils.java | 4 +- .../com/dtstack/flink/sql/util/MathUtil.java | 2 +- .../dtstack/flink/sql/util/PluginUtil.java | 10 +- ....java => AbstractCustomerWaterMarker.java} | 4 +- .../CustomerWaterMarkerForLong.java | 2 +- .../CustomerWaterMarkerForTimeStamp.java | 3 +- .../sql/watermarker/WaterMarkerAssigner.java | 8 +- .../sql/side/SidePredicatesParserTest.java | 4 +- .../flink/sql/side/db2/Db2AllReqRow.java | 15 +- .../flink/sql/side/db2/Db2AllSideInfo.java | 4 +- .../flink/sql/side/db2/Db2AsyncReqRow.java | 6 +- .../flink/sql/side/db2/Db2AsyncSideInfo.java | 4 +- .../sql/side/db2/table/Db2SideParser.java | 6 +- .../com/dtstack/flink/sql/sink/db/DbSink.java | 9 +- .../flink/sql/sink/db/table/DbSinkParser.java | 6 +- docs/kafkaSink.md | 223 ++++++++++ docs/plugin/kafkaSource.md | 226 ++-------- docs/plugin/mongoSide.md | 5 +- docs/plugin/mongoSink.md | 7 +- .../sink/elasticsearch/ElasticsearchSink.java | 4 +- .../table/ElasticsearchSinkParser.java | 8 +- .../table/ElasticsearchTableInfo.java | 4 +- .../Elasticsearch6AllReqRow.java | 21 +- .../Elasticsearch6AllSideInfo.java | 11 +- .../Elasticsearch6AsyncReqRow.java | 11 +- .../Elasticsearch6AsyncSideInfo.java | 13 +- .../table/Elasticsearch6SideParser.java | 8 +- .../table/Elasticsearch6SideTableInfo.java | 4 +- .../sql/side/elasticsearch6/util/Es6Util.java | 10 +- .../sink/elasticsearch/ElasticsearchSink.java | 4 +- .../table/ElasticsearchSinkParser.java | 8 +- .../table/ElasticsearchTableInfo.java | 4 +- .../flink/sql/side/hbase/HbaseAllReqRow.java | 22 +- .../sql/side/hbase/HbaseAllSideInfo.java | 10 +- .../sql/side/hbase/HbaseAsyncReqRow.java | 14 +- .../sql/side/hbase/HbaseAsyncSideInfo.java | 10 +- ...ler.java => AbstractRowKeyModeDealer.java} | 12 +- .../PreRowKeyModeDealerDealer.java | 12 +- .../rowkeydealer/RowKeyEqualModeDealer.java | 9 +- ...ator.java => AbstractReplaceOperator.java} | 4 +- .../sql/side/hbase/Md5ReplaceOperator.java | 2 +- .../sql/side/hbase/enums/EReplaceOpType.java | 9 +- .../sql/side/hbase/enums/EReplaceType.java | 9 + .../sql/side/hbase/table/HbaseSideParser.java | 12 +- .../side/hbase/table/HbaseSideTableInfo.java | 4 +- .../sql/side/hbase/utils/HbaseUtils.java | 2 + .../sql/sink/hbase/HbaseOutputFormat.java | 11 +- .../flink/sql/sink/hbase/HbaseSink.java | 5 +- .../sql/sink/hbase/table/HbaseSinkParser.java | 10 +- .../sql/sink/hbase/table/HbaseTableInfo.java | 4 +- .../sql/side/impala/ImpalaAllReqRow.java | 13 +- .../sql/side/impala/ImpalaAllSideInfo.java | 7 +- .../sql/side/impala/ImpalaAsyncReqRow.java | 12 +- .../sql/side/impala/ImpalaAsyncSideInfo.java | 6 +- .../side/impala/table/ImpalaSideParser.java | 6 +- .../flink/sql/sink/impala/ImpalaSink.java | 14 +- .../sink/impala/table/ImpalaSinkParser.java | 7 +- .../kafka/AbstractKafkaProducerFactory.java | 37 +- .../sql/sink/kafka/AbstractKafkaSink.java | 148 +++++++ .../sink/kafka/CustomerFlinkPartition.java | 8 +- .../CustomerKeyedSerializationSchema.java | 35 +- .../AvroCRowSerializationSchema.java | 366 ++++++++++++++++ .../CsvCRowSerializationSchema.java | 374 ++++++++++++++++ .../JsonCRowSerializationSchema.java | 234 ++++++++++ .../sql/sink/kafka/table/KafkaSinkParser.java | 13 +- .../sink/kafka/table/KafkaSinkTableInfo.java | 44 +- .../sql/source/kafka/AbstractKafkaSource.java | 117 +++++ .../sql/source/kafka/enums/EKafkaOffset.java | 43 +- .../source/kafka/table/KafkaSourceParser.java | 42 +- .../kafka/table/KafkaSourceTableInfo.java | 181 ++++---- .../flink/sql/sink/kafka/KafkaProducer.java | 6 +- .../sql/sink/kafka/KafkaProducerFactory.java | 4 +- .../flink/sql/sink/kafka/KafkaSink.java | 118 +---- .../flink/sql/source/kafka/KafkaSource.java | 78 +--- .../flink/sql/sink/kafka/KafkaProducer09.java | 5 +- .../sink/kafka/KafkaProducer09Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 124 +----- .../flink/sql/source/kafka/KafkaSource.java | 76 +--- .../sql/sink/kafka/KafkaProducer010.java | 6 +- .../sink/kafka/KafkaProducer010Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 122 +----- .../flink/sql/source/kafka/KafkaSource.java | 89 +--- .../sql/sink/kafka/KafkaProducer011.java | 6 +- .../sink/kafka/KafkaProducer011Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 121 +----- .../source/kafka/KafkaConsumer011Factory.java | 2 +- .../flink/sql/source/kafka/KafkaSource.java | 92 +--- .../flink/sql/side/kudu/KuduAllReqRow.java | 24 +- .../flink/sql/side/kudu/KuduAllSideInfo.java | 10 +- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 26 +- .../sql/side/kudu/KuduAsyncSideInfo.java | 10 +- .../sql/side/kudu/table/KuduSideParser.java | 11 +- .../side/kudu/table/KuduSideTableInfo.java | 4 +- .../flink/sql/side/kudu/utils/KuduUtil.java | 1 + .../flink/sql/sink/kudu/KuduOutputFormat.java | 8 +- .../dtstack/flink/sql/sink/kudu/KuduSink.java | 4 +- .../sql/sink/kudu/table/KuduSinkParser.java | 11 +- .../sql/sink/kudu/table/KuduTableInfo.java | 4 +- .../sql/launcher/ClusterClientFactory.java | 8 +- .../perjob/PerJobClusterClientBuilder.java | 3 +- .../sql/launcher/perjob/PerJobSubmitter.java | 1 - .../flink/sql/side/mongo/MongoAllReqRow.java | 62 +-- .../sql/side/mongo/MongoAllSideInfo.java | 10 +- .../sql/side/mongo/MongoAsyncReqRow.java | 83 ++-- .../sql/side/mongo/MongoAsyncSideInfo.java | 13 +- .../sql/side/mongo/table/MongoSideParser.java | 12 +- .../side/mongo/table/MongoSideTableInfo.java | 4 +- .../flink/sql/side/mongo/utils/MongoUtil.java | 1 + .../sql/sink/mongo/MongoOutputFormat.java | 44 +- .../flink/sql/sink/mongo/MongoSink.java | 4 +- .../sql/sink/mongo/table/MongoSinkParser.java | 10 +- .../sql/sink/mongo/table/MongoTableInfo.java | 4 +- .../test/java/com/dtstack/flinkx/AppTest.java | 58 --- .../flink/sql/side/mysql/MysqlAllReqRow.java | 12 +- .../sql/side/mysql/MysqlAllSideInfo.java | 4 +- .../sql/side/mysql/MysqlAsyncReqRow.java | 11 +- .../sql/side/mysql/MysqlAsyncSideInfo.java | 4 +- .../sql/side/mysql/table/MysqlSideParser.java | 6 +- .../flink/sql/sink/mysql/MysqlSink.java | 6 +- .../sql/sink/mysql/table/MysqlSinkParser.java | 6 +- .../test/java/com/dtstack/flinkx/AppTest.java | 58 --- .../sql/side/oracle/OracleAllReqRow.java | 12 +- .../sql/side/oracle/OracleAllSideInfo.java | 5 +- .../sql/side/oracle/OracleAsyncReqRow.java | 11 +- .../sql/side/oracle/OracleAsyncSideInfo.java | 9 +- .../side/oracle/table/OracleSideParser.java | 6 +- .../flink/sql/sink/oracle/OracleDialect.java | 8 +- .../flink/sql/sink/oracle/OracleSink.java | 6 +- .../sink/oracle/table/OracleSinkParser.java | 6 +- .../sql/side/polardb/PolardbAllReqRow.java | 34 +- .../sql/side/polardb/PolardbAllSideInfo.java | 26 +- .../sql/side/polardb/PolardbAsyncReqRow.java | 28 +- .../side/polardb/PolardbAsyncSideInfo.java | 26 +- .../side/polardb/table/PolardbSideParser.java | 29 +- .../flink/sql/sink/polardb/PolardbSink.java | 30 +- .../sink/polardb/table/PolardbSinkParser.java | 28 +- pom.xml | 16 +- .../side/postgresql/PostgresqlAllReqRow.java | 12 +- .../postgresql/PostgresqlAllSideInfo.java | 4 +- .../postgresql/PostgresqlAsyncReqRow.java | 6 +- .../postgresql/PostgresqlAsyncSideInfo.java | 4 +- .../table/PostgresqlSideParser.java | 6 +- .../sql/sink/postgresql/PostgresqlSink.java | 6 +- .../table/PostgresqlSinkParser.java | 6 +- ...lReqRow.java => AbstractRdbAllReqRow.java} | 12 +- .../sql/side/rdb/all/RdbAllSideInfo.java | 15 +- .../sql/side/rdb/async/RdbAsyncReqRow.java | 28 +- .../sql/side/rdb/async/RdbAsyncSideInfo.java | 16 +- .../provider/DTC3P0DataSourceProvider.java | 10 +- .../sql/side/rdb/table/RdbSideParser.java | 8 +- .../sql/side/rdb/table/RdbSideTableInfo.java | 4 +- .../flink/sql/side/rdb/util/SwitchUtil.java | 1 + .../{RdbSink.java => AbstractRdbSink.java} | 19 +- .../flink/sql/sink/rdb/JDBCOptions.java | 22 +- .../rdb/format/AbstractJDBCOutputFormat.java | 14 +- .../rdb/format/JDBCUpsertOutputFormat.java | 14 +- .../rdb/format/RetractJDBCOutputFormat.java | 0 .../sql/sink/rdb/table/RdbSinkParser.java | 9 +- .../sql/sink/rdb/table/RdbTableInfo.java | 6 +- ...tWriter.java => AbstractUpsertWriter.java} | 86 ++-- .../sql/sink/rdb/writer/AppendOnlyWriter.java | 13 +- .../flink/sql/side/redis/RedisAllReqRow.java | 32 +- .../sql/side/redis/RedisAllSideInfo.java | 14 +- .../sql/side/redis/RedisAsyncReqRow.java | 26 +- .../sql/side/redis/RedisAsyncSideInfo.java | 14 +- .../sql/side/redis/table/RedisSideParser.java | 11 +- .../sql/side/redis/table/RedisSideReqRow.java | 8 +- .../side/redis/table/RedisSideTableInfo.java | 8 +- .../sql/sink/redis/RedisOutputFormat.java | 29 +- .../flink/sql/sink/redis/RedisSink.java | 8 +- .../sql/sink/redis/table/RedisSinkParser.java | 12 +- .../sql/sink/redis/table/RedisTableInfo.java | 8 +- .../CustomerSocketTextStreamFunction.java | 6 +- .../serversocket/ServersocketSource.java | 4 +- .../table/ServersocketSourceParser.java | 8 +- .../table/ServersocketSourceTableInfo.java | 6 +- .../side/sqlserver/SqlserverAllReqRow.java | 20 +- .../side/sqlserver/SqlserverAllSideInfo.java | 10 +- .../side/sqlserver/SqlserverAsyncReqRow.java | 10 +- .../sqlserver/SqlserverAsyncSideInfo.java | 11 +- .../sqlserver/table/SqlserverSideParser.java | 12 +- .../sql/sink/sqlserver/SqlserverSink.java | 6 +- .../sqlserver/table/SqlserverSinkParser.java | 6 +- 256 files changed, 3487 insertions(+), 2621 deletions(-) create mode 100644 .gitlab-ci.yml create mode 100644 ci/sonar_notify.sh rename core/src/main/java/com/dtstack/flink/sql/outputformat/{DtRichOutputFormat.java => AbstractDtRichOutputFormat.java} (95%) rename core/src/main/java/com/dtstack/flink/sql/side/{SideTableInfo.java => AbstractSideTableInfo.java} (97%) rename core/src/main/java/com/dtstack/flink/sql/side/{AllReqRow.java => BaseAllReqRow.java} (79%) rename core/src/main/java/com/dtstack/flink/sql/side/{AsyncReqRow.java => BaseAsyncReqRow.java} (89%) rename core/src/main/java/com/dtstack/flink/sql/side/{SideInfo.java => BaseSideInfo.java} (92%) rename core/src/main/java/com/dtstack/flink/sql/side/cache/{AbsSideCache.java => AbstractSideCache.java} (84%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbsSideTableParser.java => AbstractSideTableParser.java} (68%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbsSourceParser.java => AbstractSourceParser.java} (81%) rename core/src/main/java/com/dtstack/flink/sql/table/{SourceTableInfo.java => AbstractSourceTableInfo.java} (97%) rename core/src/main/java/com/dtstack/flink/sql/table/{TableInfo.java => AbstractTableInfo.java} (98%) rename core/src/main/java/com/dtstack/flink/sql/table/{TableInfoParser.java => AbstractTableInfoParser.java} (83%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbsTableParser.java => AbstractTableParser.java} (88%) rename core/src/main/java/com/dtstack/flink/sql/table/{TargetTableInfo.java => AbstractTargetTableInfo.java} (94%) rename core/src/main/java/com/dtstack/flink/sql/watermarker/{AbsCustomerWaterMarker.java => AbstractCustomerWaterMarker.java} (94%) create mode 100644 docs/kafkaSink.md rename hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/{AbsRowKeyModeDealer.java => AbstractRowKeyModeDealer.java} (87%) rename hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/{ReplaceOperator.java => AbstractReplaceOperator.java} (93%) create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java create mode 100644 kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java rename cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java => kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java (56%) delete mode 100644 mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java delete mode 100644 mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java rename rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/{RdbAllReqRow.java => AbstractRdbAllReqRow.java} (96%) rename rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/{RdbSink.java => AbstractRdbSink.java} (90%) create mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java rename rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/{UpsertWriter.java => AbstractUpsertWriter.java} (84%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..2952551ff --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,10 @@ +build: + stage: test + script: + - mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q + - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.branch.name="v1.8.0_dev" -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar + - sh ci/sonar_notify.sh + only: + - v1.8.0_dev + tags: + - dt-insight-engine \ No newline at end of file diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index bc0a0f8cd..3ad8b934e 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -34,10 +34,10 @@ import com.datastax.driver.core.SocketOptions; import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -62,14 +62,12 @@ * * @author xuqianjin */ -public class CassandraAllReqRow extends AllReqRow { +public class CassandraAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 54015343561288219L; private static final Logger LOG = LoggerFactory.getLogger(CassandraAllReqRow.class); - private static final String cassandra_DRIVER = "com.cassandra.jdbc.Driver"; - private static final int CONN_RETRY_NUM = 3; private static final int FETCH_SIZE = 1000; @@ -79,7 +77,7 @@ public class CassandraAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public CassandraAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public CassandraAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new com.dtstack.flink.sql.side.cassandra.CassandraAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java index fa665f9a0..5d95dc9bf 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; @@ -37,16 +37,16 @@ * * @author xuqianjin */ -public class CassandraAllSideInfo extends SideInfo { +public class CassandraAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = -8690814317653033557L; - public CassandraAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public CassandraAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { CassandraSideTableInfo cassandraSideTableInfo = (CassandraSideTableInfo) sideTableInfo; sqlCondition = "select ${selectField} from ${tableName} "; diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index 0173f2d6f..7eee3cb5f 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -37,11 +37,11 @@ import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.google.common.base.Function; @@ -67,7 +67,7 @@ * * @author xuqianjin */ -public class CassandraAsyncReqRow extends AsyncReqRow { +public class CassandraAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 6631584128079864735L; @@ -83,7 +83,7 @@ public class CassandraAsyncReqRow extends AsyncReqRow { private transient ListenableFuture session; private transient CassandraSideTableInfo cassandraSideTableInfo; - public CassandraAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public CassandraAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new com.dtstack.flink.sql.side.cassandra.CassandraAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -216,7 +216,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce connCassandraDB(cassandraSideTableInfo); String sqlCondition = sideInfo.getSqlCondition() + " " + sqlWhere + " ALLOW FILTERING "; - System.out.println("sqlCondition:" + sqlCondition); + LOG.info("sqlCondition:{}" + sqlCondition); ListenableFuture resultSet = Futures.transformAsync(session, new AsyncFunction() { @@ -265,7 +265,6 @@ public void onSuccess(List rows) { public void onFailure(Throwable t) { LOG.error("Failed to retrieve the data: %s%n", t.getMessage()); - System.out.println("Failed to retrieve the data: " + t.getMessage()); cluster.closeAsync(); resultFuture.completeExceptionally(t); } diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java index 3557f0f73..4bee5648b 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlBasicCall; @@ -30,6 +30,8 @@ import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; @@ -39,16 +41,18 @@ * * @author xuqianjin */ -public class CassandraAsyncSideInfo extends SideInfo { +public class CassandraAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = -4403313049809013362L; + private static final Logger LOG = LoggerFactory.getLogger(CassandraAsyncSideInfo.class.getSimpleName()); - public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + + public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { CassandraSideTableInfo cassandraSideTableInfo = (CassandraSideTableInfo) sideTableInfo; String sideTableName = joinInfo.getSideTableName(); @@ -63,9 +67,9 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { } sqlCondition = "select ${selectField} from ${tableName}"; - sqlCondition = sqlCondition.replace("${tableName}", cassandraSideTableInfo.getDatabase()+"."+cassandraSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields); - System.out.println("---------side_exe_sql-----\n" + sqlCondition); + + LOG.info("---------side_exe_sql-----\n{}" + sqlCondition); } diff --git a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java index 62dd753b9..e893e56b0 100644 --- a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java +++ b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java @@ -19,18 +19,16 @@ package com.dtstack.flink.sql.side.cassandra.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; -import java.math.BigDecimal; -import java.sql.Date; import java.sql.Timestamp; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -38,7 +36,7 @@ * * @author xuqianjin */ -public class CassandraSideParser extends AbsSideTableParser { +public class CassandraSideParser extends AbstractSideTableParser { private final static String SIDE_SIGN_KEY = "sideSignKey"; @@ -73,7 +71,7 @@ public CassandraSideParser() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo cassandraSideTableInfo = new com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo(); cassandraSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, cassandraSideTableInfo); @@ -96,9 +94,10 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map { +public class CassandraOutputFormat extends AbstractDtRichOutputFormat { private static final long serialVersionUID = -7994311331389155692L; private static final Logger LOG = LoggerFactory.getLogger(CassandraOutputFormat.class); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java index eb7b23b53..26152a7d3 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.cassandra.table.CassandraTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -63,7 +63,7 @@ public CassandraSink() { } @Override - public CassandraSink genStreamSink(TargetTableInfo targetTableInfo) { + public CassandraSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { CassandraTableInfo cassandraTableInfo = (CassandraTableInfo) targetTableInfo; this.address = cassandraTableInfo.getAddress(); this.tableName = cassandraTableInfo.getTableName(); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java index 4c68e71ae..9ef8639ba 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java @@ -19,13 +19,13 @@ package com.dtstack.flink.sql.sink.cassandra.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -33,7 +33,7 @@ * * @author xuqianjin */ -public class CassandraSinkParser extends AbsTableParser { +public class CassandraSinkParser extends AbstractTableParser { public static final String ADDRESS_KEY = "address"; @@ -60,7 +60,7 @@ public class CassandraSinkParser extends AbsTableParser { public static final String POOL_TIMEOUT_MILLIS_KEY = "poolTimeoutMillis"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { CassandraTableInfo cassandraTableInfo = new CassandraTableInfo(); cassandraTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, cassandraTableInfo); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java index c6626c42a..ffb5fa876 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.cassandra.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; /** @@ -28,7 +28,7 @@ * * @author xuqianjin */ -public class CassandraTableInfo extends TargetTableInfo { +public class CassandraTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "cassandra"; diff --git a/ci/sonar_notify.sh b/ci/sonar_notify.sh new file mode 100644 index 000000000..41f8a3c0e --- /dev/null +++ b/ci/sonar_notify.sh @@ -0,0 +1,14 @@ +#!/bin/bash +#参考钉钉文档 https://open-doc.dingtalk.com/microapp/serverapi2/qf2nxq + sonarreport=$(curl -s http://172.16.100.198:8082/?projectname=dt-insight-engine/flinkStreamSQL) + curl -s "https://oapi.dingtalk.com/robot/send?access_token=71555061297a53d3ac922a6f4d94285d8e23bccdca0c00b4dc6df0a2d49da724" \ + -H "Content-Type: application/json" \ + -d "{ + \"msgtype\": \"markdown\", + \"markdown\": { + \"title\":\"sonar代码质量\", + \"text\": \"## sonar代码质量报告: \n +> [sonar地址](http://172.16.100.198:9000/dashboard?id=dt-insight-engine/flinkStreamSQL) \n +> ${sonarreport} \n\" + } + }" \ No newline at end of file diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java index 68c0c7984..c9a0c447b 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java @@ -20,40 +20,37 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; -import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Connection; import java.sql.DriverManager; import java.util.List; -import java.util.Map; -public class ClickhouseAllReqRow extends RdbAllReqRow { +public class ClickhouseAllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(ClickhouseAllReqRow.class); private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - public ClickhouseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ClickhouseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String passWord) { + public Connection getConn(String dbUrl, String userName, String passWord) { try { Connection connection ; JDBCUtils.forName(CLICKHOUSE_DRIVER, getClass().getClassLoader()); // ClickHouseProperties contains all properties if (userName == null) { - connection = DriverManager.getConnection(dbURL); + connection = DriverManager.getConnection(dbUrl); } else { - connection = DriverManager.getConnection(dbURL, userName, passWord); + connection = DriverManager.getConnection(dbUrl, userName, passWord); } return connection; } catch (Exception e) { diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java index 973c069b9..43fbeaa56 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -28,7 +28,7 @@ public class ClickhouseAllSideInfo extends RdbAllSideInfo { - public ClickhouseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index db49e4337..25478c0dd 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -37,7 +37,7 @@ public class ClickhouseAsyncReqRow extends RdbAsyncReqRow { private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - public ClickhouseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ClickhouseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -62,7 +62,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); } } diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java index 254561de0..eec5fbe74 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -29,7 +29,7 @@ public class ClickhouseAsyncSideInfo extends RdbAsyncSideInfo { - public ClickhouseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java b/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java index 7be387fd8..ab285c37f 100644 --- a/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java +++ b/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.clickhouse.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import ru.yandex.clickhouse.domain.ClickHouseDataType; import java.util.Map; @@ -38,8 +38,8 @@ public class ClickhouseSideParser extends RdbSideParser { private static final String CURR_TYPE = "clickhouse"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); clickhouseTableInfo.setType(CURR_TYPE); return clickhouseTableInfo; } diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java index bbb6a6fe4..5f7e41f7b 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java @@ -22,14 +22,11 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import java.util.List; -import java.util.Map; - -public class ClickhouseSink extends RdbSink implements IStreamSinkGener { +public class ClickhouseSink extends AbstractRdbSink implements IStreamSinkGener { public ClickhouseSink() { super(new ClickhouseDialect()); } @@ -37,7 +34,7 @@ public ClickhouseSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java index 8c3df93d7..5b0f2598f 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.clickhouse.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import ru.yandex.clickhouse.domain.ClickHouseDataType; import java.util.Map; @@ -30,8 +30,8 @@ public class ClickhouseSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "clickhouse"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); clickhouseTableInfo.setType(CURR_TYPE); return clickhouseTableInfo; } diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java index a397036ef..fc5a768c4 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.console; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.sink.console.table.TablePrintUtil; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -37,7 +37,7 @@ * * @author xuqianjin */ -public class ConsoleOutputFormat extends DtRichOutputFormat { +public class ConsoleOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(ConsoleOutputFormat.class); @@ -56,7 +56,7 @@ public void open(int taskNumber, int numTasks) throws IOException { @Override public void writeRecord(Tuple2 tuple2) throws IOException { - System.out.println("received oriainal data:" + tuple2); + LOG.info("received oriainal data:{}" + tuple2); Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); if (!retract) { diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java index 77a3efea2..6cfdd2d3d 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.console; import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -80,7 +80,7 @@ public void emitDataStream(DataStream> dataStream) { } @Override - public ConsoleSink genStreamSink(TargetTableInfo targetTableInfo) { + public ConsoleSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { return this; } } diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java index e77444bfd..93ed02420 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java @@ -18,13 +18,13 @@ package com.dtstack.flink.sql.sink.console.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -32,9 +32,9 @@ * * @author xuqianjin */ -public class ConsoleSinkParser extends AbsTableParser { +public class ConsoleSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ConsoleTableInfo consoleTableInfo = new ConsoleTableInfo(); consoleTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, consoleTableInfo); diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java index 4b286c667..6fd7063c6 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.console.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; /** * Reason: @@ -26,7 +26,7 @@ * * @author xuqianjin */ -public class ConsoleTableInfo extends TargetTableInfo { +public class ConsoleTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "console"; diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java index 8813da619..12bb2a5d4 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java @@ -22,6 +22,7 @@ public class TablePrintUtil { public static final int ALIGN_LEFT = 1;//左对齐 public static final int ALIGN_RIGHT = 2;//右对齐 public static final int ALIGN_CENTER = 3;//居中对齐 + private static final Pattern PATTERN = Pattern.compile("[\u4e00-\u9fa5]"); private int align = ALIGN_CENTER;//默认居中对齐 private boolean equilong = false;//默认不等宽 @@ -58,7 +59,9 @@ public static TablePrintUtil build(String[][] data) { public static TablePrintUtil build(List data) { TablePrintUtil self = new TablePrintUtil(); self.data = new ArrayList<>(); - if (data.size() <= 0) throw new RuntimeException("数据源至少得有一行吧"); + if (data.size() <= 0) { + throw new RuntimeException("数据源至少得有一行吧"); + } Object obj = data.get(0); @@ -70,7 +73,9 @@ public static TablePrintUtil build(List data) { int length = ((List) obj).size(); for (Object item : data) { List col = (List) item; - if (col.size() != length) throw new RuntimeException("数据源每列长度必须一致"); + if (col.size() != length) { + throw new RuntimeException("数据源每列长度必须一致"); + } self.data.add(col.toArray(new String[length])); } } else { @@ -90,7 +95,7 @@ public static TablePrintUtil build(List data) { try { value = obj.getClass().getMethod(colList.get(j).getMethodName).invoke(data.get(i)).toString(); } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { - e.printStackTrace(); + LOG.error("", e); } item[j] = value == null ? "null" : value; } @@ -115,7 +120,7 @@ private static List

getColList(Object obj) { Method[] methods = obj.getClass().getMethods(); for (Method m : methods) { StringBuilder getMethodName = new StringBuilder(m.getName()); - if (getMethodName.substring(0, 3).equals("get") && !m.getName().equals("getClass")) { + if ("get".equals(getMethodName.substring(0, 3)) && !"getClass".equals(m.getName())) { Col col = new Col(); col.getMethodName = getMethodName.toString(); char first = Character.toLowerCase(getMethodName.delete(0, 3).charAt(0)); @@ -134,8 +139,8 @@ private static ListgetColList(Object obj) { * @return */ private int getStringCharLength(String str) { - Pattern p = Pattern.compile("[\u4e00-\u9fa5]");//利用正则找到中文 - Matcher m = p.matcher(str); + //利用正则找到中文 + Matcher m = PATTERN.matcher(str); int count = 0; while (m.find()) { count++; @@ -163,7 +168,9 @@ private int[] getColLengths() { if (equilong) {//如果等宽表格 int max = 0; for (int len : result) { - if (len > max) max = len; + if (len > max) { + max = len; + } } for (int i = 0; i < result.length; i++) { result[i] = max; @@ -213,6 +220,8 @@ public String getTableString() { sb.append(cell); for (int i = 0; i < right + padding; i++) {sb.append(s);} break; + default: + break; } sb.append(v); } diff --git a/core/pom.xml b/core/pom.xml index b17209612..eb470a3c0 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -20,7 +20,7 @@ 1.16.0 2.7.9 19.0 - 1.1.7 + 1.0.0-SNAPSHOT @@ -122,13 +122,11 @@ junit 4.12 - - ch.qos.logback - logback-classic - ${logback.version} + com.aiweiergou + tools-logger + ${logger.tool.version} - diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index b54c3feb2..fe925f9a6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -21,14 +21,12 @@ -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; - import com.dtstack.flink.sql.exec.ExecuteProcessHelper; import com.dtstack.flink.sql.exec.ParamsInfo; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * Date: 2018/6/26 * Company: www.dtstack.com @@ -40,6 +38,7 @@ public class Main { public static void main(String[] args) throws Exception { ParamsInfo paramsInfo = ExecuteProcessHelper.parseParams(args); + ExecuteProcessHelper.setLogLevel(paramsInfo); StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExecution(paramsInfo); env.execute(paramsInfo.getName()); LOG.info("program {} execution success", paramsInfo.getName()); diff --git a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java index c03f60617..2e62e11ab 100644 --- a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java @@ -98,9 +98,9 @@ public static List getClassPath() { - public static URLClassLoader loadExtraJar(List jarURLList, URLClassLoader classLoader) + public static URLClassLoader loadExtraJar(List jarUrlList, URLClassLoader classLoader) throws IllegalAccessException, InvocationTargetException { - for(URL url : jarURLList){ + for(URL url : jarUrlList){ if(url.toString().endsWith(".jar")){ urlClassLoaderAddUrl(classLoader, url); } diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java index 00d1ea0a5..13e1de8c8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java @@ -63,14 +63,21 @@ public class ConfigConstrant { public static final String CHECKPOINTS_DIRECTORY_KEY = "state.checkpoints.dir"; public static final String STATE_BACKEND_INCREMENTAL_KEY = "state.backend.incremental"; + public static final String RESTOREENABLE = "restore.enable"; + public static final String LOG_LEVEL_KEY = "logLevel"; // restart plocy - public static final int failureRate = 3; + public static final int FAILUEE_RATE = 3; + + public static final int FAILUEE_INTERVAL = 6; //min + + public static final int DELAY_INTERVAL = 10; //sec + + public static final String FAILUREINTERVAL = "failure.interval"; //min - public static final int failureInterval = 6; //min + public static final String DELAYINTERVAL= "delay.interval"; //sec - public static final int delayInterval = 10; //sec } diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java index 749bbc907..7f3f0019c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java @@ -46,11 +46,66 @@ * @author huyifan.zju@163.com */ public enum ColumnType { - STRING, VARCHAR, CHAR, - INT, MEDIUMINT, TINYINT, DATETIME, SMALLINT, BIGINT, - DOUBLE, FLOAT, + /* + * string + */ + STRING, + /** + * varchar + */ + VARCHAR, + /** + * char + */ + CHAR, + /** + * int + */ + INT, + /** + * mediumint + */ + MEDIUMINT, + /** + * tinyint + */ + TINYINT, + /** + * datetime + */ + DATETIME, + /** + * smallint + */ + SMALLINT, + /** + * bigint + */ + BIGINT, + /** + * double + */ + DOUBLE, + /** + * float + */ + FLOAT, + /** + * boolean + */ BOOLEAN, - DATE, TIMESTAMP, DECIMAL; + /** + * date + */ + DATE, + /** + * timestamp + */ + TIMESTAMP, + /** + * decimal + */ + DECIMAL; public static ColumnType fromString(String type) { if(type == null) { diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java index 66160d820..49e352757 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java @@ -30,8 +30,17 @@ public enum ECacheContentType { + /** + * 无 + */ MissVal(0), + /** + * 1行 + */ SingleLine(1), + /** + * 多行 + */ MultiLine(2); int type; diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java index 582148c2c..9d5bb5d11 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java @@ -27,7 +27,18 @@ * @author xuchao */ public enum ECacheType { - NONE, LRU, ALL; + /** + * none + */ + NONE, + /** + * lru + */ + LRU, + /** + * all + */ + ALL; public static boolean isValid(String type){ for(ECacheType tmpType : ECacheType.values()){ diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java b/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java index 9b01bf052..7d2235f2e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java @@ -26,8 +26,17 @@ */ public enum EDatabaseType { + /** + * mysql + */ MYSQL, + /** + * sqlserver + */ SQLSERVER, + /** + * oracle + */ ORACLE, } diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java b/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java index 6cb027ac3..439966dd2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java @@ -26,7 +26,13 @@ */ public enum EPluginLoadMode { + /** + * 0:classpath + */ CLASSPATH(0), + /** + * 1:shipfile + */ SHIPFILE(1); private int type; diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java b/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java index a8f926175..098cb57fe 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java @@ -25,8 +25,17 @@ * @author maqi */ public enum EStateBackend { + /** + * memory + */ MEMORY, + /** + * rockdb + */ ROCKSDB, + /** + * filesystem + */ FILESYSTEM; public static EStateBackend convertFromString(String type) { diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index ce763ea99..d7b772c6e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -18,6 +18,12 @@ package com.dtstack.flink.sql.environment; +import com.dtstack.flink.sql.constrant.ConfigConstrant; +import com.dtstack.flink.sql.enums.EStateBackend; +import com.dtstack.flink.sql.util.MathUtil; +import com.dtstack.flink.sql.util.PropertiesUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.time.Time; @@ -34,13 +40,6 @@ import org.apache.flink.table.api.StreamQueryConfig; import org.apache.flink.table.api.java.StreamTableEnvironment; -import com.dtstack.flink.sql.constrant.ConfigConstrant; -import com.dtstack.flink.sql.enums.EStateBackend; -import com.dtstack.flink.sql.util.MathUtil; -import com.dtstack.flink.sql.util.PropertiesUtils; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; - import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -102,11 +101,15 @@ public static void streamExecutionEnvironmentConfig(StreamExecutionEnvironment s } }); - streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( - ConfigConstrant.failureRate, - Time.of(ConfigConstrant.failureInterval, TimeUnit.MINUTES), - Time.of(ConfigConstrant.delayInterval, TimeUnit.SECONDS) - )); + if(isRestore(confProperties).get()){ + streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( + ConfigConstrant.FAILUEE_RATE, + Time.of(getFailureInterval(confProperties).get(), TimeUnit.MINUTES), + Time.of(getDelayInterval(confProperties).get(), TimeUnit.SECONDS) + )); + } else { + streamEnv.setRestartStrategy(RestartStrategies.noRestart()); + } // checkpoint config Optional checkpointingEnabled = isCheckpointingEnabled(confProperties); @@ -164,6 +167,20 @@ public static Optional getAutoWatermarkInterval(Properties properties) { return StringUtils.isNotBlank(autoWatermarkInterval) ? Optional.of(Long.valueOf(autoWatermarkInterval)) : Optional.empty(); } + public static Optional isRestore(Properties properties){ + String restoreEnable = properties.getProperty(ConfigConstrant.RESTOREENABLE, "true"); + return Optional.of(Boolean.valueOf(restoreEnable)); + } + + public static Optional getDelayInterval(Properties properties){ + String delayInterval = properties.getProperty(ConfigConstrant.DELAYINTERVAL, "10"); + return Optional.of(Integer.valueOf(delayInterval)); + } + public static Optional getFailureInterval(Properties properties){ + String failureInterval = properties.getProperty(ConfigConstrant.FAILUREINTERVAL, "6"); + return Optional.of(Integer.valueOf(failureInterval)); + } + /** * #ProcessingTime(默认), IngestionTime, EventTime * @param properties @@ -259,6 +276,8 @@ private static Optional createStateBackend(String backendType, Str checkpointDataUriEmptyCheck(checkpointDataUri, backendType); stateBackend = new RocksDBStateBackend(checkpointDataUri, BooleanUtils.toBoolean(backendIncremental)); break; + default: + break; } return stateBackend == null ? Optional.empty() : Optional.of(stateBackend); } @@ -318,14 +337,14 @@ private static void verityTtl(String ttlMintimeStr, String ttlMaxtimeStr) { * @return */ private static Long getTtlTime(Integer timeNumber, String timeUnit) { - if (timeUnit.equalsIgnoreCase("d")) { - return timeNumber * 1000l * 60 * 60 * 24; - } else if (timeUnit.equalsIgnoreCase("h")) { - return timeNumber * 1000l * 60 * 60; - } else if (timeUnit.equalsIgnoreCase("m")) { - return timeNumber * 1000l * 60; - } else if (timeUnit.equalsIgnoreCase("s")) { - return timeNumber * 1000l; + if ("d".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L * 60 * 60 * 24; + } else if ("h".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L * 60 * 60; + } else if ("m".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L * 60; + } else if ("s".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L; } else { throw new RuntimeException("not support " + timeNumber + timeUnit); } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 3c9f37811..3858dc5c5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -18,7 +18,13 @@ package com.dtstack.flink.sql.exec; -import com.dtstack.flink.sql.parser.*; +import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; +import com.dtstack.flink.sql.parser.CreateFuncParser; +import com.dtstack.flink.sql.parser.CreateTmpTableParser; +import com.dtstack.flink.sql.parser.FlinkPlanner; +import com.dtstack.flink.sql.parser.InsertSqlParser; +import com.dtstack.flink.sql.parser.SqlParser; +import com.dtstack.flink.sql.parser.SqlTree; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -33,6 +39,7 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.dtstack.flink.sql.constrant.ConfigConstrant; import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.enums.EPluginLoadMode; @@ -42,12 +49,12 @@ import com.dtstack.flink.sql.option.OptionParser; import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.side.SideSqlExec; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.sink.StreamSinkFactory; import com.dtstack.flink.sql.source.StreamSourceFactory; -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import com.dtstack.flink.sql.watermarker.WaterMarkerAssigner; @@ -90,11 +97,8 @@ public class ExecuteProcessHelper { public static ParamsInfo parseParams(String[] args) throws Exception { LOG.info("------------program params-------------------------"); - System.out.println("------------program params-------------------------"); Arrays.stream(args).forEach(arg -> LOG.info("{}", arg)); - Arrays.stream(args).forEach(System.out::println); LOG.info("-------------------------------------------"); - System.out.println("----------------------------------------"); OptionParser optionParser = new OptionParser(args); Options options = optionParser.getOptions(); @@ -105,13 +109,14 @@ public static ParamsInfo parseParams(String[] args) throws Exception { String remoteSqlPluginPath = options.getRemoteSqlPluginPath(); String pluginLoadMode = options.getPluginLoadMode(); String deployMode = options.getMode(); + String logLevel = options.getLogLevel(); Preconditions.checkArgument(checkRemoteSqlPluginPath(remoteSqlPluginPath, deployMode, pluginLoadMode), "Non-local mode or shipfile deployment mode, remoteSqlPluginPath is required"); String confProp = URLDecoder.decode(options.getConfProp(), Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - List jarURList = getExternalJarUrls(options.getAddjar()); + List jarUrlList = getExternalJarUrls(options.getAddjar()); return ParamsInfo.builder() .setSql(sql) @@ -121,7 +126,7 @@ public static ParamsInfo parseParams(String[] args) throws Exception { .setPluginLoadMode(pluginLoadMode) .setDeployMode(deployMode) .setConfProp(confProperties) - .setJarUrlList(jarURList) + .setJarUrlList(jarUrlList) .build(); } @@ -152,7 +157,7 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf SqlParser.setLocalSqlPluginRoot(paramsInfo.getLocalSqlPluginPath()); SqlTree sqlTree = SqlParser.parseSql(paramsInfo.getSql()); - Map sideTableMap = Maps.newHashMap(); + Map sideTableMap = Maps.newHashMap(); Map registerTableCache = Maps.newHashMap(); //register udf @@ -185,14 +190,14 @@ public static List getExternalJarUrls(String addJarListStr) throws java.io. } return jarUrlList; } - + private static void sqlTranslation(String localSqlPluginPath, - StreamTableEnvironment tableEnv, - SqlTree sqlTree,Map sideTableMap, - Map registerTableCache, - StreamQueryConfig queryConfig) throws Exception { + StreamTableEnvironment tableEnv, + SqlTree sqlTree,Map sideTableMap, + Map registerTableCache, + StreamQueryConfig queryConfig) throws Exception { - SideSqlExec sideSqlExec = new SideSqlExec(); + SideSqlExec sideSqlExec = new SideSqlExec(); sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, result); @@ -224,9 +229,8 @@ private static void sqlTranslation(String localSqlPluginPath, //sql-dimensional table contains the dimension table of execution sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null); } else { - System.out.println("----------exec sql without dimension join-----------"); - System.out.println("----------real sql exec is--------------------------"); - System.out.println(result.getExecSql()); + LOG.info("----------exec sql without dimension join-----------"); + LOG.info("----------real sql exec is--------------------------\n{}", result.getExecSql()); FlinkSQLExec.sqlUpdate(tableEnv, result.getExecSql(), queryConfig); if (LOG.isInfoEnabled()) { LOG.info("exec sql: " + result.getExecSql()); @@ -266,14 +270,14 @@ public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrl * @throws Exception */ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { + String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { Set pluginClassPatshSets = Sets.newHashSet(); WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); - for (TableInfo tableInfo : sqlTree.getTableInfoMap().values()) { + for (AbstractTableInfo tableInfo : sqlTree.getTableInfoMap().values()) { - if (tableInfo instanceof SourceTableInfo) { + if (tableInfo instanceof AbstractSourceTableInfo) { - SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; + AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; Table table = StreamSourceFactory.getStreamSource(sourceTableInfo, env, tableEnv, localSqlPluginPath); tableEnv.registerTable(sourceTableInfo.getAdaptName(), table); //Note --- parameter conversion function can not be used inside a function of the type of polymerization @@ -304,21 +308,21 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment } registerTableCache.put(tableInfo.getName(), regTable); - URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), SourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractSourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sourceTablePathUrl); - } else if (tableInfo instanceof TargetTableInfo) { + } else if (tableInfo instanceof AbstractTargetTableInfo) { - TableSink tableSink = StreamSinkFactory.getTableSink((TargetTableInfo) tableInfo, localSqlPluginPath); + TableSink tableSink = StreamSinkFactory.getTableSink((AbstractTargetTableInfo) tableInfo, localSqlPluginPath); TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); - URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), TargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sinkTablePathUrl); - } else if (tableInfo instanceof SideTableInfo) { - String sideOperator = ECacheType.ALL.name().equals(((SideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; - sideTableMap.put(tableInfo.getName(), (SideTableInfo) tableInfo); + } else if (tableInfo instanceof AbstractSideTableInfo) { + String sideOperator = ECacheType.ALL.name().equals(((AbstractSideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; + sideTableMap.put(tableInfo.getName(), (AbstractSideTableInfo) tableInfo); - URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, SideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, AbstractSideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sideTablePathUrl); } else { throw new RuntimeException("not support table type:" + tableInfo.getType()); @@ -351,4 +355,12 @@ public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperti } + public static void setLogLevel(ParamsInfo paramsInfo){ + String logLevel = paramsInfo.getConfProp().getProperty(ConfigConstrant.LOG_LEVEL_KEY); + if(org.apache.commons.lang3.StringUtils.isBlank(logLevel)){ + return; + } + ChangeLogLevelProcess logLevelProcess = new ChangeLogLevelProcess(); + logLevelProcess.process(logLevel); + } } \ No newline at end of file diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java index b36dc31eb..27cc7702d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java @@ -84,7 +84,6 @@ public Properties getConfProp() { return confProp; } - @Override public String toString() { return "ParamsInfo{" + diff --git a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java index 3a5af18b1..8802198a0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java @@ -24,6 +24,7 @@ import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.Meter; import org.apache.flink.metrics.MeterView; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; @@ -34,11 +35,11 @@ * author: toutian * create: 2019/12/24 */ -public class SerializationMetricWrapper implements SerializationSchema { +public class SerializationMetricWrapper implements SerializationSchema { private static final long serialVersionUID = 1L; - private SerializationSchema serializationSchema; + private SerializationSchema serializationSchema; private transient RuntimeContext runtimeContext; @@ -47,7 +48,7 @@ public class SerializationMetricWrapper implements SerializationSchema { protected transient Meter dtNumRecordsOutRate; - public SerializationMetricWrapper(SerializationSchema serializationSchema) { + public SerializationMetricWrapper(SerializationSchema serializationSchema) { this.serializationSchema = serializationSchema; } @@ -57,7 +58,7 @@ public void initMetric() { } @Override - public byte[] serialize(Row element) { + public byte[] serialize(CRow element) { beforeSerialize(); byte[] row = serializationSchema.serialize(element); afterSerialize(); @@ -79,7 +80,7 @@ public void setRuntimeContext(RuntimeContext runtimeContext) { this.runtimeContext = runtimeContext; } - public SerializationSchema getSerializationSchema() { + public SerializationSchema getSerializationSchema() { return serializationSchema; } diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 78b579305..5f1b1c6f3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.format.dtnest; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.google.common.base.Strings; import com.google.common.collect.Maps; import org.apache.flink.api.common.serialization.AbstractDeserializationSchema; @@ -28,8 +28,9 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.*; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.NullNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.types.Row; import java.io.IOException; @@ -57,9 +58,9 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final String[] fieldNames; private final TypeInformation[] fieldTypes; - private List fieldExtraInfos; + private List fieldExtraInfos; - public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { + public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); this.rowAndFieldMapping = rowAndFieldMapping; @@ -75,7 +76,7 @@ public Row deserialize(byte[] message) throws IOException { try { for (int i = 0; i < fieldNames.length; i++) { JsonNode node = getIgnoreCase(fieldNames[i]); - TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); if (node == null) { if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java index 43f599d14..0fd057c5a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java @@ -31,7 +31,10 @@ import java.io.File; import java.io.FileInputStream; import java.net.URLEncoder; +import java.util.stream.Stream; + import org.apache.commons.codec.Charsets; +import org.apache.flink.util.FileUtils; /** @@ -92,19 +95,16 @@ public Options getOptions(){ } public List getProgramExeArgList() throws Exception { - Map mapConf = PluginUtil.ObjectToMap(properties); + Map mapConf = PluginUtil.objectToMap(properties); List args = Lists.newArrayList(); - for(Map.Entry one : mapConf.entrySet()){ + for (Map.Entry one : mapConf.entrySet()) { String key = one.getKey(); Object value = one.getValue(); - if(value == null){ + if (value == null) { continue; - }else if(OPTION_SQL.equalsIgnoreCase(key)){ + } else if (OPTION_SQL.equalsIgnoreCase(key)) { File file = new File(value.toString()); - FileInputStream in = new FileInputStream(file); - byte[] filecontent = new byte[(int) file.length()]; - in.read(filecontent); - String content = new String(filecontent, Charsets.UTF_8.name()); + String content = FileUtils.readFile(file, "UTF-8"); value = URLEncoder.encode(content, Charsets.UTF_8.name()); } args.add("-" + key); @@ -112,9 +112,4 @@ public List getProgramExeArgList() throws Exception { } return args; } - - public static void main(String[] args) throws Exception { - OptionParser OptionParser = new OptionParser(args); - System.out.println(OptionParser.getOptions()); - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java b/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java similarity index 95% rename from core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java rename to core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java index 1fc40c13b..fbcc86bbd 100644 --- a/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java +++ b/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java @@ -27,7 +27,7 @@ * extend RichOutputFormat with metric 'dtNumRecordsOut', 'dtNumDirtyRecordsOut', 'dtNumRecordsOutRate' * Created by sishu.yss on 2018/11/28. */ -public abstract class DtRichOutputFormat extends RichOutputFormat{ +public abstract class AbstractDtRichOutputFormat extends RichOutputFormat{ public transient Counter outRecords; public transient Counter outDirtyRecords; diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java index 670d98a7e..fc6a7e16c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java @@ -32,18 +32,18 @@ public class CreateFuncParser implements IParser { - private static final String funcPatternStr = "(?i)\\s*create\\s+(scala|table|aggregate)\\s+function\\s+(\\S+)\\s+WITH\\s+(\\S+)"; + private static final String FUNC_PATTERN_STR = "(?i)\\s*create\\s+(scala|table|aggregate)\\s+function\\s+(\\S+)\\s+WITH\\s+(\\S+)"; - private static final Pattern funcPattern = Pattern.compile(funcPatternStr); + private static final Pattern FUNC_PATTERN = Pattern.compile(FUNC_PATTERN_STR); @Override public boolean verify(String sql) { - return funcPattern.matcher(sql).find(); + return FUNC_PATTERN.matcher(sql).find(); } @Override public void parseSql(String sql, SqlTree sqlTree) { - Matcher matcher = funcPattern.matcher(sql); + Matcher matcher = FUNC_PATTERN.matcher(sql); if(matcher.find()){ String type = matcher.group(1); String funcName = matcher.group(2); diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index a76c1b31a..2afc76c48 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.parser; import com.dtstack.flink.sql.enums.ETableType; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.table.TableInfoParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfoParser; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import com.google.common.collect.Lists; @@ -75,7 +75,7 @@ public static SqlTree parseSql(String sql) throws Exception { List sqlArr = DtStringUtil.splitIgnoreQuota(sql, SQL_DELIMITER); SqlTree sqlTree = new SqlTree(); - TableInfoParser tableInfoParser = new TableInfoParser(); + AbstractTableInfoParser tableInfoParser = new AbstractTableInfoParser(); for(String childSql : sqlArr){ if(Strings.isNullOrEmpty(childSql)){ continue; @@ -113,7 +113,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } - TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), + AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } @@ -126,7 +126,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } - TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SINK.getType(), + AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SINK.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } @@ -144,7 +144,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } } else { - TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), + AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java index 1b64b7c68..5252ee022 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.parser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.google.common.collect.Maps; import com.google.common.collect.Lists; @@ -41,7 +41,7 @@ public class SqlTree { private Map preDealTableMap = Maps.newHashMap(); - private Map tableInfoMap = Maps.newLinkedHashMap(); + private Map tableInfoMap = Maps.newLinkedHashMap(); private List execSqlList = Lists.newArrayList(); @@ -89,11 +89,11 @@ public List getTmpSqlList(){ return tmpSqlList; } - public Map getTableInfoMap() { + public Map getTableInfoMap() { return tableInfoMap; } - public void addTableInfo(String tableName, TableInfo tableInfo){ + public void addTableInfo(String tableName, AbstractTableInfo tableInfo){ tableInfoMap.put(tableName, tableInfo); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java similarity index 97% rename from core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index 32f065123..19dd67f9a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.google.common.collect.Lists; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ * @author xuchao */ -public abstract class SideTableInfo extends TableInfo implements Serializable { +public abstract class AbstractSideTableInfo extends AbstractTableInfo implements Serializable { public static final String TARGET_SUFFIX = "Side"; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java similarity index 79% rename from core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java rename to core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java index 64fd844b1..3e3d0a8fb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java @@ -25,13 +25,15 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.dtstack.flink.sql.factory.DTThreadFactory; import org.apache.calcite.sql.JoinType; import java.sql.SQLException; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; /** @@ -41,13 +43,15 @@ * @author xuchao */ -public abstract class AllReqRow extends RichFlatMapFunction implements ISideReqRow { +public abstract class BaseAllReqRow extends RichFlatMapFunction implements ISideReqRow { - protected SideInfo sideInfo; + private static final Logger LOG = LoggerFactory.getLogger(BaseAllReqRow.class); + + protected BaseSideInfo sideInfo; private ScheduledExecutorService es; - public AllReqRow(SideInfo sideInfo){ + public BaseAllReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -60,11 +64,11 @@ public AllReqRow(SideInfo sideInfo){ public void open(Configuration parameters) throws Exception { super.open(parameters); initCache(); - System.out.println("----- all cacheRef init end-----"); + LOG.info("----- all cacheRef init end-----"); //start reload cache thread - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); - es = Executors.newSingleThreadScheduledExecutor(new DTThreadFactory("cache-all-reload")); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + es = new ScheduledThreadPoolExecutor(1,new DTThreadFactory("cache-all-reload")); es.scheduleAtFixedRate(() -> reloadCache(), sideTableInfo.getCacheTimeout(), sideTableInfo.getCacheTimeout(), TimeUnit.MILLISECONDS); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java similarity index 89% rename from core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java rename to core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 6f0dce58a..7ac4813a2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.metric.MetricConstant; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cache.LRUSideCache; import org.apache.calcite.sql.JoinType; @@ -30,14 +30,12 @@ import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; -import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collections; -import java.util.concurrent.TimeoutException; /** * All interfaces inherit naming rules: type + "AsyncReqRow" such as == "MysqlAsyncReqRow @@ -47,17 +45,17 @@ * @author xuchao */ -public abstract class AsyncReqRow extends RichAsyncFunction implements ISideReqRow { - private static final Logger LOG = LoggerFactory.getLogger(AsyncReqRow.class); +public abstract class BaseAsyncReqRow extends RichAsyncFunction implements ISideReqRow { + private static final Logger LOG = LoggerFactory.getLogger(BaseAsyncReqRow.class); private static final long serialVersionUID = 2098635244857937717L; private static int TIMEOUT_LOG_FLUSH_NUM = 10; private int timeOutNum = 0; - protected SideInfo sideInfo; + protected BaseSideInfo sideInfo; protected transient Counter parseErrorRecords; - public AsyncReqRow(SideInfo sideInfo){ + public BaseAsyncReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -70,12 +68,12 @@ public void open(Configuration parameters) throws Exception { } private void initCache(){ - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); if(sideTableInfo.getCacheType() == null || ECacheType.NONE.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ return; } - AbsSideCache sideCache; + AbstractSideCache sideCache; if(ECacheType.LRU.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ sideCache = new LRUSideCache(sideTableInfo); sideInfo.setSideCache(sideCache); @@ -124,10 +122,10 @@ protected void dealCacheData(String key, CacheObj missKeyObj) { @Override public void timeout(CRow input, ResultFuture resultFuture) throws Exception { - //TODO 需要添加数据指标 if(timeOutNum % TIMEOUT_LOG_FLUSH_NUM == 0){ LOG.info("Async function call has timed out. input:{}, timeOutNum:{}",input.toString(), timeOutNum); } + timeOutNum ++; if(timeOutNum > sideInfo.getSideTableInfo().getAsyncTimeoutNumLimit()){ resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java similarity index 92% rename from core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java rename to core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java index d8a22e022..732691d0e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; @@ -41,7 +41,7 @@ * @author xuchao */ -public abstract class SideInfo implements Serializable{ +public abstract class BaseSideInfo implements Serializable{ protected RowTypeInfo rowTypeInfo; @@ -68,12 +68,12 @@ public abstract class SideInfo implements Serializable{ //key:Returns the value of the position, value: the ref field name​in the side table protected Map sideFieldNameIndex = Maps.newHashMap(); - protected SideTableInfo sideTableInfo; + protected AbstractSideTableInfo sideTableInfo; - protected AbsSideCache sideCache; + protected AbstractSideCache sideCache; - public SideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, - SideTableInfo sideTableInfo){ + public BaseSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, + AbstractSideTableInfo sideTableInfo){ this.rowTypeInfo = rowTypeInfo; this.outFieldInfoList = outFieldInfoList; this.joinType = joinInfo.getJoinType(); @@ -173,7 +173,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName){ } } - public abstract void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo); + public abstract void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo); public RowTypeInfo getRowTypeInfo() { return rowTypeInfo; @@ -247,19 +247,19 @@ public void setSideFieldIndex(Map sideFieldIndex) { this.sideFieldIndex = sideFieldIndex; } - public SideTableInfo getSideTableInfo() { + public AbstractSideTableInfo getSideTableInfo() { return sideTableInfo; } - public void setSideTableInfo(SideTableInfo sideTableInfo) { + public void setSideTableInfo(AbstractSideTableInfo sideTableInfo) { this.sideTableInfo = sideTableInfo; } - public AbsSideCache getSideCache() { + public AbstractSideCache getSideCache() { return sideCache; } - public void setSideCache(AbsSideCache sideCache) { + public void setSideCache(AbstractSideCache sideCache) { this.sideCache = sideCache; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index 544e8ebb0..d2bb2cb0d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -22,6 +22,7 @@ import com.dtstack.flink.sql.parser.FlinkPlanner; import com.dtstack.flink.sql.util.ParseUtils; import com.dtstack.flink.sql.util.TableUtils; +import com.esotericsoftware.minlog.Log; import com.google.common.base.Preconditions; import com.google.common.collect.*; import org.apache.calcite.sql.JoinType; @@ -438,18 +439,18 @@ private void extractTemporaryQuery(SqlNode node, String tableAlias, tableRef.put(tbTmp, tableAlias); } - System.out.println("-------build temporary query-----------"); - System.out.println(tmpSelectSql); - System.out.println("---------------------------------------"); + Log.info("-------build temporary query-----------\n{}", tmpSelectSql); + Log.info("---------------------------------------"); }catch (Exception e){ - e.printStackTrace(); + Log.error("", e); throw new RuntimeException(e); } } /** * 抽取上层需用使用到的字段 + * 由于where字段已经抽取到上一层了所以不用查询出来 * @param parentSelectList * @param fromTableNameSet * @return @@ -592,7 +593,6 @@ private void extractSelectField(SqlNode selectNode, } }else if(selectNode.getKind() == CASE){ - System.out.println("selectNode"); SqlCase sqlCase = (SqlCase) selectNode; SqlNodeList whenOperands = sqlCase.getWhenOperands(); SqlNodeList thenOperands = sqlCase.getThenOperands(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java b/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java index 738e0f84e..d9d79f34e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java @@ -80,6 +80,7 @@ public static List getRowTypeInfo(SqlNode sqlNode, JoinScope scope, b switch(identifierSize) { case 1: fieldInfoList.addAll(getAllField(scope)); + break; default: SqlIdentifier tableIdentify = identifier.skipLast(1); JoinScope.ScopeChild scopeChild = scope.getScope(tableIdentify.getSimple()); @@ -99,6 +100,7 @@ public static List getRowTypeInfo(SqlNode sqlNode, JoinScope scope, b fieldInfo.setTypeInformation(type); fieldInfoList.add(fieldInfo); } + break; } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java index 4d6112aa8..fa973508f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java @@ -46,7 +46,7 @@ * @author maqi */ public class SidePredicatesParser { - public void fillPredicatesForSideTable(String exeSql, Map sideTableMap) throws SqlParseException { + public void fillPredicatesForSideTable(String exeSql, Map sideTableMap) throws SqlParseException { FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); SqlNode sqlNode = flinkPlanner.parse(exeSql); parseSql(sqlNode, sideTableMap, Maps.newHashMap()); @@ -58,7 +58,7 @@ public void fillPredicatesForSideTable(String exeSql, Map * @param sideTableMap * @param tabMapping 谓词属性中别名对应的真实维表名称 */ - private void parseSql(SqlNode sqlNode, Map sideTableMap, Map tabMapping) { + private void parseSql(SqlNode sqlNode, Map sideTableMap, Map tabMapping) { SqlKind sqlKind = sqlNode.getKind(); switch (sqlKind) { case INSERT: @@ -101,10 +101,12 @@ private void parseSql(SqlNode sqlNode, Map sideTableMap, parseSql(unionLeft, sideTableMap, tabMapping); parseSql(unionRight, sideTableMap, tabMapping); break; + default: + break; } } - private void fillToSideTableInfo(Map sideTableMap, Map tabMapping, List predicateInfoList) { + private void fillToSideTableInfo(Map sideTableMap, Map tabMapping, List predicateInfoList) { predicateInfoList.stream().filter(info -> sideTableMap.containsKey(tabMapping.getOrDefault(info.getOwnerTable(), info.getOwnerTable()))) .map(info -> sideTableMap.get(tabMapping.getOrDefault(info.getOwnerTable(), info.getOwnerTable())).getPredicateInfoes().add(info)) .count(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index f37c3f78c..2296f64c5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -47,7 +47,8 @@ import java.util.Queue; import java.util.Set; -import static org.apache.calcite.sql.SqlKind.*; +import static org.apache.calcite.sql.SqlKind.IDENTIFIER; +import static org.apache.calcite.sql.SqlKind.LITERAL; /** * Parsing sql, obtain execution information dimension table @@ -62,8 +63,6 @@ public class SideSQLParser { private Map localTableCache = Maps.newHashMap(); public Queue getExeQueue(String exeSql, Set sideTableSet) throws SqlParseException { - System.out.println("----------exec original Sql----------"); - System.out.println(exeSql); LOG.info("----------exec original Sql----------"); LOG.info(exeSql); @@ -168,6 +167,8 @@ public Object parseSql(SqlNode sqlNode, case LITERAL: return LITERAL.toString(); + default: + break; } return ""; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index f90138b2a..c41e2264a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -43,24 +43,17 @@ import com.dtstack.flink.sql.util.ParseUtils; import com.dtstack.flink.sql.util.TableUtils; import com.google.common.base.Preconditions; -import com.google.common.collect.*; -import org.apache.calcite.sql.SqlAsOperator; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlInsert; -import org.apache.calcite.sql.SqlJoin; import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlWithItem; -import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -68,7 +61,6 @@ import java.sql.Timestamp; import java.util.Arrays; -import java.util.Collection; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -97,7 +89,7 @@ public class SideSqlExec { private Map localTableCache = Maps.newHashMap(); public void exec(String sql, - Map sideTableMap, + Map sideTableMap, StreamTableEnvironment tableEnv, Map tableCache, StreamQueryConfig queryConfig, @@ -131,11 +123,9 @@ public void exec(String sql, if(pollSqlNode.getKind() == INSERT){ - System.out.println("----------real exec sql-----------" ); - System.out.println(pollSqlNode.toString()); FlinkSQLExec.sqlUpdate(tableEnv, pollSqlNode.toString(), queryConfig); if(LOG.isInfoEnabled()){ - LOG.info("exec sql: " + pollSqlNode.toString()); + LOG.info("----------real exec sql-----------\n{}", pollSqlNode.toString()); } }else if(pollSqlNode.getKind() == AS){ @@ -166,8 +156,7 @@ public void exec(String sql, } }else if (pollObj instanceof JoinInfo){ - System.out.println("----------exec join info----------"); - System.out.println(pollObj.toString()); + LOG.info("----------exec join info----------\n{}", pollObj.toString()); joinFun(pollObj, localTableCache, sideTableMap, tableEnv); } } @@ -294,7 +283,7 @@ private Table getTableFromCache(Map localTableCache, String table * * @return */ - private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, SideTableInfo sideTableInfo) { + private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, AbstractSideTableInfo sideTableInfo) { List conditionFields = getConditionFields(conditionNode, sideTableAlias, sideTableInfo); if(CollectionUtils.isEqualCollection(conditionFields, convertPrimaryAlias(sideTableInfo))){ return true; @@ -302,7 +291,7 @@ private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, return false; } - private List convertPrimaryAlias(SideTableInfo sideTableInfo) { + private List convertPrimaryAlias(AbstractSideTableInfo sideTableInfo) { List res = Lists.newArrayList(); sideTableInfo.getPrimaryKeys().forEach(field -> { res.add(sideTableInfo.getPhysicalFields().getOrDefault(field, field)); @@ -310,7 +299,7 @@ private List convertPrimaryAlias(SideTableInfo sideTableInfo) { return res; } - public List getConditionFields(SqlNode conditionNode, String specifyTableName, SideTableInfo sideTableInfo){ + public List getConditionFields(SqlNode conditionNode, String specifyTableName, AbstractSideTableInfo sideTableInfo){ List sqlNodeList = Lists.newArrayList(); ParseUtils.parseAnd(conditionNode, sqlNodeList); List conditionFields = Lists.newArrayList(); @@ -369,7 +358,7 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, private void joinFun(Object pollObj, Map localTableCache, - Map sideTableMap, + Map sideTableMap, StreamTableEnvironment tableEnv) throws Exception{ JoinInfo joinInfo = (JoinInfo) pollObj; @@ -385,7 +374,7 @@ private void joinFun(Object pollObj, JoinScope.ScopeChild rightScopeChild = new JoinScope.ScopeChild(); rightScopeChild.setAlias(joinInfo.getRightTableAlias()); rightScopeChild.setTableName(joinInfo.getRightTableName()); - SideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); + AbstractSideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); if(sideTableInfo == null){ sideTableInfo = sideTableMap.get(joinInfo.getRightTableAlias()); } @@ -394,9 +383,9 @@ private void joinFun(Object pollObj, throw new RuntimeException("can't not find side table:" + joinInfo.getRightTableName()); } -// if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ -// throw new RuntimeException("ON condition must contain all equal fields!!!"); -// } + if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ + throw new RuntimeException("ON condition must contain all equal fields!!!"); + } rightScopeChild.setRowTypeInfo(sideTableInfo.getRowTypeInfo()); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java b/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java index 8417c4519..583e4597b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java @@ -21,8 +21,8 @@ import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.enums.ECacheType; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableParser; import com.dtstack.flink.sql.util.PluginUtil; /** @@ -37,7 +37,7 @@ public class StreamSideFactory { private static final String CURR_TYPE = "side"; - public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir, String cacheType) throws Exception { + public static AbstractTableParser getSqlParser(String pluginType, String sqlRootDir, String cacheType) throws Exception { String sideOperator = ECacheType.ALL.name().equalsIgnoreCase(cacheType) ? "all" : "async"; String pluginJarPath = PluginUtil.getSideJarFileDirPath(pluginType, sideOperator, "side", sqlRootDir); @@ -45,10 +45,10 @@ public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir, return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class sideParser = cl.loadClass(className); - if (!AbsSideTableParser.class.isAssignableFrom(sideParser)) { + if (!AbstractSideTableParser.class.isAssignableFrom(sideParser)) { throw new RuntimeException("class " + sideParser.getName() + " not subClass of AbsSideTableParser"); } - return sideParser.asSubclass(AbsTableParser.class).newInstance(); + return sideParser.asSubclass(AbstractTableParser.class).newInstance(); }); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java b/core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java similarity index 84% rename from core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java rename to core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java index 757f91600..58832c583 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.cache; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; /** * Reason: @@ -30,11 +30,11 @@ * @author xuchao */ -public abstract class AbsSideCache { +public abstract class AbstractSideCache { - protected SideTableInfo sideTableInfo; + protected AbstractSideTableInfo sideTableInfo; - public AbsSideCache(SideTableInfo sideTableInfo){ + public AbstractSideCache(AbstractSideTableInfo sideTableInfo){ this.sideTableInfo = sideTableInfo; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java b/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java index 700e13bb2..2664efd34 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.cache; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -33,11 +33,11 @@ * @author xuchao */ -public class LRUSideCache extends AbsSideCache{ +public class LRUSideCache extends AbstractSideCache { protected transient Cache cache; - public LRUSideCache(SideTableInfo sideTableInfo) { + public LRUSideCache(AbstractSideTableInfo sideTableInfo) { super(sideTableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java index 290804200..3c2010907 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java @@ -20,10 +20,10 @@ package com.dtstack.flink.sql.side.operator; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.AsyncDataStream; @@ -49,20 +49,20 @@ public class SideAsyncOperator { private static final String ORDERED = "ordered"; - private static AsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, - JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { + private static BaseAsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { String pathOfType = String.format(PATH_FORMAT, sideType); String pluginJarPath = PluginUtil.getJarFileDirPath(pathOfType, sqlRootDir); String className = PluginUtil.getSqlSideClassName(sideType, "side", OPERATOR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> - cl.loadClass(className).asSubclass(AsyncReqRow.class) - .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class) + cl.loadClass(className).asSubclass(BaseAsyncReqRow.class) + .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, AbstractSideTableInfo.class) .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, - List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { - AsyncReqRow asyncDbReq = loadAsyncReq(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { + BaseAsyncReqRow asyncDbReq = loadAsyncReq(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); //TODO How much should be set for the degree of parallelism? Timeout? capacity settings? if (ORDERED.equals(sideTableInfo.getCacheMode())){ diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java index 5aa810b0f..6b6f9fe1b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java @@ -20,10 +20,10 @@ package com.dtstack.flink.sql.side.operator; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; @@ -44,22 +44,22 @@ public class SideWithAllCacheOperator { private static final String OPERATOR_TYPE = "All"; - private static AllReqRow loadFlatMap(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, - JoinInfo joinInfo, List outFieldInfoList, - SideTableInfo sideTableInfo) throws Exception { + private static BaseAllReqRow loadFlatMap(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, List outFieldInfoList, + AbstractSideTableInfo sideTableInfo) throws Exception { String pathOfType = String.format(PATH_FORMAT, sideType); String pluginJarPath = PluginUtil.getJarFileDirPath(pathOfType, sqlRootDir); String className = PluginUtil.getSqlSideClassName(sideType, "side", OPERATOR_TYPE); - return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> cl.loadClass(className).asSubclass(AllReqRow.class) - .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class) + return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> cl.loadClass(className).asSubclass(BaseAllReqRow.class) + .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, AbstractSideTableInfo.class) .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, - List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { - AllReqRow allReqRow = loadFlatMap(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { + BaseAllReqRow allReqRow = loadFlatMap(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); return inputStream.flatMap(allReqRow); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java b/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java index 3cfd48f1b..e29421369 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; /** * Reason: @@ -30,5 +30,5 @@ */ public interface IStreamSinkGener { - T genStreamSink(TargetTableInfo targetTableInfo); + T genStreamSink(AbstractTargetTableInfo targetTableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java b/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java index 53460081d..89061db3a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.sink; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.table.sinks.TableSink; @@ -40,21 +40,21 @@ public class StreamSinkFactory { private static final String DIR_NAME_FORMAT = "%ssink"; - public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { + public static AbstractTableParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { String pluginJarPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), sqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); String className = PluginUtil.getSqlParserClassName(typeNoVersion, CURR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class targetParser = cl.loadClass(className); - if(!AbsTableParser.class.isAssignableFrom(targetParser)){ + if(!AbstractTableParser.class.isAssignableFrom(targetParser)){ throw new RuntimeException("class " + targetParser.getName() + " not subClass of AbsTableParser"); } - return targetParser.asSubclass(AbsTableParser.class).newInstance(); + return targetParser.asSubclass(AbstractTableParser.class).newInstance(); }); } - public static TableSink getTableSink(TargetTableInfo targetTableInfo, String localSqlRootDir) throws Exception { + public static TableSink getTableSink(AbstractTargetTableInfo targetTableInfo, String localSqlRootDir) throws Exception { String pluginType = targetTableInfo.getType(); String pluginJarDirPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), localSqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); diff --git a/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java b/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java index b8b8f6edc..fde5a235d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java +++ b/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.source; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.java.StreamTableEnvironment; @@ -38,6 +38,6 @@ public interface IStreamSourceGener { * @param tableEnv * @return */ - T genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv); + T genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv); } diff --git a/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java b/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java index 1057fb0ed..e0cec1415 100644 --- a/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.table.AbsSourceParser; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceParser; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -43,17 +43,17 @@ public class StreamSourceFactory { private static final String DIR_NAME_FORMAT = "%ssource"; - public static AbsSourceParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { + public static AbstractSourceParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { String pluginJarPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), sqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); String className = PluginUtil.getSqlParserClassName(typeNoVersion, CURR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class sourceParser = cl.loadClass(className); - if(!AbsSourceParser.class.isAssignableFrom(sourceParser)){ + if(!AbstractSourceParser.class.isAssignableFrom(sourceParser)){ throw new RuntimeException("class " + sourceParser.getName() + " not subClass of AbsSourceParser"); } - return sourceParser.asSubclass(AbsSourceParser.class).newInstance(); + return sourceParser.asSubclass(AbstractSourceParser.class).newInstance(); }); } @@ -62,7 +62,7 @@ public static AbsSourceParser getSqlParser(String pluginType, String sqlRootDir) * @param sourceTableInfo * @return */ - public static Table getStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, + public static Table getStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String sqlRootDir) throws Exception { String sourceTypeStr = sourceTableInfo.getType(); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java similarity index 68% rename from core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 403360400..4973a891e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.table; import com.dtstack.flink.sql.enums.ECacheType; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.MathUtil; import org.apache.flink.util.Preconditions; @@ -36,24 +36,24 @@ * @author xuchao */ -public abstract class AbsSideTableParser extends AbsTableParser { +public abstract class AbstractSideTableParser extends AbstractTableParser { private final static String SIDE_SIGN_KEY = "sideSignKey"; private final static Pattern SIDE_TABLE_SIGN = Pattern.compile("(?i)^PERIOD\\s+FOR\\s+SYSTEM_TIME$"); - public AbsSideTableParser() { + public AbstractSideTableParser() { addParserHandler(SIDE_SIGN_KEY, SIDE_TABLE_SIGN, this::dealSideSign); } - private void dealSideSign(Matcher matcher, TableInfo tableInfo){ + private void dealSideSign(Matcher matcher, AbstractTableInfo tableInfo){ //FIXME SIDE_TABLE_SIGN current just used as a sign for side table; and do nothing } //Analytical create table attributes ==> Get information cache - protected void parseCacheProp(SideTableInfo sideTableInfo, Map props){ - if(props.containsKey(SideTableInfo.CACHE_KEY.toLowerCase())){ - String cacheType = MathUtil.getString(props.get(SideTableInfo.CACHE_KEY.toLowerCase())); + protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map props){ + if(props.containsKey(AbstractSideTableInfo.CACHE_KEY.toLowerCase())){ + String cacheType = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_KEY.toLowerCase())); if(cacheType == null){ return; } @@ -63,65 +63,66 @@ protected void parseCacheProp(SideTableInfo sideTableInfo, Map p } sideTableInfo.setCacheType(cacheType); - if(props.containsKey(SideTableInfo.CACHE_SIZE_KEY.toLowerCase())){ - Integer cacheSize = MathUtil.getIntegerVal(props.get(SideTableInfo.CACHE_SIZE_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.CACHE_SIZE_KEY.toLowerCase())){ + Integer cacheSize = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.CACHE_SIZE_KEY.toLowerCase())); if(cacheSize < 0){ throw new RuntimeException("cache size need > 0."); } sideTableInfo.setCacheSize(cacheSize); } - if(props.containsKey(SideTableInfo.CACHE_TTLMS_KEY.toLowerCase())){ - Long cacheTTLMS = MathUtil.getLongVal(props.get(SideTableInfo.CACHE_TTLMS_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.CACHE_TTLMS_KEY.toLowerCase())){ + Long cacheTTLMS = MathUtil.getLongVal(props.get(AbstractSideTableInfo.CACHE_TTLMS_KEY.toLowerCase())); if(cacheTTLMS < 1000){ throw new RuntimeException("cache time out need > 1000 ms."); } sideTableInfo.setCacheTimeout(cacheTTLMS); } - if(props.containsKey(SideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())){ - Boolean partitionedJoinKey = MathUtil.getBoolean(props.get(SideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())){ + Boolean partitionedJoinKey = MathUtil.getBoolean(props.get(AbstractSideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())); if(partitionedJoinKey){ sideTableInfo.setPartitionedJoin(true); } } - if(props.containsKey(SideTableInfo.CACHE_MODE_KEY.toLowerCase())){ - String cachemode = MathUtil.getString(props.get(SideTableInfo.CACHE_MODE_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.CACHE_MODE_KEY.toLowerCase())){ + String cachemode = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_MODE_KEY.toLowerCase())); - if(!cachemode.equalsIgnoreCase("ordered") && !cachemode.equalsIgnoreCase("unordered")){ + if(!"ordered".equalsIgnoreCase(cachemode) && !"unordered".equalsIgnoreCase(cachemode)){ throw new RuntimeException("cachemode must ordered or unordered!"); } sideTableInfo.setCacheMode(cachemode.toLowerCase()); } - if(props.containsKey(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())){ - Integer asyncCap = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.ASYNC_CAP_KEY.toLowerCase())){ + Integer asyncCap = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_CAP_KEY.toLowerCase())); if(asyncCap < 0){ throw new RuntimeException("asyncCapacity size need > 0."); } sideTableInfo.setAsyncCapacity(asyncCap); } - if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())){ - Integer asyncTimeout = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())){ + Integer asyncTimeout = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())); if (asyncTimeout<0){ throw new RuntimeException("asyncTimeout size need > 0."); } sideTableInfo.setAsyncTimeout(asyncTimeout); } - if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())){ - Integer asyncTimeoutNum = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())){ + Integer asyncTimeoutNum = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())); if (asyncTimeoutNum > 0){ sideTableInfo.setAsyncTimeoutNumLimit(asyncTimeoutNum); } } - if (props.containsKey(SideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())) { - Integer asyncPoolSize = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())); + if (props.containsKey(AbstractSideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())) { + Integer asyncPoolSize = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())); Preconditions.checkArgument(asyncPoolSize > 0 && asyncPoolSize <= 20, "asyncPoolSize size limit (0,20]"); sideTableInfo.setAsyncPoolSize(asyncPoolSize); } + } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java similarity index 81% rename from core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java index 745357162..308f5859f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java @@ -33,7 +33,7 @@ * @author xuchao */ -public abstract class AbsSourceParser extends AbsTableParser { +public abstract class AbstractSourceParser extends AbstractTableParser { private static final String VIRTUAL_KEY = "virtualFieldKey"; private static final String WATERMARK_KEY = "waterMarkKey"; @@ -43,21 +43,21 @@ public abstract class AbsSourceParser extends AbsTableParser { private static Pattern waterMarkKeyPattern = Pattern.compile("(?i)^\\s*WATERMARK\\s+FOR\\s+(\\S+)\\s+AS\\s+withOffset\\(\\s*(\\S+)\\s*,\\s*(\\d+)\\s*\\)$"); private static Pattern notNullKeyPattern = Pattern.compile("(?i)^(\\w+)\\s+(\\w+)\\s+NOT\\s+NULL?$"); - public AbsSourceParser() { + public AbstractSourceParser() { addParserHandler(VIRTUAL_KEY, virtualFieldKeyPattern, this::dealVirtualField); addParserHandler(WATERMARK_KEY, waterMarkKeyPattern, this::dealWaterMark); addParserHandler(NOTNULL_KEY, notNullKeyPattern, this::dealNotNull); } - protected void dealVirtualField(Matcher matcher, TableInfo tableInfo){ - SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; + protected void dealVirtualField(Matcher matcher, AbstractTableInfo tableInfo){ + AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; String fieldName = matcher.group(2); String expression = matcher.group(1); sourceTableInfo.addVirtualField(fieldName, expression); } - protected void dealWaterMark(Matcher matcher, TableInfo tableInfo){ - SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; + protected void dealWaterMark(Matcher matcher, AbstractTableInfo tableInfo){ + AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; String eventTimeField = matcher.group(1); //FIXME Temporarily resolve the second parameter row_time_field Integer offset = MathUtil.getIntegerVal(matcher.group(3)); @@ -65,11 +65,11 @@ protected void dealWaterMark(Matcher matcher, TableInfo tableInfo){ sourceTableInfo.setMaxOutOrderness(offset); } - protected void dealNotNull(Matcher matcher, TableInfo tableInfo) { + protected void dealNotNull(Matcher matcher, AbstractTableInfo tableInfo) { String fieldName = matcher.group(1); String fieldType = matcher.group(2); Class fieldClass= dbTypeConvertToJavaType(fieldType); - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); fieldExtraInfo.setNotNull(true); tableInfo.addPhysicalMappings(fieldName, fieldName); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java similarity index 97% rename from core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java index 9a41fa0a1..0fdc0e911 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java @@ -36,7 +36,7 @@ * @author xuchao */ -public abstract class SourceTableInfo extends TableInfo { +public abstract class AbstractSourceTableInfo extends AbstractTableInfo { public static final String SOURCE_SUFFIX = "Source"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java similarity index 98% rename from core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java index 2fdc297a2..3b514dd48 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java @@ -34,7 +34,7 @@ * @author xuchao */ -public abstract class TableInfo implements Serializable { +public abstract class AbstractTableInfo implements Serializable { public static final String PARALLELISM_KEY = "parallelism"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java similarity index 83% rename from core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java index 3c1be25c6..d50e1f23c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.enums.ETableType; import com.dtstack.flink.sql.parser.CreateTableParser; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.StreamSideFactory; import com.dtstack.flink.sql.sink.StreamSinkFactory; import com.dtstack.flink.sql.source.StreamSourceFactory; @@ -42,7 +42,7 @@ * @author xuchao */ -public class TableInfoParser { +public class AbstractTableInfoParser { private final static String TYPE_KEY = "type"; @@ -50,16 +50,16 @@ public class TableInfoParser { private final static Pattern SIDE_PATTERN = Pattern.compile(SIDE_TABLE_SIGN); - private Map sourceTableInfoMap = Maps.newConcurrentMap(); + private Map sourceTableInfoMap = Maps.newConcurrentMap(); - private Map targetTableInfoMap = Maps.newConcurrentMap(); + private Map targetTableInfoMap = Maps.newConcurrentMap(); - private Map sideTableInfoMap = Maps.newConcurrentMap(); + private Map sideTableInfoMap = Maps.newConcurrentMap(); //Parsing loaded plugin - public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserResult parserResult, - String localPluginRoot) throws Exception { - AbsTableParser absTableParser = null; + public AbstractTableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserResult parserResult, + String localPluginRoot) throws Exception { + AbstractTableParser absTableParser = null; Map props = parserResult.getPropMap(); String type = MathUtil.getString(props.get(TYPE_KEY)); @@ -77,9 +77,9 @@ public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserRe sourceTableInfoMap.put(type, absTableParser); } }else{ - String cacheType = MathUtil.getString(props.get(SideTableInfo.CACHE_KEY)); - absTableParser = sideTableInfoMap.get(type + cacheType); + absTableParser = sideTableInfoMap.get(type); if(absTableParser == null){ + String cacheType = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_KEY)); absTableParser = StreamSideFactory.getSqlParser(type, localPluginRoot, cacheType); sideTableInfoMap.put(type + cacheType, absTableParser); } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java similarity index 88% rename from core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 3e4027b2c..1f210cbf2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -39,7 +39,7 @@ * @author xuchao */ -public abstract class AbsTableParser { +public abstract class AbstractTableParser { private static final String PRIMARY_KEY = "primaryKey"; private static final String NEST_JSON_FIELD_KEY = "nestFieldKey"; @@ -54,7 +54,7 @@ public abstract class AbsTableParser { private Map handlerMap = Maps.newHashMap(); - public AbsTableParser() { + public AbstractTableParser() { addParserHandler(PRIMARY_KEY, primaryKeyPattern, this::dealPrimaryKey); addParserHandler(NEST_JSON_FIELD_KEY, nestJsonFieldKeyPattern, this::dealNestField); } @@ -63,9 +63,9 @@ protected boolean fieldNameNeedsUpperCase() { return true; } - public abstract TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception; + public abstract AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception; - public boolean dealKeyPattern(String fieldRow, TableInfo tableInfo){ + public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ for(Map.Entry keyPattern : patternMap.entrySet()){ Pattern pattern = keyPattern.getValue(); String key = keyPattern.getKey(); @@ -84,7 +84,7 @@ public boolean dealKeyPattern(String fieldRow, TableInfo tableInfo){ return false; } - public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ + public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); for(String fieldRow : fieldRows){ @@ -112,12 +112,12 @@ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ Class fieldClass = null; - TableInfo.FieldExtraInfo fieldExtraInfo = null; + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; Matcher matcher = charTypePattern.matcher(fieldType); if (matcher.find()) { fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); - fieldExtraInfo = new TableInfo.FieldExtraInfo(); + fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); } else { fieldClass = dbTypeConvertToJavaType(fieldType); @@ -133,7 +133,7 @@ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ tableInfo.finish(); } - public void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ + public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo){ String primaryFields = matcher.group(1).trim(); String[] splitArry = primaryFields.split(","); List primaryKes = Lists.newArrayList(splitArry); @@ -145,7 +145,7 @@ public void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ * @param matcher * @param tableInfo */ - protected void dealNestField(Matcher matcher, TableInfo tableInfo) { + protected void dealNestField(Matcher matcher, AbstractTableInfo tableInfo) { String physicalField = matcher.group(1); Preconditions.checkArgument(!physicalFieldFunPattern.matcher(physicalField).find(), "No need to add data types when using functions, The correct way is : strLen(name) as nameSize, "); @@ -154,7 +154,7 @@ protected void dealNestField(Matcher matcher, TableInfo tableInfo) { String mappingField = matcher.group(4); Class fieldClass = dbTypeConvertToJavaType(fieldType); boolean notNull = matcher.group(5) != null; - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); fieldExtraInfo.setNotNull(notNull); tableInfo.addPhysicalMappings(mappingField, physicalField); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java similarity index 94% rename from core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java index 0dc9ca77b..dbd99f993 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java @@ -29,7 +29,7 @@ * @author xuchao */ -public abstract class TargetTableInfo extends TableInfo { +public abstract class AbstractTargetTableInfo extends AbstractTableInfo { public static final String TARGET_SUFFIX = "Sink"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java b/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java index db804ea34..fab634cdc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java @@ -30,5 +30,5 @@ */ public interface ITableFieldDealHandler { - void dealPrimaryKey(Matcher matcher, TableInfo tableInfo); + void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index feebbe3f1..4bb4ff0d8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -86,7 +86,8 @@ public static Class stringConvertClass(String str) { case "decimal": case "decimalunsigned": return BigDecimal.class; - + default: + break; } throw new RuntimeException("不支持 " + str + " 类型"); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index a0f5291e7..409602baa 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -16,22 +16,17 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.util; import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.ZoneOffset; -import java.util.Calendar; -import java.util.Date; -import java.util.Locale; -import java.util.SimpleTimeZone; -import java.util.TimeZone; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.*; import java.util.regex.Pattern; import static java.time.format.DateTimeFormatter.ISO_INSTANT; @@ -39,7 +34,7 @@ /** * - * Reason: TODO ADD REASON(可选) + * 日期工具 * Date: 2017年03月10日 下午1:16:37 * Company: www.dtstack.com * @author sishu.yss @@ -47,20 +42,16 @@ */ public class DateUtil { - static final String timeZone = "GMT+8"; - static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss"; - static final String dateFormat = "yyyy-MM-dd"; - static final String timeFormat = "HH:mm:ss"; - static final SimpleDateFormat datetimeFormatter = new SimpleDateFormat(datetimeFormat); - static final SimpleDateFormat dateFormatter = new SimpleDateFormat(dateFormat); - static final SimpleDateFormat timeFormatter = new SimpleDateFormat(timeFormat); + static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss"); private static final Pattern DATETIME = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); private static final Pattern DATE = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}$"); - private static final int MILLIS_PER_SECOND = 1000; + public static java.sql.Date columnToDate(Object column) { if(column instanceof String) { return new java.sql.Date(stringToDate((String)column).getTime()); @@ -84,23 +75,32 @@ public static Date stringToDate(String strDate) { return null; } try { - return datetimeFormatter.parse(strDate); - } catch (ParseException ignored) { + ; + return localDateTimetoDate(LocalDateTime.parse(strDate, DATE_TIME_FORMATTER)); + } catch (DateTimeParseException ignored) { } try { - return dateFormatter.parse(strDate); - } catch (ParseException ignored) { + return localDateTimetoDate(LocalDate.parse(strDate, DATE_FORMATTER).atStartOfDay()); + } catch (DateTimeParseException ignored) { } try { - return timeFormatter.parse(strDate); - } catch (ParseException ignored) { + return localDateTimetoDate(LocalDateTime.of(LocalDate.now(), LocalTime.parse(strDate, TIME_FORMATTER))); + } catch (DateTimeParseException ignored) { } throw new RuntimeException("can't parse date"); } + public static Date localDateTimetoDate(LocalDateTime localDateTime){ + return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant()); + } + + public static LocalDateTime dateToLocalDateTime(Date date){ + return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } + /** * * @@ -130,13 +130,13 @@ public static long getTodayStart(long day) { * @return */ public static long getTodayStart(long day,String scope) { - if(scope.equals("MS")){ - return getTodayStart(day)*1000; - }else if(scope.equals("S")){ - return getTodayStart(day); - }else{ - return getTodayStart(day); - } + if("MS".equals(scope)){ + return getTodayStart(day)*1000; + }else if("S".equals(scope)){ + return getTodayStart(day); + }else{ + return getTodayStart(day); + } } /** @@ -168,15 +168,15 @@ public static long getNextDayStart(long day) { * @return */ public static long getNextDayStart(long day,String scope) { - if(scope.equals("MS")){ - return getNextDayStart(day)*1000; - }else if(scope.equals("S")){ - return getNextDayStart(day); - }else{ - return getNextDayStart(day); - } + if("MS".equals(scope)){ + return getNextDayStart(day)*1000; + }else if("S".equals(scope)){ + return getNextDayStart(day); + }else{ + return getNextDayStart(day); + } } - + /** * @@ -237,7 +237,7 @@ public static long getWeekFirst(long day) { /** * 根据某个日期时间戳秒值，获取所在周在一年中是第几周. - * + * * @param day * @return */ @@ -257,17 +257,17 @@ public static int getWeekOfYear(long day) { */ public static String getYesterdayByString(String day, String inFormat, String outFormat){ try { - SimpleDateFormat sdf = new SimpleDateFormat(inFormat); - Date date = sdf.parse(day); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - int calendarDay = calendar.get(Calendar.DATE); - calendar.set(Calendar.DATE, calendarDay - 1); - String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); - return dayBefore; - } catch (ParseException e) { - return null; - } + SimpleDateFormat sdf = new SimpleDateFormat(inFormat); + Date date = sdf.parse(day); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + int calendarDay = calendar.get(Calendar.DATE); + calendar.set(Calendar.DATE, calendarDay - 1); + String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); + return dayBefore; + } catch (ParseException e) { + return null; + } } /** @@ -288,7 +288,7 @@ public static String getTomorrowByString(String day, String inFormat, String out String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); return dayBefore; } - + /** * * @param date @@ -320,7 +320,7 @@ public static String get30DaysBeforeByString(String day, String inFormat, String calendar.set(Calendar.DATE, calendarDay - 30); return new SimpleDateFormat(outFormat).format(calendar.getTime()); } - + /** * * @param day @@ -330,14 +330,14 @@ public static String get30DaysBeforeByString(String day, String inFormat, String * @throws ParseException */ public static String get30DaysLaterByString(String day, String inFormat, String outFormat) throws ParseException { - SimpleDateFormat sdf = new SimpleDateFormat(inFormat); - Date date = sdf.parse(day); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - int calendarDay = calendar.get(Calendar.DATE); - calendar.set(Calendar.DATE, calendarDay + 30); - String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); - return dayBefore; + SimpleDateFormat sdf = new SimpleDateFormat(inFormat); + Date date = sdf.parse(day); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + int calendarDay = calendar.get(Calendar.DATE); + calendar.set(Calendar.DATE, calendarDay + 30); + String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); + return dayBefore; } @@ -349,7 +349,7 @@ public static String get30DaysLaterByString(String day, String inFormat, String * @return String * @throws ParseException */ - public static String getDateStrTOFormat(String day, String inFormat, String outFormat) throws ParseException { + public static String getDateStrToFormat(String day, String inFormat, String outFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -357,8 +357,8 @@ public static String getDateStrTOFormat(String day, String inFormat, String outF String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); return dayBefore; } - - public static long getDateMillTOFormat(String day, String inFormat) throws ParseException { + + public static long getDateMillToFormat(String day, String inFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -474,79 +474,79 @@ public static long getMillByLastWeekDay() { cal.set(Calendar.MILLISECOND, 0); return cal.getTimeInMillis() / 1000; } - - /** - * @return long - */ + + /** + * @return long + */ public static long getMillByDay(int severalDays,String condition) { - int dateT=0; + int dateT=0; Calendar cal = Calendar.getInstance(); - if(condition==null){ - return getMillToDay(cal,dateT); - } - if(condition.equals("-")){ - dateT = (cal.get(Calendar.DATE) - severalDays); - return getMillToDay(cal,dateT); - } - if(condition.equals("+")){ - dateT = (cal.get(Calendar.DATE) + severalDays); - return getMillToDay(cal,dateT); - } - return getMillToDay(cal,dateT); - } - + if(condition==null){ + return getMillToDay(cal,dateT); + } + if("-".equals(condition)){ + dateT = (cal.get(Calendar.DATE) - severalDays); + return getMillToDay(cal,dateT); + } + if("+".equals(condition)){ + dateT = (cal.get(Calendar.DATE) + severalDays); + return getMillToDay(cal,dateT); + } + return getMillToDay(cal,dateT); + } + /** * @return long */ public static long getStampByDay(int severalDays,String condition) { - int dateT=0; - Calendar cal = Calendar.getInstance(); - if(condition==null){ - return getStampToDay(cal,dateT); - } - if(condition.equals("-")){ - dateT = (cal.get(Calendar.DATE) - severalDays); - return getStampToDay(cal,dateT); - } - if(condition.equals("+")){ - dateT = (cal.get(Calendar.DATE) + severalDays); - return getStampToDay(cal,dateT); - } - return getStampToDay(cal,dateT); + int dateT=0; + Calendar cal = Calendar.getInstance(); + if(condition==null){ + return getStampToDay(cal,dateT); + } + if("-".equals(condition)){ + dateT = (cal.get(Calendar.DATE) - severalDays); + return getStampToDay(cal,dateT); + } + if("+".equals(condition)){ + dateT = (cal.get(Calendar.DATE) + severalDays); + return getStampToDay(cal,dateT); + } + return getStampToDay(cal,dateT); } /** * @return long */ public static long getMillByDay(){ - return getMillByDay(0,null); + return getMillByDay(0,null); } - + /** * @param cal Calendar - * @param dateT Integer + * @param dateT Integer * @return long */ public static long getMillToDay(Calendar cal,int dateT){ - if(dateT!=0){ - cal.set(Calendar.DATE, dateT); - } - cal.set(Calendar.HOUR_OF_DAY, 0); - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - return cal.getTimeInMillis()/1000; - } - + if(dateT!=0){ + cal.set(Calendar.DATE, dateT); + } + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + return cal.getTimeInMillis()/1000; + } + /** * @param cal Calendar - * @param dateT Integer + * @param dateT Integer * @return long */ public static long getStampToDay(Calendar cal,int dateT){ - if(dateT!=0){ - cal.set(Calendar.DATE, dateT); - } - return cal.getTimeInMillis(); + if(dateT!=0){ + cal.set(Calendar.DATE, dateT); + } + return cal.getTimeInMillis(); } public static String getToday() { @@ -568,7 +568,7 @@ public static String getDate(long day, String format) { SimpleDateFormat sf = new SimpleDateFormat(format); return sf.format(cal.getTime()); } - + /** * * @param date @@ -578,21 +578,21 @@ public static String getDate(Date date, String format) { SimpleDateFormat sf = new SimpleDateFormat(format); return sf.format(date); } - - + + /** * * @param day * @param format * @return long - * @throws ParseException + * @throws ParseException */ public static long stringToLong(String day, String format) throws ParseException { - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - long Date = dateFormat.parse(day).getTime(); - return Date; + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + long date = dateFormat.parse(day).getTime(); + return date; } - + /** * @param day * @param format @@ -600,30 +600,30 @@ public static long stringToLong(String day, String format) throws ParseException * @throws ParseException */ public static Date stringToDate(String day, String format) { - try { - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - Date Date = dateFormat.parse(day); - return Date; - } catch (ParseException e) { - return new Date(); - } + try { + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + Date date = dateFormat.parse(day); + return date; + } catch (ParseException e) { + return new Date(); + } } - - + + /** * long型时间戳转为String型 - * + * * @param day 秒 * @return 格式化后的日期 - * @throws ParseException + * @throws ParseException */ public static String longToString(long day, String format) throws ParseException { - if (("" + day).length() <= 10){ + if (("" + day).length() <= 10){ day=day*1000; } - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - String Date = dateFormat.format(day); - return Date; + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + String date = dateFormat.format(day); + return date; } /** @@ -653,39 +653,39 @@ public static long getMillByNow() { return cal.getTimeInMillis(); } - public static int getWeeksBetweenTwoDates(long startDay, long endDay) { - int week = getWeekOfYear(endDay) - getWeekOfYear(startDay) + 1; - if(week<1){ - week = getWeekOfYear(endDay) + getMaxWeekOfYear(startDay) - getWeekOfYear(startDay) + 1; - } - return week; - } + public static int getWeeksBetweenTwoDates(long startDay, long endDay) { + int week = getWeekOfYear(endDay) - getWeekOfYear(startDay) + 1; + if(week<1){ + week = getWeekOfYear(endDay) + getMaxWeekOfYear(startDay) - getWeekOfYear(startDay) + 1; + } + return week; + } - public static int getMaxWeekOfYear(long startDay) { - Calendar cal = Calendar.getInstance(); + public static int getMaxWeekOfYear(long startDay) { + Calendar cal = Calendar.getInstance(); cal.setTime(new Date(startDay * 1000)); return cal.getMaximum(Calendar.WEEK_OF_YEAR); - } - - public static int getMonthsBetweenTwoDates(long startDay, long endDay) { - int month = DateUtil.getMonth(endDay) - DateUtil.getMonth(startDay) + 1; - if(month<1){ - month = getMonth(endDay) + 12 - getMonth(startDay) +1; - } - return month; - } - - public static Date parseDate(String dateStr, String pattern){ - SimpleDateFormat sdf = new SimpleDateFormat(); - sdf.applyPattern(pattern); - try { - return sdf.parse(dateStr); - } catch (ParseException e) { - return null; - } - } - - /** + } + + public static int getMonthsBetweenTwoDates(long startDay, long endDay) { + int month = DateUtil.getMonth(endDay) - DateUtil.getMonth(startDay) + 1; + if(month<1){ + month = getMonth(endDay) + 12 - getMonth(startDay) +1; + } + return month; + } + + public static Date parseDate(String dateStr, String pattern){ + SimpleDateFormat sdf = new SimpleDateFormat(); + sdf.applyPattern(pattern); + try { + return sdf.parse(dateStr); + } catch (ParseException e) { + return null; + } + } + + /** * * @param time Long 时间 * @return long @@ -703,7 +703,7 @@ public static long getMinuteStart(long time) { firstDay = cal.getTimeInMillis() / 1000; return firstDay; } - + /** * @param time Long * @return long @@ -732,24 +732,24 @@ public static Date getDateByLong(long time){ date.setTime(time); return date; } - + public static Date parseDate(String dateStr, String pattern, Locale locale){ - SimpleDateFormat df = new SimpleDateFormat( - pattern, locale); - - df.setTimeZone(new SimpleTimeZone(0, "GMT")); - try { - return df.parse(dateStr); - } catch (ParseException e) { - return null; - } - } - + SimpleDateFormat df = new SimpleDateFormat( + pattern, locale); + + df.setTimeZone(new SimpleTimeZone(0, "GMT")); + try { + return df.parse(dateStr); + } catch (ParseException e) { + return null; + } + } + public static String getDate(Date date, String format, Locale locale) { - SimpleDateFormat df = new SimpleDateFormat( - format, locale); - df.setTimeZone(new SimpleTimeZone(0, "GMT")); + SimpleDateFormat df = new SimpleDateFormat( + format, locale); + df.setTimeZone(new SimpleTimeZone(0, "GMT")); return df.format(date); } @@ -777,59 +777,33 @@ public static java.sql.Timestamp columnToTimestamp(Object column) { } public static String dateToString(Date date) { - return dateFormatter.format(date); + LocalDateTime localDateTime = dateToLocalDateTime(date); + return localDateTime.format(DATE_FORMATTER); } public static String timestampToString(Date date) { - return datetimeFormatter.format(date); + LocalDateTime localDateTime = dateToLocalDateTime(date); + return localDateTime.format(DATE_TIME_FORMATTER); } - public static Timestamp getTimestampFromStr(String timeStr) { if (DATETIME.matcher(timeStr).matches()) { Instant instant = Instant.from(ISO_INSTANT.parse(timeStr)); return new Timestamp(instant.getEpochSecond() * MILLIS_PER_SECOND); - } else { - java.sql.Date date = null; - try { - date = new java.sql.Date(datetimeFormatter.parse(timeStr).getTime()); - } catch (ParseException e) { - throw new RuntimeException("getTimestampFromStr error data is " + timeStr); - } - return new Timestamp(date.getTime()); } + return new Timestamp(stringToDate(timeStr).getTime()); } public static java.sql.Date getDateFromStr(String dateStr) { - // 2020-01-01 format if (DATE.matcher(dateStr).matches()) { - // convert from local date to instant Instant instant = LocalDate.parse(dateStr).atTime(LocalTime.of(0, 0, 0, 0)).toInstant(ZoneOffset.UTC); - // calculate the timezone offset in millis int offset = TimeZone.getDefault().getOffset(instant.toEpochMilli()); - // need to remove the offset since time has no TZ component return new java.sql.Date(instant.toEpochMilli() - offset); } else if (DATETIME.matcher(dateStr).matches()) { - // 2020-01-01T12:12:12Z format Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); return new java.sql.Date(instant.toEpochMilli()); - } else { - try { - // 2020-01-01 12:12:12.0 format - return new java.sql.Date(datetimeFormatter.parse(dateStr).getTime()); - } catch (ParseException e) { - throw new RuntimeException("String convert to Date fail."); - } } - } - - - public static String getStringFromTimestamp(Timestamp timestamp) { - return datetimeFormatter.format(timestamp); - } - - public static String getStringFromDate(java.sql.Date date) { - return dateFormatter.format(date); + return new java.sql.Date(stringToDate(dateStr).getTime()); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index b2486b1ab..10a6b4f63 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -219,14 +219,14 @@ public static String addJdbcParam(String dbUrl, Map addParams, b return preStr + "?" + sb.toString(); } - public static boolean isJosn(String str){ + public static boolean isJson(String str) { boolean flag = false; - if(StringUtils.isNotBlank(str)){ + if (StringUtils.isNotBlank(str)) { try { - objectMapper.readValue(str,Map.class); + objectMapper.readValue(str, Map.class); flag = true; } catch (Throwable e) { - flag=false; + flag = false; } } return flag; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java index fde2f166e..3d66d8a6d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java @@ -28,10 +28,10 @@ public class JDBCUtils { private static final Logger LOG = LoggerFactory.getLogger(ClassUtil.class); - public final static String lock_str = "jdbc_lock_str"; + public final static String LOCK_STR = "jdbc_lock_str"; public static void forName(String clazz, ClassLoader classLoader) { - synchronized (lock_str){ + synchronized (LOCK_STR){ try { Class.forName(clazz, true, classLoader); DriverManager.setLoginTimeout(10); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index 23513a912..5aaa40a15 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -227,7 +227,7 @@ public static BigDecimal getBigDecimal(Object obj) { } else if (obj instanceof BigInteger) { return new BigDecimal((BigInteger) obj); } else if (obj instanceof Number) { - return new BigDecimal(((Number) obj).doubleValue()); + return BigDecimal.valueOf(((Number) obj).doubleValue()); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to BigDecimal."); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java index a63e5a663..8eb743cf0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java @@ -109,7 +109,7 @@ public static String getSqlSideClassName(String pluginTypeName, String type, Str return CLASS_PRE_STR + "." + type.toLowerCase() + "." + pluginTypeName + "." + pluginClassName; } - public static Map ObjectToMap(Object obj) throws Exception{ + public static Map objectToMap(Object obj) throws Exception{ return objectMapper.readValue(objectMapper.writeValueAsBytes(obj), Map.class); } @@ -173,8 +173,8 @@ public static void addPluginJar(String pluginDir, DtClassLoader classLoader) thr } for(File file : files){ - URL pluginJarURL = file.toURI().toURL(); - classLoader.addURL(pluginJarURL); + URL pluginJarUrl = file.toURI().toURL(); + classLoader.addURL(pluginJarUrl); } } @@ -191,8 +191,8 @@ public static URL[] getPluginJarUrls(String pluginDir) throws MalformedURLExcept } for(File file : files){ - URL pluginJarURL = file.toURI().toURL(); - urlList.add(pluginJarURL); + URL pluginJarUrl = file.toURI().toURL(); + urlList.add(pluginJarUrl); } return urlList.toArray(new URL[urlList.size()]); } diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java similarity index 94% rename from core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java rename to core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java index ed8f64288..d75d26a61 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java @@ -38,7 +38,7 @@ * @author xuchao */ -public abstract class AbsCustomerWaterMarker extends BoundedOutOfOrdernessTimestampExtractor implements RichFunction { +public abstract class AbstractCustomerWaterMarker extends BoundedOutOfOrdernessTimestampExtractor implements RichFunction { private static final long serialVersionUID = 1L; @@ -55,7 +55,7 @@ public abstract class AbsCustomerWaterMarker extends BoundedOutOfOrdernessTim protected TimeZone timezone; - public AbsCustomerWaterMarker(Time maxOutOfOrderness) { + public AbstractCustomerWaterMarker(Time maxOutOfOrderness) { super(maxOutOfOrderness); } diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java index 99415ba61..979b4c3d6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java @@ -35,7 +35,7 @@ * @author xuchao */ -public class CustomerWaterMarkerForLong extends AbsCustomerWaterMarker { +public class CustomerWaterMarkerForLong extends AbstractCustomerWaterMarker { private static final Logger logger = LoggerFactory.getLogger(CustomerWaterMarkerForLong.class); diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java index b7961e410..05ee3e46e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java @@ -20,7 +20,6 @@ package com.dtstack.flink.sql.watermarker; -import com.dtstack.flink.sql.util.MathUtil; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -36,7 +35,7 @@ * @author xuchao */ -public class CustomerWaterMarkerForTimeStamp extends AbsCustomerWaterMarker { +public class CustomerWaterMarkerForTimeStamp extends AbstractCustomerWaterMarker { private static final Logger logger = LoggerFactory.getLogger(CustomerWaterMarkerForTimeStamp.class); diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java index f3208db03..b0ca0335f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.watermarker; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.base.Strings; @@ -40,7 +40,7 @@ public class WaterMarkerAssigner { - public boolean checkNeedAssignWaterMarker(SourceTableInfo tableInfo){ + public boolean checkNeedAssignWaterMarker(AbstractSourceTableInfo tableInfo){ if(Strings.isNullOrEmpty(tableInfo.getEventTimeField())){ return false; } @@ -48,7 +48,7 @@ public boolean checkNeedAssignWaterMarker(SourceTableInfo tableInfo){ return true; } - public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo typeInfo, SourceTableInfo sourceTableInfo){ + public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo typeInfo, AbstractSourceTableInfo sourceTableInfo){ String eventTimeFieldName = sourceTableInfo.getEventTimeField(); @@ -75,7 +75,7 @@ public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo type TypeInformation fieldType = fieldTypes[pos]; - AbsCustomerWaterMarker waterMarker = null; + AbstractCustomerWaterMarker waterMarker = null; if(fieldType.getTypeClass().isAssignableFrom(Timestamp.class)){ waterMarker = new CustomerWaterMarkerForTimeStamp(Time.milliseconds(maxOutOrderness), pos,timeZone); }else if(fieldType.getTypeClass().isAssignableFrom(Long.class)){ diff --git a/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java b/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java index 546fcbbac..feec8e547 100644 --- a/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java +++ b/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java @@ -54,7 +54,7 @@ public void testfillPredicatesForSideTable() throws SqlParseException { " MyTable.a='1' and s.d='1' and s.d <> '3' and s.c LIKE '%xx%' and s.c in ('1','2') and s.c between '10' and '23' and s.d is not null\n"; - SideTableInfo sideTableInfo = new SideTableInfo(){ + AbstractSideTableInfo sideTableInfo = new AbstractSideTableInfo(){ @Override public boolean check() { return false; @@ -63,7 +63,7 @@ public boolean check() { sideTableInfo.setName("sideTable"); - Map sideTableMap = new HashMap<>(); + Map sideTableMap = new HashMap<>(); sideTableMap.put("sideTable", sideTableInfo); SidePredicatesParser sidePredicatesParser = new SidePredicatesParser(); diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java index a77201b80..b354a6269 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java @@ -20,10 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.google.common.collect.Maps; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +29,6 @@ import java.sql.Connection; import java.sql.DriverManager; import java.util.List; -import java.util.Map; /** * Date: 2019/11/20 @@ -40,21 +37,21 @@ * @author xiuzhu */ -public class Db2AllReqRow extends RdbAllReqRow { +public class Db2AllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(Db2AllReqRow.class); private static final String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; - public Db2AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Db2AllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Class.forName(DB2_DRIVER); - Connection conn = DriverManager.getConnection(dbURL, userName, password); + Connection conn = DriverManager.getConnection(dbUrl, userName, password); return conn; } catch (Exception e) { LOG.error("", e); diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java index b40cfd772..4ce01bcc8 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; @@ -37,7 +37,7 @@ */ public class Db2AllSideInfo extends RdbAllSideInfo { - public Db2AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index ed9fb025b..b190973e3 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -45,7 +45,7 @@ public class Db2AsyncReqRow extends RdbAsyncReqRow { private final static String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; - public Db2AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Db2AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -71,7 +71,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, db2lientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, db2lientConfig)); } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java index dbb8230e7..91418a1d0 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; @@ -37,7 +37,7 @@ */ public class Db2AsyncSideInfo extends RdbAsyncSideInfo { - public Db2AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java b/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java index db8a07dc6..96be4ec15 100644 --- a/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java +++ b/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.db2.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -36,8 +36,8 @@ public class Db2SideParser extends RdbSideParser { private static final String CURR_TYPE = "db2"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURR_TYPE); return tableInfo; diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java index 573a0896a..09844d1a3 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java @@ -1,13 +1,10 @@ package com.dtstack.flink.sql.sink.db; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import java.util.List; -import java.util.Map; - -public class DbSink extends RdbSink { +public class DbSink extends AbstractRdbSink { public DbSink() { super(new DbDialect()); @@ -15,7 +12,7 @@ public DbSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java index ba11aabf3..3e823ab20 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java @@ -1,7 +1,7 @@ package com.dtstack.flink.sql.sink.db.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -10,8 +10,8 @@ public class DbSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "db2"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURR_TYPE); return tableInfo; } diff --git a/docs/kafkaSink.md b/docs/kafkaSink.md new file mode 100644 index 000000000..5d7c7b2a7 --- /dev/null +++ b/docs/kafkaSink.md @@ -0,0 +1,223 @@ +## 1.格式： +``` +CREATE TABLE tableName( + colName colType, + ... + function(colNameX) AS aliasName, + WATERMARK FOR colName AS withOffset( colName , delayTime ) + )WITH( + type ='kafka11', + bootstrapServers ='ip:port,ip:port...', + zookeeperQuorum ='ip:port,ip:port/zkparent', + offsetReset ='latest', + topic ='topicName', + groupId='test', + parallelism ='parllNum', + ); +``` + +## 2.支持的版本 + kafka09,kafka10,kafka11及以上版本 + **kafka读取和写入的版本必须一致，否则会有兼容性错误。** + +## 3.表结构定义 + +|参数名称|含义| +|----|---| +| tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称| +| colName | 列名称| +| colType | 列类型 [colType支持的类型](colType.md)| + +## 4.参数： + +|参数名称|含义|是否必填|默认值| +|----|---|---|---| +|type | kafka09 | 是|kafka09、kafka10、kafka11、kafka(对应kafka1.0及以上版本)| +|groupId | 需要读取的 groupId 名称|否|| +|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| +|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| +|topic | 需要读取的 topic 名称|是|| +|parallelism | 并行度设置|否|1| +|partitionKeys | 用来分区的字段|否|| +|updateMode | 回溯流数据下发模式，append,upsert.upsert模式下会将是否为回溯信息以字段形式进行下发。|否|append| +|sinkdatatype | 写入kafka数据格式，json,avro,csv|否|json| +|fieldDelimiter | csv数据分隔符|否| , | + + +**kafka相关参数可以自定义，使用kafka.开头即可。** +``` +kafka.consumer.id +kafka.socket.timeout.ms +kafka.fetch.message.max.bytes +kafka.num.consumer.fetchers +kafka.auto.commit.enable +kafka.auto.commit.interval.ms +kafka.queued.max.message.chunks +kafka.rebalance.max.retries +kafka.fetch.min.bytes +kafka.fetch.wait.max.ms +kafka.rebalance.backoff.ms +kafka.refresh.leader.backoff.ms +kafka.consumer.timeout.ms +kafka.exclude.internal.topics +kafka.partition.assignment.strategy +kafka.client.id +kafka.zookeeper.session.timeout.ms +kafka.zookeeper.connection.timeout.ms +kafka.zookeeper.sync.time.ms +kafka.offsets.storage +kafka.offsets.channel.backoff.ms +kafka.offsets.channel.socket.timeout.ms +kafka.offsets.commit.max.retries +kafka.dual.commit.enabled +kafka.partition.assignment.strategy +kafka.socket.receive.buffer.bytes +kafka.fetch.min.bytes +``` + +## 5.样例： + +### json格式： +``` +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='2', + partitionKeys = 'channel,pv', + updateMode='upsert' + ); + +upsert模式下发的数据格式：{"channel":"zs","pv":"330",retract:true} +append模式下发的数据格式：{"channel":"zs","pv":"330"} + +``` + +### avro格式： + +如果updateMode='upsert',schemaInfo需要包含retract属性信息。 + +``` +CREATE TABLE MyTable( + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='1', + topicIsPattern ='false' + ); + +create table sideTable( + channel varchar, + xccount int, + PRIMARY KEY(channel), + PERIOD FOR SYSTEM_TIME + )WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', + userName='dtstack', + password='abc123', + tableName='sidetest', + cache = 'LRU', + cacheTTLMs='10000', + parallelism ='1' + + ); + + +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + --type='console' + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='1', + updateMode='upsert', + sinkdatatype = 'avro', + schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"} + ,{"name":"pv","type":"string"},{"name":"channel","type":"string"}, + {"name":"retract","type":"boolean"}]}' + + ); + + +insert +into + MyResult + select + a.channel as channel, + a.pv as pv + from + MyTable a +``` +### csv格式： + +``` +CREATE TABLE MyTable( + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='2', + topicIsPattern ='false' + ); + +create table sideTable( + channel varchar, + xccount int, + PRIMARY KEY(channel), + PERIOD FOR SYSTEM_TIME + )WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', + userName='dtstack', + password='abc123', + tableName='sidetest', + cache = 'LRU', + cacheTTLMs='10000', + parallelism ='1' + + ); + + +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='2', + updateMode='upsert', + sinkdatatype = 'csv', + fieldDelimiter='*' + + + + ); + + +insert +into + MyResult + select + a.channel as channel, + a.pv as pv + from + MyTable a +``` diff --git a/docs/plugin/kafkaSource.md b/docs/plugin/kafkaSource.md index 035e39ffb..95a60a3ff 100644 --- a/docs/plugin/kafkaSource.md +++ b/docs/plugin/kafkaSource.md @@ -1,6 +1,5 @@ ## 1.格式： ``` -数据现在支持json格式{"xx":"bb","cc":"dd"} CREATE TABLE tableName( colName colType, @@ -15,9 +14,8 @@ CREATE TABLE tableName( topic ='topicName', groupId='test', parallelism ='parllNum', - --timezone='America/Los_Angeles', timezone='Asia/Shanghai', - sourcedatatype ='json' #可不设置 + sourcedatatype ='dt_nest' #可不设置 ); ``` @@ -47,8 +45,10 @@ CREATE TABLE tableName( |topicIsPattern | topic是否是正则表达式格式(true|false) |否| false |offsetReset | 读取的topic 的offset初始位置[latest|earliest|指定offset值({"0":12312,"1":12321,"2":12312},{"partition_no":offset_value})]|否|latest| |parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型|否|json| -|timezone|时区设置[timezone支持的参数](../timeZone.md)|否|'Asia/Shanghai' +|sourcedatatype | 数据类型,avro,csv,json,dt_nest。dt_nest为默认JSON解析器，能够解析嵌套JSON数据类型，其他仅支持非嵌套格式|否|dt_nest| +|schemaInfo | avro类型使用的schema信息|否|| +|fieldDelimiter |csv类型使用的数据分隔符|否| | | +|timezone|时区设置[timezone支持的参数](timeZone.md)|否|'Asia/Shanghai' **kafka相关参数可以自定义，使用kafka.开头即可。** ``` kafka.consumer.id @@ -169,24 +169,10 @@ CREATE TABLE MyTable( parallelism ='1' ); ``` -# 二、csv格式数据源 -根据字段分隔符进行数据分隔，按顺序匹配sql中配置的列。如数据分隔列数和sql中配置的列数相等直接匹配；如不同参照lengthcheckpolicy策略处理。 -## 1.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|| -|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|topic | 需要读取的 topic 名称|是|| -|offsetReset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| -|parallelism | 并行度设置 |否|1| -|sourcedatatype | 数据类型|是 |csv| -|fielddelimiter | 字段分隔符|是 || -|lengthcheckpolicy | 单行字段条数检查策略 |否|可选，默认为SKIP,其它可选值为EXCEPTION、PAD。SKIP：字段数目不符合时跳过 。EXCEPTION:字段数目不符合时抛出异常。PAD:按顺序填充，不存在的置为null。| -**kafka相关参数可以自定义，使用kafka.开头即可。** -## 2.样例： +## 7.csv格式数据源 + + ``` CREATE TABLE MyTable( name varchar, @@ -203,186 +189,28 @@ CREATE TABLE MyTable( --topic ='mqTest.*', --topicIsPattern='true' parallelism ='1', - sourcedatatype ='csv', - fielddelimiter ='\|', - lengthcheckpolicy = 'PAD' + sourceDatatype ='csv' ); ``` -# 三、text格式数据源UDF自定义拆分 -Kafka源表数据解析流程：Kafka Source Table -> UDTF ->Realtime Compute -> SINK。从Kakfa读入的数据，都是VARBINARY（二进制）格式，对读入的每条数据，都需要用UDTF将其解析成格式化数据。 - 与其他格式不同，本格式定义DDL必须与以下SQL一摸一样，表中的五个字段顺序务必保持一致： - -## 1. 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 -``` -create table kafka_stream( - _topic STRING, - _messageKey STRING, - _message STRING, - _partition INT, - _offset BIGINT, -) with ( - type ='kafka09', - bootstrapServers ='172.16.8.198:9092', - zookeeperQuorum ='172.16.8.198:2181/kafka', - offsetReset ='latest', - topic ='nbTest1', - parallelism ='1', - sourcedatatype='text' - ） -``` -## 2.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|| -|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|topic | 需要读取的 topic 名称|是|| -|offsetReset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| -|parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型|否|text| -**kafka相关参数可以自定义，使用kafka.开头即可。** +## 8.avro格式数据源 -## 2.自定义： -从kafka读出的数据，需要进行窗口计算。 按照实时计算目前的设计，滚窗/滑窗等窗口操作，需要（且必须）在源表DDL上定义Watermark。Kafka源表比较特殊。如果要以kafka中message字段中的的Event Time进行窗口操作， -需要先从message字段，使用UDX解析出event time，才能定义watermark。 在kafka源表场景中，需要使用计算列。 假设，kafka中写入的数据如下： -2018-11-11 00:00:00|1|Anna|female整个计算流程为：Kafka SOURCE->UDTF->Realtime Compute->RDS SINK（单一分隔符可直接使用类csv格式模板，自定义适用于更复杂的数据类型，本说明只做参考） - -**SQL** ``` --- 定义解析Kakfa message的UDTF - CREATE FUNCTION kafkapaser AS 'com.XXXX.kafkaUDTF'; - CREATE FUNCTION kafkaUDF AS 'com.XXXX.kafkaUDF'; - -- 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 - create table kafka_src ( - _topic STRING, - _messageKey STRING, - _message STRING, - _partition INT, - _offset BIGINT, - ctime AS TO_TIMESTAMP(kafkaUDF(_message)), -- 定义计算列，计算列可理解为占位符，源表中并没有这一列，其中的数据可经过下游计算得出。注意计算里的类型必须为timestamp才能在做watermark。 - watermark for ctime as withoffset(ctime,0) -- 在计算列上定义watermark - ) WITH ( - type = 'kafka010', -- Kafka Source类型，与Kafka版本强相关，目前支持的Kafka版本请参考本文档 - topic = 'test_kafka_topic', - ... - ); - create table rds_sink ( - name VARCHAR, - age INT, - grade VARCHAR, - updateTime TIMESTAMP - ) WITH( - type='mysql', - url='jdbc:mysql://localhost:3306/test', - tableName='test4', - userName='test', - password='XXXXXX' +CREATE TABLE MyTable( + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='1', + topicIsPattern ='false', + kafka.group.id='mqTest', + sourceDataType ='avro', + schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"},{"name":"pv","type":"string"}]}' ); - -- 使用UDTF，将二进制数据解析成格式化数据 - CREATE VIEW input_view ( - name, - age, - grade, - updateTime - ) AS - SELECT - COUNT(*) as cnt, - T.ctime, - T.order, - T.name, - T.sex - from - kafka_src as S, - LATERAL TABLE (kafkapaser _message)) as T ( - ctime, - order, - name, - sex - ) - Group BY T.sex, - TUMBLE(ctime, INTERVAL '1' MINUTE); - -- 对input_view中输出的数据做计算 - CREATE VIEW view2 ( - cnt, - sex - ) AS - SELECT - COUNT(*) as cnt, - T.sex - from - input_view - Group BY sex, TUMBLE(ctime, INTERVAL '1' MINUTE); - -- 使用解析出的格式化数据进行计算，并将结果输出到RDS中 - insert into rds_sink - SELECT - cnt,sex - from view2; - ``` -**UDF&UDTF** + ``` -package com.XXXX; - import com.XXXX.fastjson.JSONObject; - import org.apache.flink.table.functions.TableFunction; - import org.apache.flink.table.types.DataType; - import org.apache.flink.table.types.DataTypes; - import org.apache.flink.types.Row; - import java.io.UnsupportedEncodingException; - /** - 以下例子解析输入Kafka中的JSON字符串，并将其格式化输出 - **/ - public class kafkaUDTF extends TableFunction { - public void eval(byte[] message) { - try { - // 读入一个二进制数据，并将其转换为String格式 - String msg = new String(message, "UTF-8"); - // 提取JSON Object中各字段 - String ctime = Timestamp.valueOf(data.split('\\|')[0]); - String order = data.split('\\|')[1]; - String name = data.split('\\|')[2]; - String sex = data.split('\\|')[3]; - // 将解析出的字段放到要输出的Row()对象 - Row row = new Row(4); - row.setField(0, ctime); - row.setField(1, age); - row.setField(2, grade); - row.setField(3, updateTime); - System.out.println("Kafka message str ==>" + row.toString()); - // 输出一行 - collect(row); - } catch (ClassCastException e) { - System.out.println("Input data format error. Input data " + msg + "is not json string"); - } - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - } - @Override - // 如果返回值是Row，就必须重载实现这个方法，显式地告诉系统返回的字段类型 - // 定义输出Row()对象的字段类型 - public DataType getResultType(Object[] arguments, Class[] argTypes) { - return DataTypes.createRowType(DataTypes.TIMESTAMP,DataTypes.STRING, DataTypes.Integer, DataTypes.STRING,DataTypes.STRING); - } - } - - package com.dp58; - package com.dp58.sql.udx; - import org.apache.flink.table.functions.FunctionContext; - import org.apache.flink.table.functions.ScalarFunction; - public class KafkaUDF extends ScalarFunction { - // 可选，open方法可以不写 - // 需要import org.apache.flink.table.functions.FunctionContext; - public String eval(byte[] message) { - // 读入一个二进制数据，并将其转换为String格式 - String msg = new String(message, "UTF-8"); - return msg.split('\\|')[0]; - } - public long eval(String b, String c) { - return eval(b) + eval(c); - } - //可选，close方法可以不写 - @Override - public void close() { - } - } - ``` + diff --git a/docs/plugin/mongoSide.md b/docs/plugin/mongoSide.md index 90ffa2582..62d2d8698 100644 --- a/docs/plugin/mongoSide.md +++ b/docs/plugin/mongoSide.md @@ -40,8 +40,6 @@ |----|---|---|----| | type |表明 输出表类型 mongo|是|| | address | 连接mongo数据库 jdbcUrl |是|| - | userName | mongo连接用户名|否|| - | password | mongo连接密码|否|| | tableName | mongo表名称|是|| | database | mongo表名称|是|| | cache | 维表缓存策略(NONE/LRU)|否|NONE| @@ -64,7 +62,8 @@ create table sideTable( PERIOD FOR SYSTEM_TIME )WITH( type ='mongo', - address ='172.21.32.1:27017,172.21.32.1:27017', + //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] + address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', database ='test', tableName ='sidetest', cache ='LRU', diff --git a/docs/plugin/mongoSink.md b/docs/plugin/mongoSink.md index 9900dea12..fdfe93d22 100644 --- a/docs/plugin/mongoSink.md +++ b/docs/plugin/mongoSink.md @@ -33,8 +33,6 @@ CREATE TABLE tableName( |----|----|----|----| |type |表明 输出表类型 mongo|是|| |address | 连接mongo数据库 jdbcUrl |是|| -|userName | mongo连接用户名|否|| -|password | mongo连接密码|否|| |tableName | mongo表名称|是|| |database | mongo表名称|是|| |parallelism | 并行度设置|否|1| @@ -46,9 +44,8 @@ CREATE TABLE MyResult( pv VARCHAR )WITH( type ='mongo', - address ='172.21.32.1:27017,172.21.32.1:27017', - userName ='dtstack', - password ='abc123', + //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] + address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', database ='test', tableName ='pv', parallelism ='1' diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 491d1af27..5ca81c5ed 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.sink.elasticsearch; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -33,7 +34,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -164,7 +164,7 @@ public void setBulkFlushMaxActions(int bulkFlushMaxActions) { } @Override - public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { + public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { ElasticsearchTableInfo elasticsearchTableInfo = (ElasticsearchTableInfo) targetTableInfo; esTableInfo = elasticsearchTableInfo; clusterName = elasticsearchTableInfo.getClusterName(); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 93682de46..7988e597c 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -32,7 +32,7 @@ * @author sishu.yss * @Company: www.dtstack.com */ -public class ElasticsearchSinkParser extends AbsTableParser { +public class ElasticsearchSinkParser extends AbstractTableParser { private static final String KEY_ES_ADDRESS = "address"; @@ -56,7 +56,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ElasticsearchTableInfo elasticsearchTableInfo = new ElasticsearchTableInfo(); elasticsearchTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearchTableInfo); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index 9681742c2..f254d5b4e 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -33,7 +33,7 @@ * @author sishu.yss * @Company: www.dtstack.com */ -public class ElasticsearchTableInfo extends TargetTableInfo { +public class ElasticsearchTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "elasticsearch"; diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index 648f09fba..b8cf2c46a 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -18,24 +18,22 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; - -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; import com.dtstack.flink.sql.side.elasticsearch6.util.SwitchUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -49,7 +47,6 @@ import java.io.IOException; import java.io.Serializable; -import java.sql.SQLException; import java.sql.Timestamp; import java.util.Calendar; import java.util.List; @@ -60,7 +57,7 @@ * @author yinxi * @date 2020/1/13 - 1:00 */ -public class Elasticsearch6AllReqRow extends AllReqRow implements Serializable { +public class Elasticsearch6AllReqRow extends BaseAllReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AllReqRow.class); @@ -70,7 +67,7 @@ public class Elasticsearch6AllReqRow extends AllReqRow implements Serializable { private SearchRequest searchRequest; private BoolQueryBuilder boolQueryBuilder; - public Elasticsearch6AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Elasticsearch6AllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java index bf21b3ca8..d353a583d 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java @@ -18,12 +18,13 @@ package com.dtstack.flink.sql.side.elasticsearch6; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; + import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; @@ -35,15 +36,15 @@ * @author yinxi * @date 2020/1/13 - 1:01 */ -public class Elasticsearch6AllSideInfo extends SideInfo { +public class Elasticsearch6AllSideInfo extends BaseSideInfo { - public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index f67177fe4..1f4bd1bf1 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -18,6 +18,12 @@ package com.dtstack.flink.sql.side.elasticsearch6; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.PredicateInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -26,7 +32,6 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; @@ -57,14 +62,14 @@ * @author yinxi * @date 2020/2/13 - 13:10 */ -public class Elasticsearch6AsyncReqRow extends AsyncReqRow implements Serializable { +public class Elasticsearch6AsyncReqRow extends BaseAsyncReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AsyncReqRow.class); private transient RestHighLevelClient rhlClient; private SearchRequest searchRequest; private List sqlJoinCompareOperate = Lists.newArrayList(); - public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Elasticsearch6AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); SqlNode conditionNode = joinInfo.getCondition(); ParseUtils.parseJoinCompareOperate(conditionNode, sqlJoinCompareOperate); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java index 7b3a2f9e7..4118988ab 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java @@ -18,18 +18,17 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import org.apache.flink.api.java.typeutils.RowTypeInfo; - +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -37,15 +36,15 @@ * @author yinxi * @date 2020/2/13 - 13:09 */ -public class Elasticsearch6AsyncSideInfo extends SideInfo { +public class Elasticsearch6AsyncSideInfo extends BaseSideInfo { - public Elasticsearch6AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java index 13dfe2995..1b39bbf0f 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.side.elasticsearch6.table; import com.dtstack.flink.sql.side.elasticsearch6.util.ClassUtil; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -30,7 +30,7 @@ * @author yinxi * @date 2020/1/13 - 1:07 */ -public class Elasticsearch6SideParser extends AbsSideTableParser { +public class Elasticsearch6SideParser extends AbstractSideTableParser { private static final String KEY_ES6_ADDRESS = "address"; @@ -55,7 +55,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { Elasticsearch6SideTableInfo elasticsearch6SideTableInfo = new Elasticsearch6SideTableInfo(); elasticsearch6SideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearch6SideTableInfo); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java index 0afe2d59e..a01c90a53 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side.elasticsearch6.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -26,7 +26,7 @@ * @author yinxi * @date 2020/1/13 - 15:00 */ -public class Elasticsearch6SideTableInfo extends SideTableInfo { +public class Elasticsearch6SideTableInfo extends AbstractSideTableInfo { private static final String CURR_TYPE = "elasticsearch6"; diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java index cc591789e..3d9e4d62a 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.side.elasticsearch6.util; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHost; @@ -98,7 +98,7 @@ public static RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh } // add index and type to search request - public static SearchRequest setSearchRequest(SideInfo sideInfo) { + public static SearchRequest setSearchRequest(BaseSideInfo sideInfo) { SearchRequest searchRequest = new SearchRequest(); Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); // determine existence of index @@ -129,7 +129,7 @@ public static SearchRequest setSearchRequest(SideInfo sideInfo) { } // build where cause - public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { + public static BoolQueryBuilder setPredicateclause(BaseSideInfo sideInfo) { BoolQueryBuilder boolQueryBuilder = null; List predicateInfoes = sideInfo.getSideTableInfo().getPredicateInfoes(); @@ -144,7 +144,7 @@ public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { } // build filter condition - public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, SideInfo sideInfo) { + public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, BaseSideInfo sideInfo) { switch (info.getOperatorKind()) { case "IN": return boolQueryBuilder.must(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName(), sideInfo), removeSpaceAndApostrophe(info.getCondition()))); @@ -202,7 +202,7 @@ public static String[] removeSpaceAndApostrophe(String str) { } // prevent word segmentation - public static String textConvertToKeyword(String fieldName, SideInfo sideInfo) { + public static String textConvertToKeyword(String fieldName, BaseSideInfo sideInfo) { String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); String fieldType = sideFieldTypes[fieldIndex]; diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 2646c50e9..b7d9de6fc 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.elasticsearch; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,7 +32,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.collect.Maps; import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; @@ -128,7 +128,7 @@ public void emitDataStream(DataStream> dataStream) { } @Override - public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { + public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { esTableInfo = (ElasticsearchTableInfo) targetTableInfo; clusterName = esTableInfo.getClusterName(); index = esTableInfo.getIndex(); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 8902b953c..22c2b72bc 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -29,7 +29,7 @@ * @author yinxi * @date 2020/1/9 - 15:06 */ -public class ElasticsearchSinkParser extends AbsTableParser { +public class ElasticsearchSinkParser extends AbstractTableParser { private static final String KEY_ES6_ADDRESS = "address"; @@ -55,7 +55,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ElasticsearchTableInfo elasticsearchTableInfo = new ElasticsearchTableInfo(); elasticsearchTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearchTableInfo); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index 3cc3dd9ff..d0eef18f0 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -29,7 +29,7 @@ * @author yinxi * @date 2020/1/9 - 15:06 */ -public class ElasticsearchTableInfo extends TargetTableInfo { +public class ElasticsearchTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "elasticsearch6"; diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 11380eea6..ed2931fff 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -45,7 +45,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicReference; -public class HbaseAllReqRow extends AllReqRow { +public class HbaseAllReqRow extends BaseAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(HbaseAllReqRow.class); @@ -55,7 +55,7 @@ public class HbaseAllReqRow extends AllReqRow { private AtomicReference>> cacheRef = new AtomicReference<>(); - public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new HbaseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); tableName = ((HbaseSideTableInfo)sideTableInfo).getTableName(); @@ -134,7 +134,7 @@ public void flatMap(CRow input, Collector out) throws Exception { Map cacheList = null; - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; if (hbaseSideTableInfo.isPreRowKey()) { for (Map.Entry> entry : cacheRef.get().entrySet()) { @@ -153,7 +153,7 @@ public void flatMap(CRow input, Collector out) throws Exception { } private void loadData(Map> tmpCache) throws SQLException { - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; Configuration conf = new Configuration(); conf.set("hbase.zookeeper.quorum", hbaseSideTableInfo.getHost()); @@ -182,9 +182,17 @@ private void loadData(Map> tmpCache) throws SQLExcep LOG.error("", e); } finally { try { - conn.close(); - table.close(); - resultScanner.close(); + if (null != conn && !conn.isClosed()) { + conn.close(); + } + + if (null != table) { + table.close(); + } + + if (null != resultScanner) { + resultScanner.close(); + } } catch (IOException e) { LOG.error("", e); } diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java index ea51f46e4..663c2927b 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,16 +31,16 @@ import java.util.List; -public class HbaseAllSideInfo extends SideInfo { +public class HbaseAllSideInfo extends BaseSideInfo { private RowKeyBuilder rowKeyBuilder; - public HbaseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { rowKeyBuilder = new RowKeyBuilder(); if(sideTableInfo.getPrimaryKeys().size() < 1){ throw new RuntimeException("Primary key dimension table must be filled"); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 251b88034..24ee01786 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -21,12 +21,12 @@ package com.dtstack.flink.sql.side.hbase; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; -import com.dtstack.flink.sql.side.hbase.rowkeydealer.AbsRowKeyModeDealer; +import com.dtstack.flink.sql.side.hbase.rowkeydealer.AbstractRowKeyModeDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.PreRowKeyModeDealerDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.RowKeyEqualModeDealer; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; @@ -58,7 +58,7 @@ * @author xuchao */ -public class HbaseAsyncReqRow extends AsyncReqRow { +public class HbaseAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 2098635104857937717L; @@ -73,13 +73,13 @@ public class HbaseAsyncReqRow extends AsyncReqRow { private transient HBaseClient hBaseClient; - private transient AbsRowKeyModeDealer rowKeyMode; + private transient AbstractRowKeyModeDealer rowKeyMode; private String tableName; private String[] colNames; - public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new HbaseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); tableName = ((HbaseSideTableInfo)sideTableInfo).getTableName(); @@ -89,7 +89,7 @@ public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List colRefType; - public HbaseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { rowKeyBuilder = new RowKeyBuilder(); if(sideTableInfo.getPrimaryKeys().size() < 1){ throw new RuntimeException("Primary key dimension table must be filled"); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java similarity index 87% rename from hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java rename to hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java index 1506bc440..90ee289bd 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.side.hbase.rowkeydealer; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import org.apache.calcite.sql.JoinType; import com.google.common.collect.Maps; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -41,7 +41,7 @@ * @author xuchao */ -public abstract class AbsRowKeyModeDealer { +public abstract class AbstractRowKeyModeDealer { protected Map colRefType; @@ -60,9 +60,9 @@ public abstract class AbsRowKeyModeDealer { protected Map sideFieldIndex = Maps.newHashMap(); - public AbsRowKeyModeDealer(Map colRefType, String[] colNames, HBaseClient hBaseClient, - boolean openCache, JoinType joinType, List outFieldInfoList, - Map inFieldIndex, Map sideFieldIndex){ + public AbstractRowKeyModeDealer(Map colRefType, String[] colNames, HBaseClient hBaseClient, + boolean openCache, JoinType joinType, List outFieldInfoList, + Map inFieldIndex, Map sideFieldIndex){ this.colRefType = colRefType; this.colNames = colNames; this.hBaseClient = hBaseClient; @@ -111,5 +111,5 @@ protected Row fillData(Row input, Object sideInput){ } public abstract void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbsSideCache sideCache); + AbstractSideCache sideCache); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java index a41ffe916..dcd50131d 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.google.common.collect.Maps; @@ -44,7 +44,6 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -55,7 +54,7 @@ * @author xuchao */ -public class PreRowKeyModeDealerDealer extends AbsRowKeyModeDealer { +public class PreRowKeyModeDealerDealer extends AbstractRowKeyModeDealer { private static final Logger LOG = LoggerFactory.getLogger(PreRowKeyModeDealerDealer.class); @@ -67,7 +66,7 @@ public PreRowKeyModeDealerDealer(Map colRefType, String[] colNam @Override public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbsSideCache sideCache) { + AbstractSideCache sideCache) { Scanner prefixScanner = hBaseClient.newScanner(tableName); ScanFilter scanFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(Bytes.UTF8(rowKeyStr))); prefixScanner.setFilter(scanFilter); @@ -80,7 +79,7 @@ public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultF } - private String dealOneRow(ArrayList> args, String rowKeyStr, CRow input, ResultFuture resultFuture, AbsSideCache sideCache) { + private String dealOneRow(ArrayList> args, String rowKeyStr, CRow input, ResultFuture resultFuture, AbstractSideCache sideCache) { if(args == null || args.size() == 0){ dealMissKey(input, resultFuture); if (openCache) { @@ -111,8 +110,7 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, for (String key : colNames) { Object val = sideMap.get(key); if (val == null) { - System.out.println("can't get data with column " + key); - LOG.error("can't get data with column " + key); + LOG.error("can't get data with column {}", key); } sideVal.add(val); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index b20c316db..41208c7f3 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.google.common.collect.Maps; @@ -49,7 +49,7 @@ * @author xuchao */ -public class RowKeyEqualModeDealer extends AbsRowKeyModeDealer { +public class RowKeyEqualModeDealer extends AbstractRowKeyModeDealer { private static final Logger LOG = LoggerFactory.getLogger(RowKeyEqualModeDealer.class); @@ -62,7 +62,7 @@ public RowKeyEqualModeDealer(Map colRefType, String[] colNames, @Override public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbsSideCache sideCache){ + AbstractSideCache sideCache){ //TODO 是否有查询多个col family 和多个col的方法 GetRequest getRequest = new GetRequest(tableName, rowKeyStr); hBaseClient.get(getRequest).addCallbacks(arg -> { @@ -86,8 +86,7 @@ public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultF for(String key : colNames){ Object val = sideMap.get(key); if(val == null){ - System.out.println("can't get data with column " + key); - LOG.error("can't get data with column " + key); + LOG.error("can't get data with column {}", key); } sideVal.add(val); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java similarity index 93% rename from hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java rename to hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java index c3c9b5cfe..c75fa43c1 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java @@ -31,11 +31,11 @@ * @author xuchao */ -public abstract class ReplaceOperator { +public abstract class AbstractReplaceOperator { private EReplaceOpType opType; - public ReplaceOperator(EReplaceOpType opType){ + public AbstractReplaceOperator(EReplaceOpType opType){ this.opType = opType; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java index af5e24dd6..ab826c98f 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java @@ -30,7 +30,7 @@ * @author xuchao */ -public class Md5ReplaceOperator extends ReplaceOperator{ +public class Md5ReplaceOperator extends AbstractReplaceOperator { public Md5ReplaceOperator(EReplaceOpType opType) { super(opType); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java index 834edf9ae..ac9a3b87c 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java @@ -28,5 +28,12 @@ */ public enum EReplaceOpType { - NO_FUNC, MD5_FUNC; + /** + * 没有func + */ + NO_FUNC, + /** + * md5 func + */ + MD5_FUNC; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java index f4b039ef8..bb4b1556b 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java @@ -28,7 +28,16 @@ * @author xuchao */ public enum EReplaceType { + /** + * 参数 + */ PARAM, + /** + * 函数 + */ FUNC, + /** + * 常量 + */ CONSTANT; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 7b627da4d..cf44c181f 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -20,15 +20,15 @@ package com.dtstack.flink.sql.side.hbase.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * hbase field information must include the definition of an alias -> sql which does not allow ":" @@ -37,7 +37,7 @@ * @author xuchao */ -public class HbaseSideParser extends AbsSideTableParser { +public class HbaseSideParser extends AbstractSideTableParser { private final static String FIELD_KEY = "fieldKey"; @@ -58,7 +58,7 @@ public HbaseSideParser() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseSideTableInfo hbaseTableInfo = new HbaseSideTableInfo(); hbaseTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, hbaseTableInfo); @@ -77,7 +77,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map { +public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(HbaseOutputFormat.class); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index 447b02921..9308725a2 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.hbase.table.HbaseTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -30,7 +30,6 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.apache.flink.table.sinks.AppendStreamTableSink; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; @@ -59,7 +58,7 @@ public HbaseSink() { } @Override - public HbaseSink genStreamSink(TargetTableInfo targetTableInfo) { + public HbaseSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { HbaseTableInfo hbaseTableInfo = (HbaseTableInfo) targetTableInfo; this.zookeeperQuorum = hbaseTableInfo.getHost(); this.port = hbaseTableInfo.getPort(); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index 064b13e69..8d8996a23 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.enums.EUpdateMode; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -32,14 +32,14 @@ import java.util.List; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Date: 2018/09/14 * Company: www.dtstack.com * @author sishu.yss */ -public class HbaseSinkParser extends AbsTableParser { +public class HbaseSinkParser extends AbstractTableParser { public static final String HBASE_ZOOKEEPER_QUORUM = "zookeeperQuorum"; @@ -60,7 +60,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseTableInfo hbaseTableInfo = new HbaseTableInfo(); hbaseTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, hbaseTableInfo); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java index 610eb5c30..62f7c9936 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.sink.hbase.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import java.util.Map; @@ -30,7 +30,7 @@ * Company: www.dtstack.com * @author sishu.yss */ -public class HbaseTableInfo extends TargetTableInfo { +public class HbaseTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "hbase"; diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java index 9cd8c9194..10938308a 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java @@ -20,12 +20,10 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; -import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.JDBCUtils; -import com.google.common.collect.Maps; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -36,7 +34,6 @@ import java.sql.Connection; import java.sql.DriverManager; import java.util.List; -import java.util.Map; /** * side operator with cache for all(period reload) @@ -46,7 +43,7 @@ * @author xiuzhu */ -public class ImpalaAllReqRow extends RdbAllReqRow { +public class ImpalaAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -56,13 +53,13 @@ public class ImpalaAllReqRow extends RdbAllReqRow { private ImpalaSideTableInfo impalaSideTableInfo; - public ImpalaAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ImpalaAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); this.impalaSideTableInfo = (ImpalaSideTableInfo) sideTableInfo; } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Connection connection ; String url = getUrl(); diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java index d40c5f48c..a5e643967 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java @@ -20,10 +20,9 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; -import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -32,7 +31,7 @@ public class ImpalaAllSideInfo extends RdbAllSideInfo { - public ImpalaAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -61,7 +60,7 @@ private String buildPartitionCondition(ImpalaSideTableInfo impalaSideTableInfo) private String getPartitionVaule(String fieldType, List values) { String partitionVaule = values.stream().map(val -> { - return (fieldType.toLowerCase().equals("string") || fieldType.toLowerCase().equals("varchar")) ? "'" + val + "'" : val.toString(); + return ("string".equals(fieldType.toLowerCase()) || "varchar".equals(fieldType.toLowerCase())) ? "'" + val + "'" : val.toString(); }).collect(Collectors.joining(" , ")).toString(); return partitionVaule; diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 2b76ec26c..b5e0ed5fb 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import io.vertx.core.Vertx; @@ -50,7 +50,7 @@ public class ImpalaAsyncReqRow extends RdbAsyncReqRow { private final static String IMPALA_DRIVER = "com.cloudera.impala.jdbc41.Driver"; - public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ImpalaAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -66,7 +66,10 @@ public void open(Configuration parameters) throws Exception { .put("provider_class", DT_PROVIDER_CLASS) .put("idle_connection_test_period", 300) .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN) - .put("max_idle_time", 600); + .put("max_idle_time", 600) + .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) + .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) + .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); System.setProperty("vertx.disableFileCPResolving", "true"); @@ -75,8 +78,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(impalaSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - - setRdbSQLClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); } diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java index 53559b9a7..14fbf0ed3 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -38,7 +38,7 @@ public class ImpalaAsyncSideInfo extends RdbAsyncSideInfo { - public ImpalaAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -69,7 +69,7 @@ private String buildPartitionCondition(ImpalaSideTableInfo impalaSideTableInfo) private String getPartitionVaule(String fieldType, List values) { String partitionVaule = values.stream().map(val -> { - return (fieldType.toLowerCase().equals("string") || fieldType.toLowerCase().equals("varchar")) ? "'" + val + "'" : val.toString(); + return ("string".equals(fieldType.toLowerCase()) || "varchar".equals(fieldType.toLowerCase())) ? "'" + val + "'" : val.toString(); }).collect(Collectors.joining(" , ")).toString(); return partitionVaule; diff --git a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java index b6a54ca3f..2f1d73be7 100644 --- a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java +++ b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.impala.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang3.StringUtils; @@ -48,7 +48,7 @@ public class ImpalaSideParser extends RdbSideParser { private static final String CURR_TYPE = "impala"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ImpalaSideTableInfo impalaSideTableInfo = new ImpalaSideTableInfo(); impalaSideTableInfo.setType(CURR_TYPE); impalaSideTableInfo.setName(tableName); @@ -147,6 +147,8 @@ public Class dbTypeConvertToJavaType(String fieldType) { return String.class; case "timestamp": return Timestamp.class; + default: + break; } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index 41fa00926..cd1e1e945 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -21,9 +21,9 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.impala.table.ImpalaTableInfo; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -36,7 +36,7 @@ * @author xiuzhu */ -public class ImpalaSink extends RdbSink implements IStreamSinkGener { +public class ImpalaSink extends AbstractRdbSink implements IStreamSinkGener { private ImpalaTableInfo impalaTableInfo; @@ -47,7 +47,7 @@ public ImpalaSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(getImpalaJdbcUrl()) + .setDbUrl(getImpalaJdbcUrl()) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) @@ -70,8 +70,8 @@ public JDBCUpsertOutputFormat getOutputFormat() { public String getImpalaJdbcUrl() { Integer authMech = impalaTableInfo.getAuthMech(); - String newUrl = dbURL; - StringBuffer urlBuffer = new StringBuffer(dbURL); + String newUrl = dbUrl; + StringBuffer urlBuffer = new StringBuffer(dbUrl); if (authMech == EAuthMech.NoAuthentication.getType()) { return newUrl; } else if (authMech == EAuthMech.Kerberos.getType()) { @@ -121,7 +121,7 @@ public String getImpalaJdbcUrl() { } @Override - public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { + public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { super.genStreamSink(targetTableInfo); this.impalaTableInfo = (ImpalaTableInfo) targetTableInfo; return this; diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java index 4921f5e51..5b6fdeafe 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java @@ -19,12 +19,11 @@ package com.dtstack.flink.sql.sink.impala.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; import java.sql.Timestamp; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -42,7 +41,7 @@ public class ImpalaSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "impala"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ImpalaTableInfo impalaTableInfo = new ImpalaTableInfo(); impalaTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, impalaTableInfo); @@ -117,6 +116,8 @@ public Class dbTypeConvertToJavaType(String fieldType) { return String.class; case "timestamp": return Timestamp.class; + default: + break; } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java index 88c2ca939..9958a2544 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java @@ -19,16 +19,16 @@ import com.dtstack.flink.sql.format.FormatType; import com.dtstack.flink.sql.format.SerializationMetricWrapper; +import com.dtstack.flink.sql.sink.kafka.serialization.AvroCRowSerializationSchema; +import com.dtstack.flink.sql.sink.kafka.serialization.CsvCRowSerializationSchema; +import com.dtstack.flink.sql.sink.kafka.serialization.JsonCRowSerializationSchema; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.formats.avro.AvroRowSerializationSchema; -import org.apache.flink.formats.csv.CsvRowSerializationSchema; -import org.apache.flink.formats.json.JsonRowSerializationSchema; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import java.util.Optional; import java.util.Properties; @@ -51,42 +51,37 @@ public abstract class AbstractKafkaProducerFactory { * @param partitioner * @return */ - public abstract RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys); + public abstract RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys); - protected SerializationMetricWrapper createSerializationMetricWrapper(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { - return new SerializationMetricWrapper(createSerializationSchema(kafkaSinkTableInfo, typeInformation)); + protected SerializationMetricWrapper createSerializationMetricWrapper(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { + SerializationSchema serializationSchema = createSerializationSchema(kafkaSinkTableInfo, typeInformation); + return new SerializationMetricWrapper(serializationSchema); } - private SerializationSchema createSerializationSchema(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { - SerializationSchema serializationSchema = null; + private SerializationSchema createSerializationSchema(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { + SerializationSchema serializationSchema = null; if (FormatType.JSON.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { - if (StringUtils.isNotBlank(kafkaSinkTableInfo.getSchemaString())) { - serializationSchema = new JsonRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); + serializationSchema = new JsonCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(), kafkaSinkTableInfo.getUpdateMode()); } else if (typeInformation != null && typeInformation.getArity() != 0) { - serializationSchema = new JsonRowSerializationSchema(typeInformation); + serializationSchema = new JsonCRowSerializationSchema(typeInformation, kafkaSinkTableInfo.getUpdateMode()); } else { throw new IllegalArgumentException("sinkDataType:" + FormatType.JSON.name() + " must set schemaString（JSON Schema）or TypeInformation"); } - } else if (FormatType.CSV.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { - if (StringUtils.isBlank(kafkaSinkTableInfo.getFieldDelimiter())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.CSV.name() + " must set fieldDelimiter"); } - - final CsvRowSerializationSchema.Builder serSchemaBuilder = new CsvRowSerializationSchema.Builder(typeInformation); + final CsvCRowSerializationSchema.Builder serSchemaBuilder = new CsvCRowSerializationSchema.Builder(typeInformation); serSchemaBuilder.setFieldDelimiter(kafkaSinkTableInfo.getFieldDelimiter().toCharArray()[0]); - serializationSchema = serSchemaBuilder.build(); + serSchemaBuilder.setUpdateMode(kafkaSinkTableInfo.getUpdateMode()); + serializationSchema = serSchemaBuilder.build(); } else if (FormatType.AVRO.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { - if (StringUtils.isBlank(kafkaSinkTableInfo.getSchemaString())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.AVRO.name() + " must set schemaString"); } - - serializationSchema = new AvroRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); - + serializationSchema = new AvroCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(), kafkaSinkTableInfo.getUpdateMode()); } if (null == serializationSchema) { diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java new file mode 100644 index 000000000..7234216a7 --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka; + +import com.dtstack.flink.sql.sink.IStreamSinkGener; +import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.functions.sink.SinkFunction; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; +import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; +import org.apache.kafka.clients.consumer.ConsumerConfig; + +import java.util.Optional; +import java.util.Properties; +import java.util.stream.IntStream; + +/** + * Date: 2020/4/1 + * Company: www.dtstack.com + * @author maqi + */ +public abstract class AbstractKafkaSink implements RetractStreamTableSink, IStreamSinkGener { + + public static final String SINK_OPERATOR_NAME_TPL = "${topic}_${table}"; + + protected String[] fieldNames; + protected TypeInformation[] fieldTypes; + + protected String[] partitionKeys; + protected String sinkOperatorName; + protected Properties properties; + protected int parallelism; + protected String topic; + protected String tableName; + + protected TableSchema schema; + protected SinkFunction kafkaProducer; + + + protected Optional> partitioner; + + protected Properties getKafkaProperties(KafkaSinkTableInfo KafkaSinkTableInfo) { + Properties props = new Properties(); + props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KafkaSinkTableInfo.getBootstrapServers()); + + for (String key : KafkaSinkTableInfo.getKafkaParamKeys()) { + props.setProperty(key, KafkaSinkTableInfo.getKafkaParam(key)); + } + return props; + } + + protected TypeInformation[] getTypeInformations(KafkaSinkTableInfo kafka11SinkTableInfo) { + Class[] fieldClasses = kafka11SinkTableInfo.getFieldClasses(); + TypeInformation[] types = IntStream.range(0, fieldClasses.length) + .mapToObj(i -> TypeInformation.of(fieldClasses[i])) + .toArray(TypeInformation[]::new); + return types; + } + + + protected TableSchema buildTableSchema(String[] fieldNames, TypeInformation[] fieldTypes) { + Preconditions.checkArgument(fieldNames.length == fieldTypes.length, "fieldNames length must equals fieldTypes length !"); + + TableSchema.Builder builder = TableSchema.builder(); + IntStream.range(0, fieldTypes.length) + .forEach(i -> builder.field(fieldNames[i], fieldTypes[i])); + + return builder.build(); + } + + + @Override + public void emitDataStream(DataStream> dataStream) { + DataStream mapDataStream = dataStream + .map((Tuple2 record) -> new CRow(record.f1, record.f0)) + .returns(getRowTypeInfo()) + .setParallelism(parallelism); + + mapDataStream.addSink(kafkaProducer).name(sinkOperatorName); + } + + public CRowTypeInfo getRowTypeInfo() { + return new CRowTypeInfo(new RowTypeInfo(fieldTypes, fieldNames)); + } + + protected String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo) { + if (StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())) { + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } + + @Override + public TupleTypeInfo> getOutputType() { + return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); + } + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } + + @Override + public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + return this; + } + + @Override + public TypeInformation getRecordType() { + return new RowTypeInfo(fieldTypes, fieldNames); + } + + +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index e212d1f57..90dfe996b 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -11,13 +11,13 @@ public class CustomerFlinkPartition extends FlinkKafkaPartitioner { public CustomerFlinkPartition() { } - + @Override public void open(int parallelInstanceId, int parallelInstances) { Preconditions.checkArgument(parallelInstanceId >= 0, "Id of this subtask cannot be negative."); Preconditions.checkArgument(parallelInstances > 0, "Number of subtasks must be larger than 0."); this.parallelInstanceId = parallelInstanceId; } - + @Override public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); if(key == null){ @@ -25,11 +25,11 @@ public int partition(T record, byte[] key, byte[] value, String targetTopic, int } return partitions[Math.abs(new String(key).hashCode()) % partitions.length]; } - + @Override public boolean equals(Object o) { return this == o || o instanceof CustomerFlinkPartition; } - + @Override public int hashCode() { return CustomerFlinkPartition.class.hashCode(); } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index 498766564..cde8d1b1d 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -2,18 +2,20 @@ import com.dtstack.flink.sql.format.SerializationMetricWrapper; +import com.dtstack.flink.sql.sink.kafka.serialization.JsonCRowSerializationSchema; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.formats.json.JsonRowSerializationSchema; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.concurrent.atomic.AtomicLong; -public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { +public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { private static final Logger LOG = LoggerFactory.getLogger(CustomerKeyedSerializationSchema.class); @@ -30,38 +32,41 @@ public CustomerKeyedSerializationSchema(SerializationMetricWrapper serialization this.mapper = new ObjectMapper(); } - public byte[] serializeKey(Row element) { - if(partitionKeys == null || partitionKeys.length <=0){ + @Override + public byte[] serializeKey(CRow element) { + if (partitionKeys == null || partitionKeys.length <= 0) { return null; - } - SerializationSchema serializationSchema = serializationMetricWrapper.getSerializationSchema(); - if(serializationSchema instanceof JsonRowSerializationSchema){ - return serializeJsonKey((JsonRowSerializationSchema) serializationSchema, element); + } + SerializationSchema serializationSchema = serializationMetricWrapper.getSerializationSchema(); + if (serializationSchema instanceof JsonCRowSerializationSchema) { + return serializeJsonKey((JsonCRowSerializationSchema) serializationSchema, element); } return null; } - public byte[] serializeValue(Row element) { + @Override + public byte[] serializeValue(CRow element) { return this.serializationMetricWrapper.serialize(element); } - public String getTargetTopic(Row element) { + @Override + public String getTargetTopic(CRow element) { return null; } - private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationSchema, Row element) { + private byte[] serializeJsonKey(JsonCRowSerializationSchema jsonCRowSerializationSchema, CRow element) { try { - byte[] data = jsonRowSerializationSchema.serialize(element); + byte[] data = jsonCRowSerializationSchema.serialize(element); ObjectNode objectNode = mapper.readValue(data, ObjectNode.class); StringBuilder sb = new StringBuilder(); - for(String key : partitionKeys){ - if(objectNode.has(key)){ + for (String key : partitionKeys) { + if (objectNode.has(key)) { sb.append(objectNode.get(key.trim())); } } return sb.toString().getBytes(); - } catch (Exception e){ - if(COUNTER.getAndIncrement() % 1000 == 0){ + } catch (Exception e) { + if (COUNTER.getAndIncrement() % 1000 == 0) { LOG.error("serializeJsonKey error", e); } } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java new file mode 100644 index 000000000..692e208b5 --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka.serialization; + +import com.dtstack.flink.sql.enums.EUpdateMode; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.SchemaParseException; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.avro.specific.SpecificRecord; +import org.apache.avro.util.Utf8; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.TimeZone; +import java.util.stream.Collectors; + +/** + * Serialization schema that serializes CROW into Avro bytes. + * + *

Serializes objects that are represented in (nested) Flink rows. It support types that + * are compatible with Flink's Table & SQL API. + ** + * @author maqi + */ +public class AvroCRowSerializationSchema implements SerializationSchema { + + /** + * Used for time conversions from SQL types. + */ + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + + /** + * Avro record class for serialization. Might be null if record class is not available. + */ + private Class recordClazz; + + /** + * Schema string for deserialization. + */ + private String schemaString; + + /** + * Avro serialization schema. + */ + private transient Schema schema; + + /** + * Writer to serialize Avro record into a byte array. + */ + private transient DatumWriter datumWriter; + + /** + * Output stream to serialize records into byte array. + */ + private transient ByteArrayOutputStream arrayOutputStream; + + /** + * Low-level class for serialization of Avro values. + */ + private transient Encoder encoder; + + private String updateMode; + + private String retractKey = "retract"; + + /** + * Creates an Avro serialization schema for the given specific record class. + * + * @param recordClazz Avro record class used to serialize Flink's row to Avro's record + */ + public AvroCRowSerializationSchema(Class recordClazz, String updateMode) { + Preconditions.checkNotNull(recordClazz, "Avro record class must not be null."); + this.recordClazz = recordClazz; + this.schema = SpecificData.get().getSchema(recordClazz); + this.schemaString = schema.toString(); + this.datumWriter = new SpecificDatumWriter<>(schema); + this.arrayOutputStream = new ByteArrayOutputStream(); + this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); + this.updateMode = updateMode; + } + + /** + * Creates an Avro serialization schema for the given Avro schema string. + * + * @param avroSchemaString Avro schema string used to serialize Flink's row to Avro's record + */ + public AvroCRowSerializationSchema(String avroSchemaString,String updateMode) { + Preconditions.checkNotNull(avroSchemaString, "Avro schema must not be null."); + this.recordClazz = null; + this.schemaString = avroSchemaString; + try { + this.schema = new Schema.Parser().parse(avroSchemaString); + } catch (SchemaParseException e) { + throw new IllegalArgumentException("Could not parse Avro schema string.", e); + } + this.datumWriter = new GenericDatumWriter<>(schema); + this.arrayOutputStream = new ByteArrayOutputStream(); + this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); + this.updateMode = updateMode; + } + + @Override + public byte[] serialize(CRow crow) { + try { + Row row = crow.row(); + boolean change = crow.change(); + + // convert to record + final GenericRecord record = convertRowToAvroRecord(schema, row); + + dealRetractField(change, record); + + arrayOutputStream.reset(); + datumWriter.write(record, encoder); + encoder.flush(); + return arrayOutputStream.toByteArray(); + } catch (Exception e) { + throw new RuntimeException("Failed to serialize row.", e); + } + } + + protected void dealRetractField(boolean change, GenericRecord record) { + schema.getFields() + .stream() + .filter(field -> StringUtils.equalsIgnoreCase(field.name(), retractKey)) + .findFirst() + .ifPresent(field -> { + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + record.put(retractKey, convertFlinkType(field.schema(), change)); + } + }); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final AvroCRowSerializationSchema that = (AvroCRowSerializationSchema) o; + return Objects.equals(recordClazz, that.recordClazz) && Objects.equals(schemaString, that.schemaString); + } + + @Override + public int hashCode() { + return Objects.hash(recordClazz, schemaString); + } + + // -------------------------------------------------------------------------------------------- + + private GenericRecord convertRowToAvroRecord(Schema schema, Row row) { + + final List fields = schema.getFields() + .stream() + .filter(field -> !StringUtils.equalsIgnoreCase(field.name(), retractKey)) + .collect(Collectors.toList()); + + final int length = fields.size(); + final GenericRecord record = new GenericData.Record(schema); + for (int i = 0; i < length; i++) { + final Schema.Field field = fields.get(i); + record.put(i, convertFlinkType(field.schema(), row.getField(i))); + } + return record; + } + + private Object convertFlinkType(Schema schema, Object object) { + if (object == null) { + return null; + } + switch (schema.getType()) { + case RECORD: + if (object instanceof Row) { + return convertRowToAvroRecord(schema, (Row) object); + } + throw new IllegalStateException("Row expected but was: " + object.getClass()); + case ENUM: + return new GenericData.EnumSymbol(schema, object.toString()); + case ARRAY: + final Schema elementSchema = schema.getElementType(); + final Object[] array = (Object[]) object; + final GenericData.Array convertedArray = new GenericData.Array<>(array.length, schema); + for (Object element : array) { + convertedArray.add(convertFlinkType(elementSchema, element)); + } + return convertedArray; + case MAP: + final Map map = (Map) object; + final Map convertedMap = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + convertedMap.put( + new Utf8(entry.getKey().toString()), + convertFlinkType(schema.getValueType(), entry.getValue())); + } + return convertedMap; + case UNION: + final List types = schema.getTypes(); + final int size = types.size(); + final Schema actualSchema; + if (size == 2 && types.get(0).getType() == Schema.Type.NULL) { + actualSchema = types.get(1); + } else if (size == 2 && types.get(1).getType() == Schema.Type.NULL) { + actualSchema = types.get(0); + } else if (size == 1) { + actualSchema = types.get(0); + } else { + // generic type + return object; + } + return convertFlinkType(actualSchema, object); + case FIXED: + // check for logical type + if (object instanceof BigDecimal) { + return new GenericData.Fixed( + schema, + convertFromDecimal(schema, (BigDecimal) object)); + } + return new GenericData.Fixed(schema, (byte[]) object); + case STRING: + return new Utf8(object.toString()); + case BYTES: + // check for logical type + if (object instanceof BigDecimal) { + return ByteBuffer.wrap(convertFromDecimal(schema, (BigDecimal) object)); + } + return ByteBuffer.wrap((byte[]) object); + case INT: + // check for logical types + if (object instanceof Date) { + return convertFromDate(schema, (Date) object); + } else if (object instanceof Time) { + return convertFromTime(schema, (Time) object); + } + return object; + case LONG: + // check for logical type + if (object instanceof Timestamp) { + return convertFromTimestamp(schema, (Timestamp) object); + } + return object; + case FLOAT: + case DOUBLE: + case BOOLEAN: + return object; + } + throw new RuntimeException("Unsupported Avro type:" + schema); + } + + private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType instanceof LogicalTypes.Decimal) { + final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType; + // rescale to target type + final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY); + // byte array must contain the two's-complement representation of the + // unscaled integer value in big-endian byte order + return decimal.unscaledValue().toByteArray(); + } else { + throw new RuntimeException("Unsupported decimal type."); + } + } + + private int convertFromDate(Schema schema, Date date) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType == LogicalTypes.date()) { + // adopted from Apache Calcite + final long time = date.getTime(); + final long converted = time + (long) LOCAL_TZ.getOffset(time); + return (int) (converted / 86400000L); + } else { + throw new RuntimeException("Unsupported date type."); + } + } + + private int convertFromTime(Schema schema, Time date) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType == LogicalTypes.timeMillis()) { + // adopted from Apache Calcite + final long time = date.getTime(); + final long converted = time + (long) LOCAL_TZ.getOffset(time); + return (int) (converted % 86400000L); + } else { + throw new RuntimeException("Unsupported time type."); + } + } + + private long convertFromTimestamp(Schema schema, Timestamp date) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType == LogicalTypes.timestampMillis()) { + // adopted from Apache Calcite + final long time = date.getTime(); + return time + (long) LOCAL_TZ.getOffset(time); + } else { + throw new RuntimeException("Unsupported timestamp type."); + } + } + + private void writeObject(ObjectOutputStream outputStream) throws IOException { + outputStream.writeObject(recordClazz); + outputStream.writeObject(schemaString); // support for null + outputStream.writeObject(retractKey); + outputStream.writeObject(updateMode); + } + + @SuppressWarnings("unchecked") + private void readObject(ObjectInputStream inputStream) throws ClassNotFoundException, IOException { + recordClazz = (Class) inputStream.readObject(); + schemaString = (String) inputStream.readObject(); + if (recordClazz != null) { + schema = SpecificData.get().getSchema(recordClazz); + } else { + schema = new Schema.Parser().parse(schemaString); + } + retractKey = (String) inputStream.readObject(); + updateMode = (String) inputStream.readObject(); + + datumWriter = new SpecificDatumWriter<>(schema); + arrayOutputStream = new ByteArrayOutputStream(); + encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); + } +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java new file mode 100644 index 000000000..4e57b6f2a --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java @@ -0,0 +1,374 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka.serialization; + +import com.dtstack.flink.sql.enums.EUpdateMode; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.formats.csv.CsvRowDeserializationSchema; +import org.apache.flink.formats.csv.CsvRowSchemaConverter; +import org.apache.flink.formats.csv.CsvRowSerializationSchema; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectWriter; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Objects; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +/** + * Serialization schema that serializes an object of Flink types into a CSV bytes. + * + *

Serializes the input row into a {@link ObjectNode} and + * converts it into byte[]. + * + *

Result byte[] messages can be deserialized using {@link CsvRowDeserializationSchema}. + */ +@PublicEvolving +public final class CsvCRowSerializationSchema implements SerializationSchema { + + private static final long serialVersionUID = 2098447220136965L; + + /** Type information describing the input CSV data. */ + private final RowTypeInfo typeInfo; + + /** Runtime instance that performs the actual work. */ + private final RuntimeConverter runtimeConverter; + + /** CsvMapper used to write {@link JsonNode} into bytes. */ + private final CsvMapper csvMapper; + + /** Schema describing the input CSV data. */ + private CsvSchema csvSchema; + + /** Object writer used to write rows. It is configured by {@link CsvSchema}. */ + private ObjectWriter objectWriter; + + /** Reusable object node. */ + private transient ObjectNode root; + + private String updateMode; + + private String retractKey = "retract"; + + private CsvCRowSerializationSchema( + RowTypeInfo typeInfo, + CsvSchema csvSchema, + String updateMode) { + this.typeInfo = typeInfo; + this.runtimeConverter = createRowRuntimeConverter(typeInfo, true); + this.csvMapper = new CsvMapper(); + this.csvSchema = csvSchema; + this.updateMode = updateMode; + this.objectWriter = csvMapper.writer(csvSchema); + } + + /** + * A builder for creating a {@link CsvRowSerializationSchema}. + */ + @PublicEvolving + public static class Builder { + + private final RowTypeInfo typeInfo; + private CsvSchema csvSchema; + private String updateMode; + + /** + * Creates a {@link CsvRowSerializationSchema} expecting the given {@link TypeInformation}. + * + * @param typeInfo type information used to create schema. + */ + public Builder(TypeInformation typeInfo) { + Preconditions.checkNotNull(typeInfo, "Type information must not be null."); + + if (!(typeInfo instanceof CRowTypeInfo)) { + throw new IllegalArgumentException("Row type information expected."); + } + RowTypeInfo rowTypeInfo = ((CRowTypeInfo) typeInfo).rowType(); + this.typeInfo = rowTypeInfo; + this.csvSchema = CsvRowSchemaConverter.convert(rowTypeInfo); + } + + public Builder setFieldDelimiter(char c) { + this.csvSchema = this.csvSchema.rebuild().setColumnSeparator(c).build(); + return this; + } + + public Builder setLineDelimiter(String delimiter) { + Preconditions.checkNotNull(delimiter, "Delimiter must not be null."); + if (!delimiter.equals("\n") && !delimiter.equals("\r") && !delimiter.equals("\r\n")) { + throw new IllegalArgumentException( + "Unsupported new line delimiter. Only \\n, \\r, or \\r\\n are supported."); + } + this.csvSchema = this.csvSchema.rebuild().setLineSeparator(delimiter).build(); + return this; + } + + public Builder setArrayElementDelimiter(String delimiter) { + Preconditions.checkNotNull(delimiter, "Delimiter must not be null."); + this.csvSchema = this.csvSchema.rebuild().setArrayElementSeparator(delimiter).build(); + return this; + } + + public Builder setQuoteCharacter(char c) { + this.csvSchema = this.csvSchema.rebuild().setQuoteChar(c).build(); + return this; + } + + public Builder setEscapeCharacter(char c) { + this.csvSchema = this.csvSchema.rebuild().setEscapeChar(c).build(); + return this; + } + + public Builder setNullLiteral(String s) { + this.csvSchema = this.csvSchema.rebuild().setNullValue(s).build(); + return this; + } + + public Builder setUpdateMode(String updateMode) { + this.updateMode = updateMode; + return this; + } + + public CsvCRowSerializationSchema build() { + return new CsvCRowSerializationSchema( + typeInfo, + csvSchema, + updateMode); + } + } + + @Override + public byte[] serialize(CRow crow) { + Row row = crow.row(); + boolean change = crow.change(); + if (root == null) { + root = csvMapper.createObjectNode(); + } + try { + runtimeConverter.convert(csvMapper, root, row); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + fillRetractField(row, change); + } + + return objectWriter.writeValueAsBytes(root); + } catch (Throwable t) { + throw new RuntimeException("Could not serialize row '" + row + "'.", t); + } + } + + protected void fillRetractField(Row row, boolean change) { + root.put(retractKey, change); + CsvSchema.Builder newBuilder = new CsvSchema.Builder(csvSchema); + + CsvSchema.Column retractColumn = new CsvSchema.Column(row.getArity(), retractKey, CsvSchema.ColumnType.BOOLEAN); + newBuilder.addColumn(retractColumn); + csvSchema = newBuilder.build(); + + this.objectWriter = csvMapper.writer(csvSchema); + + } + + @Override + public boolean equals(Object o) { + if (o == null || o.getClass() != this.getClass()) { + return false; + } + if (this == o) { + return true; + } + final CsvCRowSerializationSchema that = (CsvCRowSerializationSchema) o; + final CsvSchema otherSchema = that.csvSchema; + + return typeInfo.equals(that.typeInfo) && + csvSchema.getColumnSeparator() == otherSchema.getColumnSeparator() && + Arrays.equals(csvSchema.getLineSeparator(), otherSchema.getLineSeparator()) && + csvSchema.getArrayElementSeparator().equals(otherSchema.getArrayElementSeparator()) && + csvSchema.getQuoteChar() == otherSchema.getQuoteChar() && + csvSchema.getEscapeChar() == otherSchema.getEscapeChar() && + Arrays.equals(csvSchema.getNullValue(), otherSchema.getNullValue()); + } + + @Override + public int hashCode() { + return Objects.hash( + typeInfo, + csvSchema.getColumnSeparator(), + csvSchema.getLineSeparator(), + csvSchema.getArrayElementSeparator(), + csvSchema.getQuoteChar(), + csvSchema.getEscapeChar(), + csvSchema.getNullValue()); + } + + // -------------------------------------------------------------------------------------------- + + private interface RuntimeConverter extends Serializable { + JsonNode convert(CsvMapper csvMapper, ContainerNode container, Object obj); + } + + private static RuntimeConverter createRowRuntimeConverter(RowTypeInfo rowTypeInfo, boolean isTopLevel) { + final TypeInformation[] fieldTypes = rowTypeInfo.getFieldTypes(); + final String[] fieldNames = rowTypeInfo.getFieldNames(); + + final RuntimeConverter[] fieldConverters = createFieldRuntimeConverters(fieldTypes); + + return assembleRowRuntimeConverter(isTopLevel, fieldNames, fieldConverters); + } + + private static RuntimeConverter[] createFieldRuntimeConverters(TypeInformation[] fieldTypes) { + final RuntimeConverter[] fieldConverters = new RuntimeConverter[fieldTypes.length]; + for (int i = 0; i < fieldTypes.length; i++) { + fieldConverters[i] = createNullableRuntimeConverter(fieldTypes[i]); + } + return fieldConverters; + } + + private static RuntimeConverter assembleRowRuntimeConverter( + boolean isTopLevel, + String[] fieldNames, + RuntimeConverter[] fieldConverters) { + final int rowArity = fieldNames.length; + // top level reuses the object node container + if (isTopLevel) { + return (csvMapper, container, obj) -> { + final Row row = (Row) obj; + + validateArity(rowArity, row.getArity()); + + final ObjectNode objectNode = (ObjectNode) container; + for (int i = 0; i < rowArity; i++) { + objectNode.set( + fieldNames[i], + fieldConverters[i].convert(csvMapper, container, row.getField(i))); + } + return objectNode; + }; + } else { + return (csvMapper, container, obj) -> { + final Row row = (Row) obj; + + validateArity(rowArity, row.getArity()); + + final ArrayNode arrayNode = csvMapper.createArrayNode(); + for (int i = 0; i < rowArity; i++) { + arrayNode.add(fieldConverters[i].convert(csvMapper, arrayNode, row.getField(i))); + } + return arrayNode; + }; + } + } + + private static RuntimeConverter createNullableRuntimeConverter(TypeInformation info) { + final RuntimeConverter valueConverter = createRuntimeConverter(info); + return (csvMapper, container, obj) -> { + if (obj == null) { + return container.nullNode(); + } + return valueConverter.convert(csvMapper, container, obj); + }; + } + + private static RuntimeConverter createRuntimeConverter(TypeInformation info) { + if (info.equals(Types.VOID)) { + return (csvMapper, container, obj) -> container.nullNode(); + } else if (info.equals(Types.STRING)) { + return (csvMapper, container, obj) -> container.textNode((String) obj); + } else if (info.equals(Types.BOOLEAN)) { + return (csvMapper, container, obj) -> container.booleanNode((Boolean) obj); + } else if (info.equals(Types.BYTE)) { + return (csvMapper, container, obj) -> container.numberNode((Byte) obj); + } else if (info.equals(Types.SHORT)) { + return (csvMapper, container, obj) -> container.numberNode((Short) obj); + } else if (info.equals(Types.INT)) { + return (csvMapper, container, obj) -> container.numberNode((Integer) obj); + } else if (info.equals(Types.LONG)) { + return (csvMapper, container, obj) -> container.numberNode((Long) obj); + } else if (info.equals(Types.FLOAT)) { + return (csvMapper, container, obj) -> container.numberNode((Float) obj); + } else if (info.equals(Types.DOUBLE)) { + return (csvMapper, container, obj) -> container.numberNode((Double) obj); + } else if (info.equals(Types.BIG_DEC)) { + return (csvMapper, container, obj) -> container.numberNode((BigDecimal) obj); + } else if (info.equals(Types.BIG_INT)) { + return (csvMapper, container, obj) -> container.numberNode((BigInteger) obj); + } else if (info.equals(Types.SQL_DATE)) { + return (csvMapper, container, obj) -> container.textNode(obj.toString()); + } else if (info.equals(Types.SQL_TIME)) { + return (csvMapper, container, obj) -> container.textNode(obj.toString()); + } else if (info.equals(Types.SQL_TIMESTAMP)) { + return (csvMapper, container, obj) -> container.textNode(obj.toString()); + } else if (info instanceof RowTypeInfo){ + return createRowRuntimeConverter((RowTypeInfo) info, false); + } else if (info instanceof BasicArrayTypeInfo) { + return createObjectArrayRuntimeConverter(((BasicArrayTypeInfo) info).getComponentInfo()); + } else if (info instanceof ObjectArrayTypeInfo) { + return createObjectArrayRuntimeConverter(((ObjectArrayTypeInfo) info).getComponentInfo()); + } else if (info instanceof PrimitiveArrayTypeInfo && + ((PrimitiveArrayTypeInfo) info).getComponentType() == Types.BYTE) { + return createByteArrayRuntimeConverter(); + } + else { + throw new RuntimeException("Unsupported type information '" + info + "'."); + } + } + + private static RuntimeConverter createObjectArrayRuntimeConverter(TypeInformation elementType) { + final RuntimeConverter elementConverter = createNullableRuntimeConverter(elementType); + return (csvMapper, container, obj) -> { + final Object[] array = (Object[]) obj; + final ArrayNode arrayNode = csvMapper.createArrayNode(); + for (Object element : array) { + arrayNode.add(elementConverter.convert(csvMapper, arrayNode, element)); + } + return arrayNode; + }; + } + + private static RuntimeConverter createByteArrayRuntimeConverter() { + return (csvMapper, container, obj) -> container.binaryNode((byte[]) obj); + } + + private static void validateArity(int expected, int actual) { + if (expected != actual) { + throw new RuntimeException("Row length mismatch. " + expected + + " fields expected but was " + actual + "."); + } + } +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java new file mode 100644 index 000000000..bfe801d52 --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka.serialization; + +import com.dtstack.flink.sql.enums.EUpdateMode; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.formats.json.JsonRowSchemaConverter; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Time; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.util.Objects; + +/** + * + * Serialization schema that serializes an object of Flink types into a JSON bytes. + * + *

Serializes the input Flink object into a JSON string and + * converts it into byte[]. + * + */ +public class JsonCRowSerializationSchema implements SerializationSchema { + + private static final long serialVersionUID = -2885556750743978636L; + + /** Type information describing the input type. */ + private final TypeInformation typeInfo; + + /** Object mapper that is used to create output JSON objects. */ + private final ObjectMapper mapper = new ObjectMapper(); + + /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone, without milliseconds). */ + private SimpleDateFormat timeFormat = new SimpleDateFormat("HH:mm:ss'Z'"); + + /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone). */ + private SimpleDateFormat timeFormatWithMillis = new SimpleDateFormat("HH:mm:ss.SSS'Z'"); + + /** Formatter for RFC 3339-compliant string representation of a timestamp value (with UTC timezone). */ + private SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + + /** Reusable object node. */ + private transient ObjectNode node; + + private String updateMode; + + private final String retractKey = "retract"; + + public JsonCRowSerializationSchema(String jsonSchema, String updateMode) { + this(JsonRowSchemaConverter.convert(jsonSchema), updateMode); + } + + /** + * Creates a JSON serialization schema for the given type information. + * + * @param typeInfo The field names of {@link Row} are used to map to JSON properties. + */ + public JsonCRowSerializationSchema(TypeInformation typeInfo, String updateMode) { + Preconditions.checkNotNull(typeInfo, "Type information"); + this.typeInfo = typeInfo; + this.updateMode = updateMode; + } + + + @Override + public byte[] serialize(CRow crow) { + Row row = crow.row(); + boolean change = crow.change(); + if (node == null) { + node = mapper.createObjectNode(); + } + + RowTypeInfo rowTypeInfo = ((CRowTypeInfo) typeInfo).rowType(); + try { + convertRow(node, rowTypeInfo, row); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + node.put(retractKey, change); + } + return mapper.writeValueAsBytes(node); + } catch (Throwable t) { + throw new RuntimeException("Could not serialize row '" + row + "'. " + + "Make sure that the schema matches the input.", t); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final JsonCRowSerializationSchema that = (JsonCRowSerializationSchema) o; + return Objects.equals(typeInfo, that.typeInfo); + } + + @Override + public int hashCode() { + return Objects.hash(typeInfo); + } + + // -------------------------------------------------------------------------------------------- + + private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) { + if (reuse == null) { + reuse = mapper.createObjectNode(); + } + final String[] fieldNames = info.getFieldNames(); + + final TypeInformation[] fieldTypes = info.getFieldTypes(); + + // validate the row + if (row.getArity() != fieldNames.length) { + throw new IllegalStateException(String.format( + "Number of elements in the row '%s' is different from number of field names: %d", row, fieldNames.length)); + } + + for (int i = 0; i < fieldNames.length; i++) { + final String name = fieldNames[i]; + + final JsonNode fieldConverted = convert(reuse, reuse.get(name), fieldTypes[i], row.getField(i)); + reuse.set(name, fieldConverted); + } + + return reuse; + } + + private JsonNode convert(ContainerNode container, JsonNode reuse, TypeInformation info, Object object) { + if (info == Types.VOID || object == null) { + return container.nullNode(); + } else if (info == Types.BOOLEAN) { + return container.booleanNode((Boolean) object); + } else if (info == Types.STRING) { + return container.textNode((String) object); + } else if (info == Types.BIG_DEC) { + // convert decimal if necessary + if (object instanceof BigDecimal) { + return container.numberNode((BigDecimal) object); + } + return container.numberNode(BigDecimal.valueOf(((Number) object).doubleValue())); + } else if (info == Types.BIG_INT) { + // convert integer if necessary + if (object instanceof BigInteger) { + return container.numberNode((BigInteger) object); + } + return container.numberNode(BigInteger.valueOf(((Number) object).longValue())); + } else if (info == Types.SQL_DATE) { + return container.textNode(object.toString()); + } else if (info == Types.SQL_TIME) { + final Time time = (Time) object; + // strip milliseconds if possible + if (time.getTime() % 1000 > 0) { + return container.textNode(timeFormatWithMillis.format(time)); + } + return container.textNode(timeFormat.format(time)); + } else if (info == Types.SQL_TIMESTAMP) { + return container.textNode(timestampFormat.format((Timestamp) object)); + } else if (info instanceof RowTypeInfo) { + if (reuse != null && reuse instanceof ObjectNode) { + return convertRow((ObjectNode) reuse, (RowTypeInfo) info, (Row) object); + } else { + return convertRow(null, (RowTypeInfo) info, (Row) object); + } + } else if (info instanceof ObjectArrayTypeInfo) { + if (reuse != null && reuse instanceof ArrayNode) { + return convertObjectArray((ArrayNode) reuse, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } else { + return convertObjectArray(null, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } + } else if (info instanceof BasicArrayTypeInfo) { + if (reuse != null && reuse instanceof ArrayNode) { + return convertObjectArray((ArrayNode) reuse, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } else { + return convertObjectArray(null, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } + } else if (info instanceof PrimitiveArrayTypeInfo && ((PrimitiveArrayTypeInfo) info).getComponentType() == Types.BYTE) { + return container.binaryNode((byte[]) object); + } else { + // for types that were specified without JSON schema + // e.g. POJOs + try { + return mapper.valueToTree(object); + } catch (IllegalArgumentException e) { + throw new IllegalStateException("Unsupported type information '" + info + "' for object: " + object, e); + } + } + } + + private ArrayNode convertObjectArray(ArrayNode reuse, TypeInformation info, Object[] array) { + if (reuse == null) { + reuse = mapper.createArrayNode(); + } else { + reuse.removeAll(); + } + + for (Object object : array) { + reuse.add(convert(reuse, null, info, object)); + } + return reuse; + } +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index 8ba21b953..4ad8947a8 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -18,9 +18,10 @@ package com.dtstack.flink.sql.sink.kafka.table; +import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.format.FormatType; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -32,12 +33,13 @@ * @author DocLi * @modifyer maqi */ -public class KafkaSinkParser extends AbsTableParser { +public class KafkaSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KafkaSinkTableInfo kafkaSinkTableInfo = new KafkaSinkTableInfo(); kafkaSinkTableInfo.setName(tableName); kafkaSinkTableInfo.setType(MathUtil.getString(props.get(KafkaSinkTableInfo.TYPE_KEY.toLowerCase()))); + parseFieldsInfo(fieldsInfo, kafkaSinkTableInfo); if (props.get(KafkaSinkTableInfo.SINK_DATA_TYPE) != null) { @@ -46,11 +48,14 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map kafkaParam = new HashMap(); @@ -59,6 +70,8 @@ public class KafkaSinkTableInfo extends TargetTableInfo { private String partitionKeys; + private String updateMode; + public void addKafkaParam(String key, String value) { kafkaParam.put(key, value); } @@ -71,7 +84,6 @@ public Set getKafkaParamKeys() { return kafkaParam.keySet(); } - public String getBootstrapServers() { return bootstrapServers; } @@ -104,14 +116,40 @@ public void setFieldDelimiter(String fieldDelimiter) { this.fieldDelimiter = fieldDelimiter; } + public String getUpdateMode() { + return updateMode; + } + + public void setUpdateMode(String updateMode) { + this.updateMode = updateMode; + } + @Override public boolean check() { Preconditions.checkNotNull(getType(), "kafka of type is required"); Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); Preconditions.checkNotNull(topic, "kafka of topic is required"); + + if (StringUtils.equalsIgnoreCase(getSinkDataType(), FormatType.AVRO.name())) { + avroParamCheck(); + } + return false; } + public void avroParamCheck() { + Preconditions.checkNotNull(schemaString, "avro type schemaInfo is required"); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + Schema schema = new Schema.Parser().parse(schemaString); + schema.getFields() + .stream() + .filter(field -> StringUtils.equalsIgnoreCase(field.name(), RETRACT_FIELD_KEY)) + .findFirst() + .orElseThrow(() -> + new NullPointerException(String.valueOf("arvo upsert mode the retract attribute must be contained in schemaInfo field "))); + } + } + public String getEnableKeyPartition() { return enableKeyPartition; } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java new file mode 100644 index 000000000..852a381e2 --- /dev/null +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.source.kafka; + +import com.dtstack.flink.sql.source.IStreamSourceGener; +import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; +import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.util.PluginUtil; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; +import org.apache.flink.table.api.Table; +import org.apache.flink.types.Row; +import org.apache.kafka.clients.consumer.ConsumerConfig; + +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * Date: 2020/3/20 + * Company: www.dtstack.com + * @author maqi + */ +public abstract class AbstractKafkaSource implements IStreamSourceGener

{ + + private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; + + protected Properties getKafkaProperties(KafkaSourceTableInfo kafkaSourceTableInfo) { + Properties props = new Properties(); + props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaSourceTableInfo.getBootstrapServers()); + + if (DtStringUtil.isJson(kafkaSourceTableInfo.getOffsetReset())) { + props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, EKafkaOffset.NONE.name().toLowerCase()); + } else { + props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, kafkaSourceTableInfo.getOffsetReset()); + } + + if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())) { + props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, kafkaSourceTableInfo.getGroupId()); + } + + for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { + props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); + } + return props; + } + + protected String generateOperatorName(String tabName, String topicName) { + return SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", tabName); + } + + protected TypeInformation getRowTypeInformation(KafkaSourceTableInfo kafkaSourceTableInfo) { + Class[] fieldClasses = kafkaSourceTableInfo.getFieldClasses(); + TypeInformation[] types = IntStream.range(0, fieldClasses.length) + .mapToObj(i -> TypeInformation.of(fieldClasses[i])) + .toArray(TypeInformation[]::new); + + return new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + } + + protected void setStartPosition(String offset, String topicName, FlinkKafkaConsumerBase kafkaSrc) { + if (StringUtils.equalsIgnoreCase(offset, EKafkaOffset.EARLIEST.name())) { + kafkaSrc.setStartFromEarliest(); + } else if (DtStringUtil.isJson(offset)) { + Map specificStartupOffsets = buildOffsetMap(offset, topicName); + kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); + } else { + kafkaSrc.setStartFromLatest(); + } + } + + /** + * kafka offset,eg.. {"0":12312,"1":12321,"2":12312} + * @param offsetJson + * @param topicName + * @return + */ + protected Map buildOffsetMap(String offsetJson, String topicName) { + try { + Properties properties = PluginUtil.jsonStrToObject(offsetJson, Properties.class); + Map offsetMap = PluginUtil.objectToMap(properties); + Map specificStartupOffsets = offsetMap + .entrySet() + .stream() + .collect(Collectors.toMap( + (Map.Entry entry) -> new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), + (Map.Entry entry) -> Long.valueOf(entry.getValue().toString())) + ); + + return specificStartupOffsets; + } catch (Exception e) { + throw new RuntimeException("not support offsetReset type:" + offsetJson); + } + } + +} diff --git a/cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java similarity index 56% rename from cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java rename to kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java index 33a0233ac..476ccc8b7 100644 --- a/cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java @@ -16,43 +16,16 @@ * limitations under the License. */ - - -package com.dtstack.flinkx; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; +package com.dtstack.flink.sql.source.kafka.enums; /** - * Unit test for simple App. + * Date: 2020/3/20 + * Company: www.dtstack.com + * @author maqi */ -public class AppTest - extends TestCase -{ - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest( String testName ) - { - super( testName ); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() - { - return new TestSuite( AppTest.class ); - } +public enum EKafkaOffset { - /** - * Rigourous Test :-) - */ - public void testApp() - { - assertTrue( true ); - } + LATEST, + EARLIEST, + NONE } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 005bfb4c2..bef86f10d 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -19,11 +19,14 @@ package com.dtstack.flink.sql.source.kafka.table; -import com.dtstack.flink.sql.table.AbsSourceParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.format.FormatType; +import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; +import com.dtstack.flink.sql.table.AbstractSourceParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; +import java.util.stream.Collectors; /** * Reason: @@ -32,33 +35,38 @@ * * @author sishu.yss */ -public class KafkaSourceParser extends AbsSourceParser { +public class KafkaSourceParser extends AbstractSourceParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { KafkaSourceTableInfo kafkaSourceTableInfo = new KafkaSourceTableInfo(); - kafkaSourceTableInfo.setName(tableName); - kafkaSourceTableInfo.setType(MathUtil.getString(props.get(KafkaSourceTableInfo.TYPE_KEY.toLowerCase()))); parseFieldsInfo(fieldsInfo, kafkaSourceTableInfo); + kafkaSourceTableInfo.setName(tableName); + kafkaSourceTableInfo.setType(MathUtil.getString(props.get(KafkaSourceTableInfo.TYPE_KEY.toLowerCase()))); kafkaSourceTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(KafkaSourceTableInfo.PARALLELISM_KEY.toLowerCase()))); - String bootstrapServer = MathUtil.getString(props.get(KafkaSourceTableInfo.BOOTSTRAPSERVERS_KEY.toLowerCase())); - if (bootstrapServer == null || bootstrapServer.trim().equals("")) { - throw new Exception("BootstrapServers can not be empty!"); - } else { - kafkaSourceTableInfo.setBootstrapServers(bootstrapServer); - } + kafkaSourceTableInfo.setBootstrapServers(MathUtil.getString(props.get(KafkaSourceTableInfo.BOOTSTRAPSERVERS_KEY.toLowerCase()))); kafkaSourceTableInfo.setGroupId(MathUtil.getString(props.get(KafkaSourceTableInfo.GROUPID_KEY.toLowerCase()))); kafkaSourceTableInfo.setTopic(MathUtil.getString(props.get(KafkaSourceTableInfo.TOPIC_KEY.toLowerCase()))); kafkaSourceTableInfo.setOffsetReset(MathUtil.getString(props.get(KafkaSourceTableInfo.OFFSETRESET_KEY.toLowerCase()))); kafkaSourceTableInfo.setTopicIsPattern(MathUtil.getBoolean(props.get(KafkaSourceTableInfo.TOPICISPATTERN_KEY.toLowerCase()), false)); + kafkaSourceTableInfo.setOffsetReset(MathUtil.getString(props.getOrDefault(KafkaSourceTableInfo.OFFSETRESET_KEY.toLowerCase(), EKafkaOffset.LATEST.name().toLowerCase()))); + kafkaSourceTableInfo.setTopicIsPattern(MathUtil.getBoolean(props.get(KafkaSourceTableInfo.TOPICISPATTERN_KEY.toLowerCase()))); kafkaSourceTableInfo.setTimeZone(MathUtil.getString(props.get(KafkaSourceTableInfo.TIME_ZONE_KEY.toLowerCase()))); - for (String key : props.keySet()) { - if (!key.isEmpty() && key.startsWith("kafka.")) { - kafkaSourceTableInfo.addKafkaParam(key.substring(6), props.get(key).toString()); - } - } + + kafkaSourceTableInfo.setSchemaString(MathUtil.getString(props.get(KafkaSourceTableInfo.SCHEMA_STRING_KEY.toLowerCase()))); + kafkaSourceTableInfo.setFieldDelimiter(MathUtil.getString(props.getOrDefault(KafkaSourceTableInfo.CSV_FIELD_DELIMITER_KEY.toLowerCase(), "|"))); + kafkaSourceTableInfo.setSourceDataType(MathUtil.getString(props.getOrDefault(KafkaSourceTableInfo.SOURCE_DATA_TYPE_KEY.toLowerCase(), FormatType.DT_NEST.name()))); + + Map kafkaParams = props.keySet().stream() + .filter(key -> !key.isEmpty() && key.startsWith("kafka.")) + .collect(Collectors.toMap( + key -> key.substring(6), key -> props.get(key).toString()) + ); + + kafkaSourceTableInfo.addKafkaParam(kafkaParams); kafkaSourceTableInfo.check(); + return kafkaSourceTableInfo; } } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index 4f4851335..c27eee376 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -17,11 +17,10 @@ */ - package com.dtstack.flink.sql.source.kafka.table; import com.dtstack.flink.sql.format.FormatType; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.google.common.base.Preconditions; import java.util.HashMap; @@ -35,136 +34,126 @@ * @author sishu.yss */ -public class KafkaSourceTableInfo extends SourceTableInfo { +public class KafkaSourceTableInfo extends AbstractSourceTableInfo { + + public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; - public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; + public static final String TOPIC_KEY = "topic"; - public static final String TOPIC_KEY = "topic"; + public static final String TYPE_KEY = "type"; - public static final String TYPE_KEY = "type"; + public static final String GROUPID_KEY = "groupId"; - public static final String GROUPID_KEY = "groupId"; + public static final String OFFSETRESET_KEY = "offsetReset"; - public static final String OFFSETRESET_KEY = "offsetReset"; + public static final String TOPICISPATTERN_KEY = "topicIsPattern"; - public static final String TOPICISPATTERN_KEY = "topicIsPattern"; + public static final String SCHEMA_STRING_KEY = "schemaInfo"; - private String bootstrapServers; + public static final String CSV_FIELD_DELIMITER_KEY = "fieldDelimiter"; - private String topic; + public static final String SOURCE_DATA_TYPE_KEY = "sourceDataType"; - private String groupId; + private String bootstrapServers; - //latest, earliest - private String offsetReset = "latest"; + private String topic; - private String offset; + private String groupId; - private Boolean topicIsPattern = false; + private String offsetReset; - private String sourceDataType = FormatType.DT_NEST.name(); + private Boolean topicIsPattern = false; - private String schemaString; + private String sourceDataType; - private String fieldDelimiter; + private String schemaString; - public String getBootstrapServers() { - return bootstrapServers; - } + private String fieldDelimiter; - public void setBootstrapServers(String bootstrapServers) { - this.bootstrapServers = bootstrapServers; - } + public Map kafkaParam = new HashMap<>(); - public String getTopic() { - return topic; - } - public void setTopic(String topic) { - this.topic = topic; - } + public String getBootstrapServers() { + return bootstrapServers; + } - public String getGroupId() { - return groupId; - } + public void setBootstrapServers(String bootstrapServers) { + this.bootstrapServers = bootstrapServers; + } - public void setGroupId(String groupId) { - this.groupId = groupId; - } + public String getTopic() { + return topic; + } - public String getOffsetReset() { - return offsetReset; - } + public void setTopic(String topic) { + this.topic = topic; + } - public void setOffsetReset(String offsetReset) { - if(offsetReset == null){ - return; - } - this.offsetReset = offsetReset; - } + public String getGroupId() { + return groupId; + } - public String getOffset() { - return offset; - } + public void setGroupId(String groupId) { + this.groupId = groupId; + } - public void setOffset(String offset) { - if (offsetReset == null) { - return; - } - this.offset = offset; - } + public String getOffsetReset() { + return offsetReset; + } - public Boolean getTopicIsPattern() { - return topicIsPattern; - } + public void setOffsetReset(String offsetReset) { + this.offsetReset = offsetReset; + } - public void setTopicIsPattern(Boolean topicIsPattern) { - this.topicIsPattern = topicIsPattern; - } + public Boolean getTopicIsPattern() { + return topicIsPattern; + } - public Map kafkaParam = new HashMap<>(); + public void setTopicIsPattern(Boolean topicIsPattern) { + this.topicIsPattern = topicIsPattern; + } - public void addKafkaParam(String key, String value) { - kafkaParam.put(key, value); - } + public void addKafkaParam(Map kafkaParam) { + kafkaParam.putAll(kafkaParam); + } - public String getKafkaParam(String key) { - return kafkaParam.get(key); - } + public String getKafkaParam(String key) { + return kafkaParam.get(key); + } - public Set getKafkaParamKeys() { - return kafkaParam.keySet(); - } + public Set getKafkaParamKeys() { + return kafkaParam.keySet(); + } - public String getSourceDataType() { - return sourceDataType; - } + public String getSourceDataType() { + return sourceDataType; + } - public void setSourceDataType(String sourceDataType) { - this.sourceDataType = sourceDataType; - } + public void setSourceDataType(String sourceDataType) { + this.sourceDataType = sourceDataType; + } - public String getSchemaString() { - return schemaString; - } + public String getSchemaString() { + return schemaString; + } - public void setSchemaString(String schemaString) { - this.schemaString = schemaString; - } + public void setSchemaString(String schemaString) { + this.schemaString = schemaString; + } - public String getFieldDelimiter() { - return fieldDelimiter; - } + public String getFieldDelimiter() { + return fieldDelimiter; + } - public void setFieldDelimiter(String fieldDelimiter) { - this.fieldDelimiter = fieldDelimiter; - } + public void setFieldDelimiter(String fieldDelimiter) { + this.fieldDelimiter = fieldDelimiter; + } - @Override - public boolean check() { - Preconditions.checkNotNull(getType(), "kafka of type is required"); - Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); - Preconditions.checkNotNull(topic, "kafka of topic is required"); - return false; - } + @Override + public boolean check() { + Preconditions.checkNotNull(getType(), "kafka of type is required"); + Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); + Preconditions.checkNotNull(topic, "kafka of topic is required"); + return false; + } } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java index 71a9cc386..1cbbeafd9 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java @@ -23,7 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,7 @@ * * @author maqi */ -public class KafkaProducer extends FlinkKafkaProducer { +public class KafkaProducer extends FlinkKafkaProducer { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer.class); @@ -45,7 +45,7 @@ public class KafkaProducer extends FlinkKafkaProducer { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] parititonKeys) { + public KafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] parititonKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, parititonKeys), producerConfig, customPartitioner); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java index f3a2f40f5..6bf9014df 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java @@ -22,7 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import java.util.Optional; import java.util.Properties; @@ -36,7 +36,7 @@ public class KafkaProducerFactory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index d0d477f4e..632bb720e 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,23 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import java.util.Optional; import java.util.Properties; @@ -44,104 +29,23 @@ * @create: 2019-11-05 11:45 * @description: **/ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected int parallelism; - - protected Properties properties; - - protected FlinkKafkaProducer flinkKafkaProducer; - - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - - private String[] partitionKeys; - +public class KafkaSink extends AbstractKafkaSink { @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafkaSinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafkaSinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafkaSinkTableInfo.getBootstrapServers()); - for (String key : kafkaSinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafkaSinkTableInfo.getKafkaParam(key)); - } + Properties kafkaProperties = getKafkaProperties(kafkaSinkTableInfo); + this.tableName = kafkaSinkTableInfo.getName(); + this.topic = kafkaSinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafkaSinkTableInfo); this.fieldNames = kafkaSinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafkaSinkTableInfo.getFields().length]; - for (int i = 0; i < kafkaSinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i) new KafkaProducerFactory().createKafkaProducer(kafkaSinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); + this.fieldTypes = getTypeInformations(kafkaSinkTableInfo); + this.schema = buildTableSchema(fieldNames, fieldTypes); + this.parallelism = kafkaSinkTableInfo.getParallelism(); + this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); + this.kafkaProducer = new KafkaProducerFactory().createKafkaProducer(kafkaSinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); return this; } - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) - .setParallelism(parallelism); - - mapDataStream.addSink(flinkKafkaProducer).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } } diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index a445a6529..394ea86ee 100644 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -18,24 +18,17 @@ package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; /** @@ -43,75 +36,24 @@ * @create: 2019-11-05 10:55 * @description: **/ -public class KafkaSource implements IStreamSourceGener
{ +public class KafkaSource extends AbstractKafkaSource { - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) { - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())) { - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } - - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } - - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer kafkaSrc = (FlinkKafkaConsumer) new KafkaConsumerFactory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - FlinkKafkaConsumer kafkaSrc = (FlinkKafkaConsumer) new KafkaConsumerFactory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); - - - //earliest,latest - if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for (Map.Entry entry : offsetMap.entrySet()) { - specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - } else { - kafkaSrc.setStartFromLatest(); - } + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } return tableEnv.fromDataStream(kafkaSource, fields); } } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java index c815e134a..bee1865dd 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java @@ -23,6 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +38,7 @@ * * @author maqi */ -public class KafkaProducer09 extends FlinkKafkaProducer09 { +public class KafkaProducer09 extends FlinkKafkaProducer09 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer09.class); @@ -45,7 +46,7 @@ public class KafkaProducer09 extends FlinkKafkaProducer09 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer09(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner,String[] partitionKeys) { + public KafkaProducer09(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.orElse(null)); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java index 7fb3909ee..ee3423b07 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -36,7 +37,7 @@ public class KafkaProducer09Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner,String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer09(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 52caa7bd2..d22be3d59 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,23 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import java.util.Optional; import java.util.Properties; @@ -46,109 +31,22 @@ * @author DocLi * @modifyer maqi */ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected Properties properties; - - protected FlinkKafkaProducer09 kafkaProducer09; - - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - - private String[] partitionKeys; - - protected int parallelism; - - - +public class KafkaSink extends AbstractKafkaSink{ @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka09SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafka09SinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafka09SinkTableInfo.getBootstrapServers()); - for (String key : kafka09SinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafka09SinkTableInfo.getKafkaParam(key)); - } + Properties kafkaProperties = getKafkaProperties(kafka09SinkTableInfo); + this.tableName = kafka09SinkTableInfo.getName(); + this.topic = kafka09SinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka09SinkTableInfo); this.fieldNames = kafka09SinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafka09SinkTableInfo.getFields().length]; - for (int i = 0; i < kafka09SinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafka09SinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i) new KafkaProducer09Factory() - .createKafkaProducer(kafka09SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); + this.fieldTypes = getTypeInformations(kafka09SinkTableInfo); + this.schema = buildTableSchema(fieldNames, fieldTypes); + this.parallelism = kafka09SinkTableInfo.getParallelism(); + this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); + this.kafkaProducer = new KafkaProducer09Factory().createKafkaProducer(kafka09SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); return this; } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) - .setParallelism(parallelism); - - mapDataStream.addSink(kafkaProducer09) - .name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } - } diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 21ed7c3e3..9f8917761 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -17,17 +17,13 @@ */ - package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer09; @@ -36,7 +32,6 @@ import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; -import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -47,74 +42,23 @@ * @author xuchao */ -public class KafkaSource implements IStreamSourceGener
{ - - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") +public class KafkaSource extends AbstractKafkaSource { @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } - // only required for Kafka 0.8 - //TODO props.setProperty("zookeeper.connect", kafkaSourceTableInfo.) + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer09 kafkaSrc = (FlinkKafkaConsumer09) new KafkaConsumer09Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for(int i = 0; i< kafkaSourceTableInfo.getFieldClasses().length; i++){ - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } - - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); - FlinkKafkaConsumer09 kafkaSrc = (FlinkKafkaConsumer09) new KafkaConsumer09Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); - - //earliest,latest - if("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())){ - kafkaSrc.setStartFromEarliest(); - }else if(DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for(Map.Entry entry:offsetMap.entrySet()){ - specificStartupOffsets.put(new KafkaTopicPartition(topicName,Integer.valueOf(entry.getKey())),Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - }else { - kafkaSrc.setStartFromLatest(); - } + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } return tableEnv.fromDataStream(kafkaSource, fields); } } \ No newline at end of file diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java index 3cdc45dec..3936575ef 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java @@ -23,7 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,7 @@ * * @author maqi */ -public class KafkaProducer010 extends FlinkKafkaProducer010 { +public class KafkaProducer010 extends FlinkKafkaProducer010 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer010.class); @@ -45,7 +45,7 @@ public class KafkaProducer010 extends FlinkKafkaProducer010 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.get()); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java index c44a9fe86..e0e023586 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -36,7 +37,7 @@ public class KafkaProducer010Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer010(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 42f1045bf..eea78e121 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,21 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import java.util.Optional; import java.util.Properties; @@ -46,105 +33,22 @@ * @modifyer maqi * */ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected Properties properties; - - protected int parallelism; - - protected KafkaSinkTableInfo kafka10SinkTableInfo; - - /** The schema of the table. */ - private TableSchema schema; - - private String[] partitionKeys; - +public class KafkaSink extends AbstractKafkaSink { @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { - this.kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafka10SinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafka10SinkTableInfo.getBootstrapServers()); - - for (String key : kafka10SinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafka10SinkTableInfo.getKafkaParam(key)); - } + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + KafkaSinkTableInfo kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; + Properties kafkaProperties = getKafkaProperties(kafka10SinkTableInfo); + this.tableName = kafka10SinkTableInfo.getName(); + this.topic = kafka10SinkTableInfo.getTopic(); + this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka10SinkTableInfo); this.fieldNames = kafka10SinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafka10SinkTableInfo.getFields().length]; - for (int i = 0; i < kafka10SinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafka10SinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - - RichSinkFunction kafkaProducer010 = new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, - Optional.of(new CustomerFlinkPartition<>()), partitionKeys); - - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) - .setParallelism(parallelism); - - mapDataStream.addSink(kafkaProducer010).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } - } diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 55cffbf1e..23989ab7e 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -19,24 +19,17 @@ package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; /** @@ -47,74 +40,24 @@ * @author sishu.yss */ -public class KafkaSource implements IStreamSourceGener
{ +public class KafkaSource extends AbstractKafkaSource { - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; + @Override + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; + String topicName = kafkaSourceTableInfo.getTopic(); - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") - @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer010 kafkaSrc = (FlinkKafkaConsumer010) new KafkaConsumer010Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; - String topicName = kafkaSourceTableInfo.getTopic(); + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); + String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } - - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); - - FlinkKafkaConsumer010 kafkaSrc = (FlinkKafkaConsumer010) new KafkaConsumer010Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); - - //earliest,latest - if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for (Map.Entry entry : offsetMap.entrySet()) { - specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - } else { - kafkaSrc.setStartFromLatest(); - } - - String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } - return tableEnv.fromDataStream(kafkaSource, fields); - } + return tableEnv.fromDataStream(kafkaSource, fields); + } } \ No newline at end of file diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java index 7880dd419..429d21a79 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java @@ -24,7 +24,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +38,7 @@ * * @author maqi */ -public class KafkaProducer011 extends FlinkKafkaProducer011 { +public class KafkaProducer011 extends FlinkKafkaProducer011 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer011.class); @@ -46,7 +46,7 @@ public class KafkaProducer011 extends FlinkKafkaProducer011 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer011(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + public KafkaProducer011(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java index e2272b16e..0cb11da82 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -36,7 +37,7 @@ public class KafkaProducer011Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer011(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index c5da782d0..ea45280c7 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,23 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import java.util.Optional; import java.util.Properties; @@ -49,106 +34,22 @@ * @modifyer maqi * */ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected int parallelism; - - protected Properties properties; - - protected FlinkKafkaProducer011 kafkaProducer011; - - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - private String[] partitionKeys; - - +public class KafkaSink extends AbstractKafkaSink { @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka11SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafka11SinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafka11SinkTableInfo.getBootstrapServers()); - for (String key : kafka11SinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafka11SinkTableInfo.getKafkaParam(key)); - } + Properties kafkaProperties = getKafkaProperties(kafka11SinkTableInfo); + this.tableName = kafka11SinkTableInfo.getName(); + this.topic = kafka11SinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka11SinkTableInfo); this.fieldNames = kafka11SinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafka11SinkTableInfo.getFields().length]; - for (int i = 0; i < kafka11SinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafka11SinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i) new KafkaProducer011Factory() - .createKafkaProducer(kafka11SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); + this.fieldTypes = getTypeInformations(kafka11SinkTableInfo); + this.schema = buildTableSchema(fieldNames, fieldTypes); + this.parallelism = kafka11SinkTableInfo.getParallelism(); + this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); + this.kafkaProducer = new KafkaProducer011Factory().createKafkaProducer(kafka11SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); return this; } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) - .setParallelism(parallelism); - - mapDataStream.addSink(kafkaProducer011).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink>configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } - } diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java index 3f804fc6c..c20f0678b 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java @@ -31,8 +31,8 @@ /** * company: www.dtstack.com - * author: toutian * create: 2019/12/24 + * @author: toutian */ public class KafkaConsumer011Factory extends AbstractKafkaConsumerFactory { diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 1a4eed08a..f58d59d05 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -19,24 +19,16 @@ package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; - -import java.util.HashMap; -import java.util.Map; import java.util.Properties; /** @@ -47,76 +39,24 @@ * @author sishu.yss */ -public class KafkaSource implements IStreamSourceGener
{ - - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") - @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; - String topicName = kafkaSourceTableInfo.getTopic(); - - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } - // only required for Kafka 0.8 - //TODO props.setProperty("zookeeper.connect", kafka09SourceTableInfo.) - - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } +public class KafkaSource extends AbstractKafkaSource { - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + @Override + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; + String topicName = kafkaSourceTableInfo.getTopic(); - FlinkKafkaConsumer011 kafkaSrc = (FlinkKafkaConsumer011) new KafkaConsumer011Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer011 kafkaSrc = (FlinkKafkaConsumer011) new KafkaConsumer011Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - //earliest,latest - if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for (Map.Entry entry : offsetMap.entrySet()) { - specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - } else { - kafkaSrc.setStartFromLatest(); - } + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); - String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); + String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } - return tableEnv.fromDataStream(kafkaSource, fields); - } + return tableEnv.fromDataStream(kafkaSource, fields); + } } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 63ff31c50..31f9aa13d 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -1,24 +1,23 @@ package com.dtstack.flink.sql.side.kudu; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; -import org.apache.flink.util.Preconditions; - -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.KuduClient; @@ -41,7 +40,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; -public class KuduAllReqRow extends AllReqRow { +public class KuduAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 6051774809356082219L; @@ -62,7 +61,7 @@ public class KuduAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public KuduAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new KuduAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -229,9 +228,6 @@ private KuduScanner getConn(KuduSideTableInfo tableInfo) { if (null != workerCount) { kuduClientBuilder.workerCount(workerCount); } - if (null != defaultSocketReadTimeoutMs) { - kuduClientBuilder.defaultSocketReadTimeoutMs(defaultSocketReadTimeoutMs); - } if (null != defaultOperationTimeoutMs) { kuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java index ddfa73201..1241a6f37 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java @@ -2,8 +2,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; @@ -12,16 +12,16 @@ import java.util.List; -public class KuduAllSideInfo extends SideInfo { +public class KuduAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = 9005389633060174746L; - public KuduAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { // no use } diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index e1705d796..e534a85f9 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -1,20 +1,12 @@ package com.dtstack.flink.sql.side.kudu; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; - import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; @@ -24,6 +16,13 @@ import com.stumbleupon.async.Deferred; import io.vertx.core.json.JsonArray; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.AsyncKuduClient; @@ -42,7 +41,7 @@ import java.util.List; import java.util.Map; -public class KuduAsyncReqRow extends AsyncReqRow { +public class KuduAsyncReqRow extends BaseAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(KuduAsyncReqRow.class); /** @@ -65,7 +64,7 @@ public class KuduAsyncReqRow extends AsyncReqRow { private AsyncKuduScanner.AsyncKuduScannerBuilder scannerBuilder; - public KuduAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new KuduAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -95,9 +94,6 @@ private void connKuDu() throws KuduException { if (null != workerCount) { asyncKuduClientBuilder.workerCount(workerCount); } - if (null != defaultSocketReadTimeoutMs) { - asyncKuduClientBuilder.defaultSocketReadTimeoutMs(defaultSocketReadTimeoutMs); - } if (null != defaultOperationTimeoutMs) { asyncKuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java index de75f23fb..5ace515f7 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java @@ -2,8 +2,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; @@ -12,15 +12,15 @@ import java.util.List; -public class KuduAsyncSideInfo extends SideInfo { +public class KuduAsyncSideInfo extends BaseSideInfo { - public KuduAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } @Override diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java index c03c470a7..62d215d87 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java @@ -1,7 +1,7 @@ package com.dtstack.flink.sql.side.kudu.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; @@ -9,9 +9,9 @@ import java.sql.Timestamp; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; -public class KuduSideParser extends AbsSideTableParser { +public class KuduSideParser extends AbstractSideTableParser { public static final String KUDU_MASTERS = "kuduMasters"; @@ -52,7 +52,7 @@ public class KuduSideParser extends AbsSideTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KuduSideTableInfo kuduSideTableInfo = new KuduSideTableInfo(); kuduSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, kuduSideTableInfo); @@ -107,6 +107,7 @@ public Class dbTypeConvertToJavaType(String fieldType) { return BigDecimal.class; case "binary": return byte[].class; + default: } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java index c527ec268..eb8df5ad9 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java @@ -1,9 +1,9 @@ package com.dtstack.flink.sql.side.kudu.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; -public class KuduSideTableInfo extends SideTableInfo { +public class KuduSideTableInfo extends AbstractSideTableInfo { private static final String CURR_TYPE = "kudu"; diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java index b750e2d4a..dfc02dee6 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java @@ -191,6 +191,7 @@ public static KuduPredicate buildKuduPredicate(Schema schema, PredicateInfo info return KuduPredicate.newIsNotNullPredicate(column); case "IS NULL": return KuduPredicate.newIsNullPredicate(column); + default: } return null; diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index 99d19cdef..4543b9618 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -18,13 +18,11 @@ package com.dtstack.flink.sql.sink.kudu; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; - -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import org.apache.kudu.client.AsyncKuduClient; import org.apache.kudu.client.AsyncKuduSession; import org.apache.kudu.client.KuduClient; @@ -44,7 +42,7 @@ * @author gituser * @modify xiuzhu */ -public class KuduOutputFormat extends DtRichOutputFormat { +public class KuduOutputFormat extends AbstractDtRichOutputFormat { private static final long serialVersionUID = 1L; @@ -125,7 +123,7 @@ public void writeRecord(Tuple2 record) throws IOException { Row row = tupleTrans.getField(1); if (row.getArity() != fieldNames.length) { if(outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0) { - LOG.error("record insert failed:{}", row.toString()); + LOG.error("record insert failed ..{}", row.toString()); LOG.error("cause by row.getArity() != fieldNames.length"); } outDirtyRecords.inc(); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java index 0c90ffe94..4c7c3bea8 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java @@ -3,7 +3,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kudu.table.KuduTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -39,7 +39,7 @@ public class KuduSink implements RetractStreamTableSink, Serializable, IStr private int parallelism = -1; @Override - public KuduSink genStreamSink(TargetTableInfo targetTableInfo) { + public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KuduTableInfo kuduTableInfo = (KuduTableInfo) targetTableInfo; this.kuduMasters = kuduTableInfo.getKuduMasters(); this.tableName = kuduTableInfo.getTableName(); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java index ea61bd325..20302d44f 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java @@ -1,8 +1,8 @@ package com.dtstack.flink.sql.sink.kudu.table; import com.dtstack.flink.sql.sink.kudu.KuduOutputFormat; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; @@ -10,9 +10,9 @@ import java.sql.Timestamp; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; -public class KuduSinkParser extends AbsTableParser { +public class KuduSinkParser extends AbstractTableParser { public static final String KUDU_MASTERS = "kuduMasters"; @@ -27,7 +27,7 @@ public class KuduSinkParser extends AbsTableParser { public static final String SOCKET_READ_TIMEOUT_MS = "defaultSocketReadTimeoutMs"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KuduTableInfo kuduTableInfo = new KuduTableInfo(); kuduTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, kuduTableInfo); @@ -86,6 +86,7 @@ public Class dbTypeConvertToJavaType(String fieldType) { return BigDecimal.class; case "binary": return byte[].class; + default: } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java index c086c7298..80e47761b 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java @@ -1,10 +1,10 @@ package com.dtstack.flink.sql.sink.kudu.table; import com.dtstack.flink.sql.sink.kudu.KuduOutputFormat; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; -public class KuduTableInfo extends TargetTableInfo { +public class KuduTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "kudu"; diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 14cd847b5..8f0090db5 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -21,11 +21,11 @@ import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.util.PluginUtil; +import com.esotericsoftware.minlog.Log; import org.apache.commons.io.Charsets; import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.MiniClusterClient; -import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; @@ -89,7 +89,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { if (StringUtils.isNotBlank(yarnConfDir)) { try { - config.setString(ConfigConstants.PATH_HADOOP_CONFIG, yarnConfDir); + config.setString("fs.hdfs.hadoopconf", yarnConfDir); FileSystem.initialize(config); YarnConfiguration yarnConf = YarnConfLoader.getYarnConf(yarnConfDir); @@ -109,7 +109,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { applicationId = getYarnClusterApplicationId(yarnClient); } - System.out.println("applicationId=" + applicationId.toString()); + Log.info("applicationId={}", applicationId.toString()); if (StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); @@ -166,7 +166,7 @@ private static ApplicationId getYarnClusterApplicationId(YarnClient yarnClient) private static ApplicationId toApplicationId(String appIdStr) { Iterator it = StringHelper._split(appIdStr).iterator(); - if (!(it.next()).equals("application")) { + if (!"application".equals(it.next())) { throw new IllegalArgumentException("Invalid ApplicationId prefix: " + appIdStr + ". The valid ApplicationId should start with prefix " + "application"); } else { try { diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 3de1cdbc5..0e5089bc9 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.enums.EPluginLoadMode; import com.dtstack.flink.sql.launcher.YarnConfLoader; import com.dtstack.flink.sql.option.Options; +import com.esotericsoftware.minlog.Log; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.cache.DistributedCache; import org.apache.flink.configuration.Configuration; @@ -79,7 +80,7 @@ public void init(String yarnConfDir, Configuration flinkConfig, Properties userC yarnClient.init(yarnConf); yarnClient.start(); - System.out.println("----init yarn success ----"); + Log.info("----init yarn success ----"); } public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJarPath, Options launcherOptions, JobGraph jobGraph) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index 29cc4890a..ca2d5bdd7 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -73,7 +73,6 @@ public static String submit(Options launcherOptions, JobGraph jobGraph, Configur String flinkJobId = jobGraph.getJobID().toString(); String tips = String.format("deploy per_job with appId: %s, jobId: %s", applicationId, flinkJobId); - System.out.println(tips); LOG.info(tips); return applicationId; diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index f90cfd973..3373f3667 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -18,25 +18,17 @@ package com.dtstack.flink.sql.side.mongo; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; - -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.mongodb.BasicDBObject; import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; -import com.mongodb.MongoCredential; -import com.mongodb.ServerAddress; +import com.mongodb.MongoClientURI; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; @@ -44,13 +36,19 @@ import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.SQLException; import java.sql.Timestamp; -import java.util.ArrayList; import java.util.Calendar; import java.util.List; import java.util.Map; @@ -62,7 +60,7 @@ * * @author xuqianjin */ -public class MongoAllReqRow extends AllReqRow { +public class MongoAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = -675332795591842778L; @@ -78,7 +76,7 @@ public class MongoAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public MongoAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MongoAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -181,34 +179,13 @@ private String buildKey(Map val, List equalFieldList) { return sb.toString(); } - private MongoCollection getConn(String address, String userName, String password, String database, String tableName) { + private MongoCollection getConn(String address, String database, String tableName) { MongoCollection dbCollection; - try { - MongoCredential credential; - String[] servers = StringUtils.split(address, ","); - String host; - Integer port; - String[] hostAndPort; - List lists = new ArrayList<>(); - for (String server : servers) { - hostAndPort = StringUtils.split(server, ":"); - host = hostAndPort[0]; - port = Integer.parseInt(hostAndPort[1]); - lists.add(new ServerAddress(host, port)); - } - if (!StringUtils.isEmpty(userName) || !StringUtils.isEmpty(password)) { - credential = MongoCredential.createCredential(userName, database, password.toCharArray()); - // To connect to mongodb server - mongoClient = new MongoClient(lists, credential, new MongoClientOptions.Builder().build()); - } else { - mongoClient = new MongoClient(lists); - } - db = mongoClient.getDatabase(database); - dbCollection = db.getCollection(tableName, Document.class); - return dbCollection; - } catch (Exception e) { - throw new RuntimeException("[connMongoDB]:" + e.getMessage()); - } + mongoClient = new MongoClient(new MongoClientURI(address)); + db = mongoClient.getDatabase(database); + dbCollection = db.getCollection(tableName, Document.class); + return dbCollection; + } private void loadData(Map>> tmpCache) throws SQLException { @@ -218,8 +195,7 @@ private void loadData(Map>> tmpCache) throws SQ try { for (int i = 0; i < CONN_RETRY_NUM; i++) { try { - dbCollection = getConn(tableInfo.getAddress(), tableInfo.getUserName(), tableInfo.getPassword(), - tableInfo.getDatabase(), tableInfo.getTableName()); + dbCollection = getConn(tableInfo.getAddress(), tableInfo.getDatabase(), tableInfo.getTableName()); break; } catch (Exception e) { if (i == CONN_RETRY_NUM - 1) { diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java index b2d5c2686..4a33f1a69 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; @@ -36,16 +36,16 @@ * * @author xuqianjin */ -public class MongoAllSideInfo extends SideInfo{ +public class MongoAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = -1696860430075523841L; - public MongoAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } @Override diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 42749e935..d85a9990d 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -27,33 +27,35 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.google.common.collect.Lists; import com.mongodb.BasicDBObject; import com.mongodb.Block; -import com.mongodb.MongoCredential; -import com.mongodb.ServerAddress; +import com.mongodb.ConnectionString; import com.mongodb.async.SingleResultCallback; import com.mongodb.async.client.MongoClient; -import com.mongodb.async.client.MongoClientSettings; +import com.mongodb.MongoClientSettings; import com.mongodb.async.client.MongoClients; import com.mongodb.async.client.MongoCollection; import com.mongodb.async.client.MongoDatabase; -import com.mongodb.connection.ClusterSettings; -import com.mongodb.connection.ConnectionPoolSettings; -import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import com.google.common.collect.Lists; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -66,68 +68,43 @@ * * @author xuqianjin */ -public class MongoAsyncReqRow extends AsyncReqRow { +public class MongoAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = -1183158242862673706L; private static final Logger LOG = LoggerFactory.getLogger(MongoAsyncReqRow.class); - private final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = 20; - private transient MongoClient mongoClient; private MongoDatabase db; - private MongoSideTableInfo MongoSideTableInfo; + private MongoSideTableInfo mongoSideTableInfo; - public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MongoAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override public void open(Configuration parameters) throws Exception { super.open(parameters); - MongoSideTableInfo = (MongoSideTableInfo) sideInfo.getSideTableInfo(); - connMongoDB(); + mongoSideTableInfo = (MongoSideTableInfo) sideInfo.getSideTableInfo(); + connMongoDb(); } - public void connMongoDB() throws Exception { - MongoCredential mongoCredential; - String[] servers = StringUtils.split(MongoSideTableInfo.getAddress(), ","); - String host; - Integer port; - String[] hostAndPort; - List lists = new ArrayList<>(); - for (String server : servers) { - hostAndPort = StringUtils.split(server, ":"); - host = hostAndPort[0]; - port = Integer.parseInt(hostAndPort[1]); - lists.add(new ServerAddress(host, port)); - } - ClusterSettings clusterSettings = ClusterSettings.builder().hosts(lists).build(); - ConnectionPoolSettings connectionPoolSettings = ConnectionPoolSettings.builder() - .maxSize(DEFAULT_MAX_DB_CONN_POOL_SIZE) + public void connMongoDb() throws Exception { + String address = mongoSideTableInfo.getAddress(); + ConnectionString connectionString = new ConnectionString(address); + + MongoClientSettings settings = MongoClientSettings.builder() + .applyConnectionString(connectionString) .build(); - if (!StringUtils.isEmpty(MongoSideTableInfo.getUserName()) || !StringUtils.isEmpty(MongoSideTableInfo.getPassword())) { - mongoCredential = MongoCredential.createCredential(MongoSideTableInfo.getUserName(), MongoSideTableInfo.getDatabase(), - MongoSideTableInfo.getPassword().toCharArray()); - MongoClientSettings settings = MongoClientSettings.builder().credential(mongoCredential) - .clusterSettings(clusterSettings) - .connectionPoolSettings(connectionPoolSettings) - .build(); - mongoClient = MongoClients.create(settings); - } else { - MongoClientSettings settings = MongoClientSettings.builder().clusterSettings(clusterSettings) - .connectionPoolSettings(connectionPoolSettings) - .build(); - mongoClient = MongoClients.create(settings); - } - db = mongoClient.getDatabase(MongoSideTableInfo.getDatabase()); + mongoClient = MongoClients.create(settings); + db = mongoClient.getDatabase(mongoSideTableInfo.getDatabase()); } @Override public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { CRow inputCopy = new CRow(input.row(), input.change()); - BasicDBObject basicDBObject = new BasicDBObject(); + BasicDBObject basicDbObject = new BasicDBObject(); for (int i = 0; i < sideInfo.getEqualFieldList().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = inputCopy.row().getField(conValIndex); @@ -135,14 +112,14 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce dealMissKey(inputCopy, resultFuture); return; } - basicDBObject.put(sideInfo.getEqualFieldList().get(i), equalObj); + basicDbObject.put(sideInfo.getEqualFieldList().get(i), equalObj); } try { // 填充谓词 sideInfo.getSideTableInfo().getPredicateInfoes().stream().map(info -> { BasicDBObject filterCondition = MongoUtil.buildFilterObject(info); if (null != filterCondition) { - basicDBObject.append(info.getFieldName(), filterCondition); + basicDbObject.append(info.getFieldName(), filterCondition); } return info; }).count(); @@ -150,7 +127,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce LOG.info("add predicate infoes error ", e); } - String key = buildCacheKey(basicDBObject.values()); + String key = buildCacheKey(basicDbObject.values()); if (openCache()) { CacheObj val = getFromCache(key); if (val != null) { @@ -172,7 +149,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce } } AtomicInteger atomicInteger = new AtomicInteger(0); - MongoCollection dbCollection = db.getCollection(MongoSideTableInfo.getTableName(), Document.class); + MongoCollection dbCollection = db.getCollection(mongoSideTableInfo.getTableName(), Document.class); List cacheContent = Lists.newArrayList(); Block printDocumentBlock = new Block() { @Override @@ -198,7 +175,7 @@ public void onResult(final Void result, final Throwable t) { } } }; - dbCollection.find(basicDBObject).forEach(printDocumentBlock, callbackWhenFinished); + dbCollection.find(basicDbObject).forEach(printDocumentBlock, callbackWhenFinished); } @Override diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java index de81ed1f1..19534d4e1 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java @@ -20,16 +20,13 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; -import com.dtstack.flink.sql.util.ParseUtils; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; import java.util.List; @@ -39,16 +36,16 @@ * * @author xuqianjin */ -public class MongoAsyncSideInfo extends SideInfo { +public class MongoAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = -3694857194254465989L; - public MongoAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java index 5edc12419..4fe1ebee1 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java @@ -19,15 +19,13 @@ package com.dtstack.flink.sql.side.mongo.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -35,7 +33,7 @@ * * @author xuqianjin */ -public class MongoSideParser extends AbsSideTableParser { +public class MongoSideParser extends AbstractSideTableParser { public static final String ADDRESS_KEY = "address"; @@ -48,7 +46,7 @@ public class MongoSideParser extends AbsSideTableParser { public static final String DATABASE_KEY = "database"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { MongoSideTableInfo mongoSideTableInfo = new MongoSideTableInfo(); mongoSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, mongoSideTableInfo); diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java index a5c834469..c83de7bbc 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.mongo.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; /** @@ -28,7 +28,7 @@ * * @author xuqianjin */ -public class MongoSideTableInfo extends SideTableInfo { +public class MongoSideTableInfo extends AbstractSideTableInfo { private static final long serialVersionUID = -1L; diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java index 2f1a2092d..cbcba8f97 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java @@ -55,6 +55,7 @@ public static BasicDBObject buildFilterObject(PredicateInfo info) { return new BasicDBObject("$exists", true); case "IS NULL": return new BasicDBObject("$exists", false); + default: } return null; } diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index 337fbad59..3f1f9f7e2 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -19,20 +19,16 @@ package com.dtstack.flink.sql.sink.mongo; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.types.Row; - -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; -import com.mongodb.MongoCredential; -import com.mongodb.ServerAddress; +import com.mongodb.MongoClientURI; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.result.UpdateResult; -import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.types.Row; import org.bson.Document; import org.bson.types.ObjectId; import org.slf4j.Logger; @@ -48,7 +44,7 @@ * * @author xuqianjin */ -public class MongoOutputFormat extends DtRichOutputFormat { +public class MongoOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(MongoOutputFormat.class); private String address; @@ -125,30 +121,8 @@ public void close() { } private void establishConnection() { - try { - MongoCredential credential; - String[] servers = StringUtils.split(address, ","); - String host; - Integer port; - String[] hostAndPort; - List lists = new ArrayList<>(); - for (String server : servers) { - hostAndPort = StringUtils.split(server, ":"); - host = hostAndPort[0]; - port = Integer.parseInt(hostAndPort[1]); - lists.add(new ServerAddress(host, port)); - } - if (!StringUtils.isEmpty(userName) || !StringUtils.isEmpty(password)) { - credential = MongoCredential.createCredential(userName, database, password.toCharArray()); - // To connect to mongodb server - mongoClient = new MongoClient(lists, credential, new MongoClientOptions.Builder().build()); - } else { - mongoClient = new MongoClient(lists); - } - db = mongoClient.getDatabase(database); - } catch (Exception e) { - throw new IllegalArgumentException("[connMongoDB]:" + e.getMessage()); - } + mongoClient = new MongoClient(new MongoClientURI(address)); + db = mongoClient.getDatabase(database); } private MongoOutputFormat() { diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java index 4e28d8fd2..3f34055ec 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.mongo.table.MongoTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -54,7 +54,7 @@ public MongoSink() { } @Override - public MongoSink genStreamSink(TargetTableInfo targetTableInfo) { + public MongoSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { MongoTableInfo mongoTableInfo = (MongoTableInfo) targetTableInfo; this.address = mongoTableInfo.getAddress(); this.tableName = mongoTableInfo.getTableName(); diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java index d4810fb6d..f093b70aa 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java @@ -19,13 +19,13 @@ package com.dtstack.flink.sql.sink.mongo.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -34,7 +34,7 @@ * @author xuqianjin */ -public class MongoSinkParser extends AbsTableParser { +public class MongoSinkParser extends AbstractTableParser { private static final String CURR_TYPE = "mongo"; @@ -49,7 +49,7 @@ public class MongoSinkParser extends AbsTableParser { public static final String DATABASE_KEY = "database"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { MongoTableInfo mongoTableInfo = new MongoTableInfo(); mongoTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, mongoTableInfo); diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java index 02a96d6bb..c9d94dfe6 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.mongo.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; /** @@ -30,7 +30,7 @@ */ -public class MongoTableInfo extends TargetTableInfo { +public class MongoTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "mongo"; diff --git a/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java deleted file mode 100644 index 33a0233ac..000000000 --- a/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.dtstack.flinkx; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -/** - * Unit test for simple App. - */ -public class AppTest - extends TestCase -{ - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest( String testName ) - { - super( testName ); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() - { - return new TestSuite( AppTest.class ); - } - - /** - * Rigourous Test :-) - */ - public void testApp() - { - assertTrue( true ); - } -} diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java index b6b7f45cb..eb294e64e 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; @@ -41,7 +41,7 @@ * @author xuchao */ -public class MysqlAllReqRow extends RdbAllReqRow { +public class MysqlAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -49,18 +49,18 @@ public class MysqlAllReqRow extends RdbAllReqRow { private static final String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; - public MysqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MysqlAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Class.forName(MYSQL_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java index 07b06ba38..503dbf9a3 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class MysqlAllSideInfo extends RdbAllSideInfo { - public MysqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index de673aea7..d52f1918f 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -30,8 +30,6 @@ import io.vertx.ext.jdbc.JDBCClient; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; @@ -44,12 +42,9 @@ */ public class MysqlAsyncReqRow extends RdbAsyncReqRow { - - private static final Logger LOG = LoggerFactory.getLogger(MysqlAsyncReqRow.class); - private final static String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; - public MysqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MysqlAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -76,7 +71,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); } } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java index 5d6ed370d..9555897aa 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -36,7 +36,7 @@ public class MysqlAsyncSideInfo extends RdbAsyncSideInfo { - public MysqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java index 40f68e7e4..b0a38e344 100644 --- a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java +++ b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.mysql.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -37,8 +37,8 @@ public class MysqlSideParser extends RdbSideParser { private static final String CURR_TYPE = "mysql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java index 8c1146ede..0a1749f04 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; @@ -33,7 +33,7 @@ * @author xuchao */ -public class MysqlSink extends RdbSink implements IStreamSinkGener { +public class MysqlSink extends AbstractRdbSink implements IStreamSinkGener { public MysqlSink() { super(new MySQLDialect()); @@ -42,7 +42,7 @@ public MysqlSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java index 2247eb8cc..49105a7a8 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.mysql.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -36,8 +36,8 @@ public class MysqlSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "mysql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java deleted file mode 100644 index 33a0233ac..000000000 --- a/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.dtstack.flinkx; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -/** - * Unit test for simple App. - */ -public class AppTest - extends TestCase -{ - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest( String testName ) - { - super( testName ); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() - { - return new TestSuite( AppTest.class ); - } - - /** - * Rigourous Test :-) - */ - public void testApp() - { - assertTrue( true ); - } -} diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java index 18d9ba045..3be6687ec 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; @@ -36,22 +36,22 @@ /** * side operator with cache for all(period reload) */ -public class OracleAllReqRow extends RdbAllReqRow { +public class OracleAllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(OracleAllReqRow.class); private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; - public OracleAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new OracleAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Class.forName(ORACLE_DRIVER); Map addParams = Maps.newHashMap(); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java index 361366929..342533681 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java @@ -19,18 +19,17 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; public class OracleAllSideInfo extends RdbAllSideInfo { - public OracleAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index 20904a061..938fd870c 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -30,19 +30,14 @@ import io.vertx.ext.jdbc.JDBCClient; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; public class OracleAsyncReqRow extends RdbAsyncReqRow { - - private static final Logger LOG = LoggerFactory.getLogger(OracleAsyncReqRow.class); - private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; - public OracleAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new OracleAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -68,6 +63,6 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); } } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java index 8e801970f..658020b1c 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java @@ -20,19 +20,20 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; + import java.util.List; public class OracleAsyncSideInfo extends RdbAsyncSideInfo { - public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -55,7 +56,7 @@ public String wrapperPlaceholder(String fieldName) { String rpadFormat = "rpad(?, %d, ' ')"; if (StringUtils.contains(type.toLowerCase(), "char")) { - TableInfo.FieldExtraInfo fieldExtraInfo = sideTableInfo.getFieldExtraInfoList().get(pos); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = sideTableInfo.getFieldExtraInfoList().get(pos); int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); if (charLength > 0) { return String.format(rpadFormat, charLength); diff --git a/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java b/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java index f9124bd10..897b77d30 100644 --- a/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java +++ b/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side.oracle.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -27,8 +27,8 @@ public class OracleSideParser extends RdbSideParser { private static final String CURR_TYPE = "oracle"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); oracleTableInfo.setType(CURR_TYPE); return oracleTableInfo; } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java index a687aa012..3a320658c 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.oracle; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; @@ -41,7 +41,7 @@ public class OracleDialect implements JDBCDialect { private List fieldList; private List fieldTypeList; - private List fieldExtraInfoList; + private List fieldExtraInfoList; @Override public boolean canHandle(String url) { @@ -133,7 +133,7 @@ public String wrapperPlaceholder(String fieldName) { String type = fieldTypeList.get(pos); if (StringUtils.contains(type.toLowerCase(), DEAL_CHAR_KEY)) { - TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfoList.get(pos); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfoList.get(pos); int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); if (charLength > 0) { return String.format(RPAD_FORMAT, charLength); @@ -151,7 +151,7 @@ public void setFieldTypeList(List fieldTypeList) { this.fieldTypeList = fieldTypeList; } - public void setFieldExtraInfoList(List fieldExtraInfoList) { + public void setFieldExtraInfoList(List fieldExtraInfoList) { this.fieldExtraInfoList = fieldExtraInfoList; } } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index 7a37aa6cd..f916b55f2 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** @@ -29,7 +29,7 @@ * * @author maqi */ -public class OracleSink extends RdbSink implements IStreamSinkGener { +public class OracleSink extends AbstractRdbSink implements IStreamSinkGener { public OracleSink() { super(new OracleDialect()); @@ -42,7 +42,7 @@ public JDBCUpsertOutputFormat getOutputFormat() { ((OracleDialect) jdbcDialect).setFieldExtraInfoList(fieldExtraInfoList); JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java index aff096bd3..1732c8cc2 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.oracle.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -33,8 +33,8 @@ public class OracleSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "oracle"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); oracleTableInfo.setType(CURR_TYPE); return oracleTableInfo; } diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java index f9e0950dd..5fbf0824b 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java @@ -1,9 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.collect.Maps; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -15,7 +32,12 @@ import java.util.List; import java.util.Map; -public class PolardbAllReqRow extends RdbAllReqRow { +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ +public class PolardbAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -23,18 +45,18 @@ public class PolardbAllReqRow extends RdbAllReqRow { private static final String POLARDB_DRIVER = "com.mysql.cj.jdbc.Driver"; - public PolardbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PolardbAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Class.forName(POLARDB_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java index 822cbaa70..c46c9dfd6 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java @@ -1,15 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbAllSideInfo extends RdbAllSideInfo { - public PolardbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 52e57e262..13d25b111 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -1,8 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -16,13 +33,18 @@ import java.util.List; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbAsyncReqRow extends RdbAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(PolardbAsyncReqRow.class); private final static String POLARDB_DRIVER = "com.mysql.cj.jdbc.Driver"; - public PolardbAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PolardbAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -48,6 +70,6 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); } } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java index bad71375f..15237956b 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java @@ -1,16 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbAsyncSideInfo extends RdbAsyncSideInfo { - public PolardbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java index 348b3aed9..71aad7cbb 100644 --- a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java +++ b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java @@ -1,16 +1,39 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbSideParser extends RdbSideParser { private static final String CURR_TYPE = "polardb"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java index 44d558140..c69f31e8f 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java @@ -1,11 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.dtstack.flink.sql.sink.polardb; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; - -public class PolardbSink extends RdbSink { +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ +public class PolardbSink extends AbstractRdbSink { public PolardbSink() { super(new PolardbDialect()); } @@ -13,7 +35,7 @@ public PolardbSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java index 68eead938..b4f02665e 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java @@ -1,16 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.sink.polardb.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "polardb"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo polardbTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo polardbTableInfo = super.getTableInfo(tableName, fieldsInfo, props); polardbTableInfo.setType(CURR_TYPE); return polardbTableInfo; } diff --git a/pom.xml b/pom.xml index 2794f9004..de963240c 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,7 @@ mysql hbase elasticsearch5 + elasticsearch6 mongo redis5 launcher @@ -33,7 +34,6 @@ impala db2 polardb - elasticsearch6 @@ -54,6 +54,20 @@ + + org.jacoco + jacoco-maven-plugin + 0.7.8 + + + + prepare-agent + report + + + + + pl.project13.maven git-commit-id-plugin diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java index 6d68cfdca..0ec8f96ce 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; @@ -41,7 +41,7 @@ * @author tcm */ -public class PostgresqlAllReqRow extends RdbAllReqRow { +public class PostgresqlAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -49,18 +49,18 @@ public class PostgresqlAllReqRow extends RdbAllReqRow { private static final String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - public PostgresqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PostgresqlAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Class.forName(POSTGRESQL_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java index d383ee46d..a4a52eabc 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class PostgresqlAllSideInfo extends RdbAllSideInfo { - public PostgresqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index bed10f206..42aa39a51 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -48,7 +48,7 @@ public class PostgresqlAsyncReqRow extends RdbAsyncReqRow { private final static String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - public PostgresqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PostgresqlAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -71,7 +71,7 @@ public void open(Configuration parameters) throws Exception { vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, pgClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, pgClientConfig)); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java index 1d89f4894..f19a488bc 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -36,7 +36,7 @@ public class PostgresqlAsyncSideInfo extends RdbAsyncSideInfo { - public PostgresqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java b/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java index faee2c704..be0c5ea8b 100644 --- a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java +++ b/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.postgresql.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -37,8 +37,8 @@ public class PostgresqlSideParser extends RdbSideParser { private static final String CURR_TYPE = "postgresql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); pgTableInfo.setType(CURR_TYPE); return pgTableInfo; } diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java index cf45a6bd9..bb731152c 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java @@ -22,13 +22,13 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** * @author maqi */ -public class PostgresqlSink extends RdbSink implements IStreamSinkGener { +public class PostgresqlSink extends AbstractRdbSink implements IStreamSinkGener { public PostgresqlSink() { super(new PostgresqlDialect()); } @@ -36,7 +36,7 @@ public PostgresqlSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java index 5943b5e52..e94cfc8a0 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.postgresql.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -35,8 +35,8 @@ public class PostgresqlSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "postgresql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); pgTableInfo.setType(CURR_TYPE); return pgTableInfo; } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java similarity index 96% rename from rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java rename to rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index c3220968d..bb88c7787 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -25,8 +25,9 @@ import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import com.dtstack.flink.sql.side.AllReqRow; -import com.dtstack.flink.sql.side.SideInfo; + +import com.dtstack.flink.sql.side.BaseAllReqRow; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import com.google.common.collect.Lists; @@ -57,11 +58,11 @@ * @author maqi */ -public abstract class RdbAllReqRow extends AllReqRow { +public abstract class AbstractRdbAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 2098635140857937718L; - private static final Logger LOG = LoggerFactory.getLogger(RdbAllReqRow.class); + private static final Logger LOG = LoggerFactory.getLogger(AbstractRdbAllReqRow.class); private static final int CONN_RETRY_NUM = 3; @@ -69,7 +70,7 @@ public abstract class RdbAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public RdbAllReqRow(SideInfo sideInfo) { + public AbstractRdbAllReqRow(BaseSideInfo sideInfo) { super(sideInfo); } @@ -91,6 +92,7 @@ protected void initCache() throws SQLException { protected void reloadCache() { //reload cacheRef and replace to old cacheRef Map>> newCache = Maps.newConcurrentMap(); + cacheRef.set(newCache); try { loadData(newCache); } catch (SQLException e) { diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 3b7444b77..143017133 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -23,14 +23,16 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; @@ -44,20 +46,21 @@ * @author maqi */ -public class RdbAllSideInfo extends SideInfo { +public class RdbAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = -5858335638589472159L; + private static final Logger LOG = LoggerFactory.getLogger(RdbAllSideInfo.class.getSimpleName()); - public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideTableInfo; sqlCondition = getSelectFromStatement(getTableName(rdbSideTableInfo), Arrays.asList(StringUtils.split(sideSelectFields, ",")), sideTableInfo.getPredicateInfoes()); - System.out.println("--------dimension sql query-------\n" + sqlCondition); + LOG.info("--------dimension sql query-------\n{}" + sqlCondition); } public String getAdditionalWhereClause() { diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 971285e52..9af6d02fd 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -20,7 +20,9 @@ package com.dtstack.flink.sql.side.rdb.async; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; @@ -30,6 +32,7 @@ import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import com.google.common.collect.Lists; +import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -50,7 +53,7 @@ * @author maqi */ -public class RdbAsyncReqRow extends AsyncReqRow { +public class RdbAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 2098635244857937720L; @@ -72,20 +75,27 @@ public class RdbAsyncReqRow extends AsyncReqRow { public final static String PREFERRED_TEST_QUERY_SQL = "select 1 from dual"; - private transient SQLClient rdbSQLClient; + private transient SQLClient rdbSqlClient; - public RdbAsyncReqRow(SideInfo sideInfo) { + public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); init(sideInfo); } - protected void init(SideInfo sideInfo) { + protected void init(BaseSideInfo sideInfo) { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); int defaultAsyncPoolSize = Math.min(MAX_DB_CONN_POOL_SIZE_LIMIT, DEFAULT_DB_CONN_POOL_SIZE); int rdbPoolSize = rdbSideTableInfo.getAsyncPoolSize() > 0 ? rdbSideTableInfo.getAsyncPoolSize() : defaultAsyncPoolSize; rdbSideTableInfo.setAsyncPoolSize(rdbPoolSize); } + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + LOG.info("rdb dim table config info: {} ", rdbSideTableInfo.toString()); + } + @Override public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { CRow copyCrow = new CRow(input.row(), input.change()); @@ -238,8 +248,8 @@ public Row fillData(Row input, Object line) { @Override public void close() throws Exception { super.close(); - if (rdbSQLClient != null) { - rdbSQLClient.close(); + if (rdbSqlClient != null) { + rdbSqlClient.close(); } } @@ -254,8 +264,8 @@ public String buildCacheKey(JsonArray jsonArray) { return sb.toString(); } - public void setRdbSQLClient(SQLClient rdbSQLClient) { - this.rdbSQLClient = rdbSQLClient; + public void setRdbSqlClient(SQLClient rdbSqlClient) { + this.rdbSqlClient = rdbSqlClient; } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java index f3830793e..4b6782494 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java @@ -23,8 +23,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; @@ -33,6 +33,8 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; @@ -48,16 +50,18 @@ * @author maqi */ -public class RdbAsyncSideInfo extends SideInfo { +public class RdbAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = 1942629132469918611L; + private static final Logger LOG = LoggerFactory.getLogger(RdbAsyncSideInfo.class); - public RdbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + + public RdbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideTableInfo; String sideTableName = joinInfo.getSideTableName(); @@ -75,7 +79,7 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { sqlCondition = getSelectFromStatement(getTableName(rdbSideTableInfo), Arrays.asList(StringUtils.split(sideSelectFields, ",")), equalFieldList, sqlJoinCompareOperate, sideTableInfo.getPredicateInfoes()); - System.out.println("----------dimension sql query-----------\n" + sqlCondition); + LOG.info("----------dimension sql query-----------\n{}", sqlCondition); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java index fcca4f0ef..542258257 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java @@ -25,13 +25,19 @@ import javax.sql.DataSource; import java.beans.PropertyVetoException; import java.sql.SQLException; - +/** + * Date: 2019/9/17 + * Company: www.dtstack.com + * @author maqi + */ public class DTC3P0DataSourceProvider extends C3P0DataSourceProvider { @Override public DataSource getDataSource(JsonObject config) throws SQLException { String url = config.getString("url"); - if (url == null) throw new NullPointerException("url cannot be null"); + if (url == null) { + throw new NullPointerException("url cannot be null"); + } String driverClass = config.getString("driver_class"); String user = config.getString("user"); String password = config.getString("password"); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java index b9811b0ee..79942ae11 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.side.rdb.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -33,10 +33,10 @@ * @author maqi */ -public class RdbSideParser extends AbsSideTableParser { +public class RdbSideParser extends AbstractSideTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RdbSideTableInfo rdbTableInfo = new RdbSideTableInfo(); rdbTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, rdbTableInfo); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java index fdb024b1d..46a60a10b 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java @@ -17,7 +17,7 @@ */ package com.dtstack.flink.sql.side.rdb.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; /** @@ -27,7 +27,7 @@ * * @author maqi */ -public class RdbSideTableInfo extends SideTableInfo { +public class RdbSideTableInfo extends AbstractSideTableInfo { private static final long serialVersionUID = -1L; diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java index 1205c5521..49d163d4d 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java @@ -78,6 +78,7 @@ public static Object getTarget(Object obj, String targetType) { case "timestamp": case "datetime": return MathUtil.getTimestamp(obj); + default: } return obj; } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java similarity index 90% rename from rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java rename to rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java index 84eea68ff..ee51c22d1 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -44,11 +44,8 @@ * * @author maqi */ -public abstract class RdbSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { - - protected String driverName; - - protected String dbURL; +public abstract class AbstractRdbSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { + protected String dbUrl; protected String userName; @@ -86,20 +83,20 @@ public abstract class RdbSink implements RetractStreamTableSink, Serializab public List fieldList; public List fieldTypeList; - public List fieldExtraInfoList; + public List fieldExtraInfoList; - public RdbSink(JDBCDialect jdbcDialect) { + public AbstractRdbSink(JDBCDialect jdbcDialect) { this.jdbcDialect = jdbcDialect; } @Override - public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { + public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { RdbTableInfo rdbTableInfo = (RdbTableInfo) targetTableInfo; this.batchNum = rdbTableInfo.getBatchSize() == null ? batchNum : rdbTableInfo.getBatchSize(); this.batchWaitInterval = rdbTableInfo.getBatchWaitInterval() == null ? batchWaitInterval : rdbTableInfo.getBatchWaitInterval(); this.parallelism = rdbTableInfo.getParallelism() == null ? parallelism : rdbTableInfo.getParallelism(); - this.dbURL = rdbTableInfo.getUrl(); + this.dbUrl = rdbTableInfo.getUrl(); this.userName = rdbTableInfo.getUserName(); this.password = rdbTableInfo.getPassword(); this.tableName = rdbTableInfo.getTableName(); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java index e9a4f391f..f476bfadf 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java @@ -29,7 +29,7 @@ public class JDBCOptions { - private String dbURL; + private String dbUrl; private String tableName; private String driverName; private String username; @@ -37,9 +37,9 @@ public class JDBCOptions { private String schema; private JDBCDialect dialect; - private JDBCOptions(String dbURL, String tableName, String driverName, String username, + private JDBCOptions(String dbUrl, String tableName, String driverName, String username, String password, String schema, JDBCDialect dialect) { - this.dbURL = dbURL; + this.dbUrl = dbUrl; this.tableName = tableName; this.driverName = driverName; this.username = username; @@ -48,8 +48,8 @@ private JDBCOptions(String dbURL, String tableName, String driverName, String us this.dialect = dialect; } - public String getDbURL() { - return dbURL; + public String getDbUrl() { + return dbUrl; } public String getTableName() { @@ -84,7 +84,7 @@ public static Builder builder() { public boolean equals(Object o) { if (o instanceof JDBCOptions) { JDBCOptions options = (JDBCOptions) o; - return Objects.equals(dbURL, options.dbURL) && + return Objects.equals(dbUrl, options.dbUrl) && Objects.equals(tableName, options.tableName) && Objects.equals(driverName, options.driverName) && Objects.equals(username, options.username) && @@ -100,7 +100,7 @@ public boolean equals(Object o) { * Builder of {@link JDBCOptions}. */ public static class Builder { - private String dbURL; + private String dbUrl; private String tableName; private String driverName; private String username; @@ -152,8 +152,8 @@ public Builder setSchema(String schema) { /** * required, JDBC DB url. */ - public Builder setDBUrl(String dbURL) { - this.dbURL = dbURL; + public Builder setDbUrl(String dbUrl) { + this.dbUrl = dbUrl; return this; } @@ -163,7 +163,7 @@ public Builder setDialect(JDBCDialect dialect) { } public JDBCOptions build() { - checkNotNull(dbURL, "No dbURL supplied."); + checkNotNull(dbUrl, "No dbURL supplied."); checkNotNull(tableName, "No tableName supplied."); if (this.driverName == null) { @@ -173,7 +173,7 @@ public JDBCOptions build() { }); } - return new JDBCOptions(dbURL, tableName, driverName, username, password, schema, dialect); + return new JDBCOptions(dbUrl, tableName, driverName, username, password, schema, dialect); } } } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java index 73f17d8fd..2a3ce5e90 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.format; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; @@ -36,7 +36,7 @@ * @see Row * @see DriverManager */ -public abstract class AbstractJDBCOutputFormat extends DtRichOutputFormat { +public abstract class AbstractJDBCOutputFormat extends AbstractDtRichOutputFormat { private static final long serialVersionUID = 1L; public static final int DEFAULT_FLUSH_MAX_SIZE = 100; @@ -47,16 +47,16 @@ public abstract class AbstractJDBCOutputFormat extends DtRichOutputFormat protected final String username; protected final String password; - private final String drivername; + private final String driverName; protected final String dbURL; protected transient Connection connection; - public AbstractJDBCOutputFormat(String username, String password, String drivername, String dbURL) { + public AbstractJDBCOutputFormat(String username, String password, String driverName, String dbUrl) { this.username = username; this.password = password; - this.drivername = drivername; - this.dbURL = dbURL; + this.driverName = driverName; + this.dbURL = dbUrl; } @Override @@ -64,7 +64,7 @@ public void configure(Configuration parameters) { } protected void establishConnection() throws SQLException, ClassNotFoundException, IOException { - JDBCUtils.forName(drivername, getClass().getClassLoader()); + JDBCUtils.forName(driverName, getClass().getClassLoader()); if (username == null) { connection = DriverManager.getConnection(dbURL); } else { diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index e1b2cc6f4..e8b6dc8a4 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -25,7 +25,7 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.dtstack.flink.sql.sink.rdb.writer.AppendOnlyWriter; import com.dtstack.flink.sql.sink.rdb.writer.JDBCWriter; -import com.dtstack.flink.sql.sink.rdb.writer.UpsertWriter; +import com.dtstack.flink.sql.sink.rdb.writer.AbstractUpsertWriter; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; @@ -44,6 +44,7 @@ /** * An upsert OutputFormat for JDBC. + * @author maqi */ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat> { @@ -83,7 +84,7 @@ public JDBCUpsertOutputFormat( long flushIntervalMills, boolean allReplace, String updateMode) { - super(options.getUsername(), options.getPassword(), options.getDriverName(), options.getDbURL()); + super(options.getUsername(), options.getPassword(), options.getDriverName(), options.getDbUrl()); this.schema = options.getSchema(); this.tableName = options.getTableName(); this.dialect = options.getDialect(); @@ -111,12 +112,11 @@ public void open(int taskNumber, int numTasks) throws IOException { initMetric(); if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.APPEND.name()) || keyFields == null || keyFields.length == 0) { - String insertSQL = dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields); - LOG.info("execute insert sql： {}", insertSQL); - System.out.println("execute insert sql :" + insertSQL); - jdbcWriter = new AppendOnlyWriter(insertSQL, fieldTypes, this); + String insertSql = dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields); + LOG.info("execute insert sql： {}", insertSql); + jdbcWriter = new AppendOnlyWriter(insertSql, fieldTypes, this); } else { - jdbcWriter = UpsertWriter.create( + jdbcWriter = AbstractUpsertWriter.create( dialect, schema, tableName, fieldNames, fieldTypes, keyFields, partitionFields, getRuntimeContext().getExecutionConfig().isObjectReuseEnabled(), allReplace, this); } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java new file mode 100644 index 000000000..e69de29bb diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java index 9bbff3b4d..aca7acb6c 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java @@ -17,9 +17,8 @@ */ package com.dtstack.flink.sql.sink.rdb.table; -import com.dtstack.flink.sql.constrant.ConfigConstrant; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -31,9 +30,9 @@ * * @author maqi */ -public class RdbSinkParser extends AbsTableParser { +public class RdbSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RdbTableInfo rdbTableInfo = new RdbTableInfo(); rdbTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, rdbTableInfo); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java index e490978e4..ccdf28e48 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java @@ -18,12 +18,10 @@ package com.dtstack.flink.sql.sink.rdb.table; import com.dtstack.flink.sql.enums.EUpdateMode; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; -import java.util.Arrays; - /** * Reason: * Date: 2018/11/27 @@ -31,7 +29,7 @@ * * @author maqi */ -public class RdbTableInfo extends TargetTableInfo { +public class RdbTableInfo extends AbstractTargetTableInfo { public static final String URL_KEY = "url"; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java similarity index 84% rename from rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java rename to rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java index ce3a1c680..2c070b680 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.writer; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; @@ -41,14 +41,15 @@ /** * Upsert writer to deal with upsert, delete message.dd + * @author maqi */ -public abstract class UpsertWriter implements JDBCWriter { +public abstract class AbstractUpsertWriter implements JDBCWriter { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(UpsertWriter.class); + private static final Logger LOG = LoggerFactory.getLogger(AbstractUpsertWriter.class); - public static UpsertWriter create( + public static AbstractUpsertWriter create( JDBCDialect dialect, String schema, String tableName, @@ -58,7 +59,7 @@ public static UpsertWriter create( String[] partitionFields, boolean objectReuse, boolean allReplace, - DtRichOutputFormat metricOutputFormat) { + AbstractDtRichOutputFormat metricOutputFormat) { checkNotNull(keyFields); @@ -67,20 +68,18 @@ public static UpsertWriter create( int[] pkTypes = fieldTypes == null ? null : Arrays.stream(pkFields).map(f -> fieldTypes[f]).toArray(); - String deleteSQL = dialect.getDeleteStatement(schema, tableName, keyFields); - LOG.info("deleteSQL is :{}", deleteSQL); - System.out.println("deleteSQL is :" + deleteSQL); + String deleteSql = dialect.getDeleteStatement(schema, tableName, keyFields); + LOG.info("deleteSQL is :{}", deleteSql); - Optional upsertSQL = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); - LOG.info("execute UpsertStatement: {}", upsertSQL.orElse("use UsingInsertUpdateStatement")); - System.out.println("execute UpsertStatement: " + upsertSQL.orElse("use UsingInsertUpdateStatement")); + Optional upsertSql = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); + LOG.info("execute UpsertStatement: {}", upsertSql.orElse("use UsingInsertUpdateStatement")); - return upsertSQL.map((Function) sql -> + return upsertSql.map((Function) sql -> new UpsertWriterUsingUpsertStatement( - fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, sql, metricOutputFormat)) + fieldTypes, pkFields, pkTypes, objectReuse, deleteSql, sql, metricOutputFormat)) .orElseGet(() -> new UpsertWriterUsingInsertUpdateStatement( - fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, + fieldTypes, pkFields, pkTypes, objectReuse, deleteSql, dialect.getRowExistsStatement(tableName, keyFields), dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields), dialect.getUpdateStatement(tableName, fieldNames, keyFields), @@ -90,19 +89,19 @@ public static UpsertWriter create( final int[] fieldTypes; final int[] pkTypes; private final int[] pkFields; - private final String deleteSQL; + private final String deleteSql; private final boolean objectReuse; private transient Map> keyToRows; private transient PreparedStatement deleteStatement; // only use metric - private transient DtRichOutputFormat metricOutputFormat; + private transient AbstractDtRichOutputFormat metricOutputFormat; - private UpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSQL, boolean objectReuse, DtRichOutputFormat metricOutputFormat) { + private AbstractUpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSql, boolean objectReuse, AbstractDtRichOutputFormat metricOutputFormat) { this.fieldTypes = fieldTypes; this.pkFields = pkFields; this.pkTypes = pkTypes; - this.deleteSQL = deleteSQL; + this.deleteSql = deleteSql; this.objectReuse = objectReuse; this.metricOutputFormat = metricOutputFormat; } @@ -115,7 +114,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { - this.deleteStatement = connection.prepareStatement(deleteSQL); + this.deleteStatement = connection.prepareStatement(deleteSql); } @Override @@ -170,7 +169,6 @@ public void executeUpdate(Connection connection) throws SQLException { } connection.commit(); } catch (Exception e) { - System.out.println(e.getCause()); // deal pg error: current transaction is aborted, commands ignored until end of transaction block connection.rollback(); connection.commit(); @@ -207,10 +205,10 @@ private Row getPrimaryKey(Row row) { // ---------------------------------------------------------------------------------------- - private static final class UpsertWriterUsingUpsertStatement extends UpsertWriter { + private static final class UpsertWriterUsingUpsertStatement extends AbstractUpsertWriter { private static final long serialVersionUID = 1L; - private final String upsertSQL; + private final String upsertSql; private transient PreparedStatement upsertStatement; @@ -219,11 +217,11 @@ private UpsertWriterUsingUpsertStatement( int[] pkFields, int[] pkTypes, boolean objectReuse, - String deleteSQL, - String upsertSQL, - DtRichOutputFormat metricOutputFormat) { - super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); - this.upsertSQL = upsertSQL; + String deleteSql, + String upsertSql, + AbstractDtRichOutputFormat metricOutputFormat) { + super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); + this.upsertSql = upsertSql; } @Override @@ -234,7 +232,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { super.prepareStatement(connection); - upsertStatement = connection.prepareStatement(upsertSQL); + upsertStatement = connection.prepareStatement(upsertSql); } @Override @@ -264,12 +262,12 @@ public void close() throws SQLException { } } - private static final class UpsertWriterUsingInsertUpdateStatement extends UpsertWriter { + private static final class UpsertWriterUsingInsertUpdateStatement extends AbstractUpsertWriter { private static final long serialVersionUID = 1L; - private final String existSQL; - private final String insertSQL; - private final String updateSQL; + private final String existSql; + private final String insertSql; + private final String updateSql; private transient PreparedStatement existStatement; private transient PreparedStatement insertStatement; @@ -280,15 +278,15 @@ private UpsertWriterUsingInsertUpdateStatement( int[] pkFields, int[] pkTypes, boolean objectReuse, - String deleteSQL, - String existSQL, - String insertSQL, - String updateSQL, - DtRichOutputFormat metricOutputFormat) { - super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); - this.existSQL = existSQL; - this.insertSQL = insertSQL; - this.updateSQL = updateSQL; + String deleteSql, + String existSql, + String insertSql, + String updateSql, + AbstractDtRichOutputFormat metricOutputFormat) { + super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); + this.existSql = existSql; + this.insertSql = insertSql; + this.updateSql = updateSql; } @Override @@ -299,9 +297,9 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { super.prepareStatement(connection); - existStatement = connection.prepareStatement(existSQL); - insertStatement = connection.prepareStatement(insertSQL); - updateStatement = connection.prepareStatement(updateSQL); + existStatement = connection.prepareStatement(existSql); + insertStatement = connection.prepareStatement(insertSql); + updateStatement = connection.prepareStatement(updateSql); } @Override diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index ca2bd4c66..3559d4376 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.writer; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -34,6 +34,7 @@ /** * Just append record to jdbc, can not receive retract/delete message. + * @author maqi */ public class AppendOnlyWriter implements JDBCWriter { @@ -41,16 +42,16 @@ public class AppendOnlyWriter implements JDBCWriter { private static final Logger LOG = LoggerFactory.getLogger(AppendOnlyWriter.class); - private final String insertSQL; + private final String insertSql; private final int[] fieldTypes; private transient PreparedStatement statement; private transient List rows; // only use metric - private transient DtRichOutputFormat metricOutputFormat; + private transient AbstractDtRichOutputFormat metricOutputFormat; - public AppendOnlyWriter(String insertSQL, int[] fieldTypes, DtRichOutputFormat metricOutputFormat) { - this.insertSQL = insertSQL; + public AppendOnlyWriter(String insertSql, int[] fieldTypes, AbstractDtRichOutputFormat metricOutputFormat) { + this.insertSql = insertSql; this.fieldTypes = fieldTypes; this.metricOutputFormat = metricOutputFormat; } @@ -63,7 +64,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { - this.statement = connection.prepareStatement(insertSQL); + this.statement = connection.prepareStatement(insertSql); } /** diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 0d257c5b9..5d09c8f3d 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -18,6 +18,10 @@ package com.dtstack.flink.sql.side.redis; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAllReqRow; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.redis.enums.RedisType; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; @@ -29,22 +33,34 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Maps; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; +import com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.*; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.JedisCommands; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisSentinelPool; import java.io.Closeable; import java.io.IOException; import java.sql.SQLException; -import java.util.*; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; -public class RedisAllReqRow extends AllReqRow{ +import java.util.Calendar; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicReference; +/** + * @author yanxi + */ +public class RedisAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 7578879189085344807L; @@ -62,7 +78,7 @@ public class RedisAllReqRow extends AllReqRow{ private RedisSideReqRow redisSideReqRow; - public RedisAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new RedisAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); this.redisSideReqRow = new RedisSideReqRow(super.sideInfo); } @@ -90,7 +106,7 @@ protected void reloadCache() { } cacheRef.set(newCache); - LOG.info("----- Redis all cacheRef reload end:{}", newCache.size()); + LOG.info("----- Redis all cacheRef reload end:{}", Calendar.getInstance()); } @Override diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java index 81e7b1f4f..ec0fa0d15 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java @@ -20,25 +20,27 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import java.util.List; - -public class RedisAllSideInfo extends SideInfo { +/** + * @author yanxi + */ +public class RedisAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = 1998703966487857613L; - public RedisAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index f2e164922..d53bbed8c 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -18,8 +18,18 @@ package com.dtstack.flink.sql.side.redis; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; + import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.redis.enums.RedisType; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; @@ -29,24 +39,20 @@ import io.lettuce.core.api.StatefulRedisConnection; import io.lettuce.core.api.async.RedisHashAsyncCommands; import io.lettuce.core.api.async.RedisKeyAsyncCommands; -import io.lettuce.core.api.async.RedisStringAsyncCommands; import io.lettuce.core.cluster.RedisClusterClient; import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Consumer; - -public class RedisAsyncReqRow extends AsyncReqRow { +/** + * @author yanxi + */ +public class RedisAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = -2079908694523987738L; @@ -64,7 +70,7 @@ public class RedisAsyncReqRow extends AsyncReqRow { private RedisSideReqRow redisSideReqRow; - public RedisAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new RedisAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); redisSideReqRow = new RedisSideReqRow(super.sideInfo); } diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java index 9bdefe8c5..c23035ec7 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java @@ -20,24 +20,26 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import java.util.List; - -public class RedisAsyncSideInfo extends SideInfo { +/** + * @author yanxi + */ +public class RedisAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = -4851348392924455039L; - public RedisAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java index f8a83b868..514984f0a 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java @@ -18,16 +18,19 @@ package com.dtstack.flink.sql.side.redis.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -public class RedisSideParser extends AbsSideTableParser { +/** + * @author yanxi + */ +public class RedisSideParser extends AbstractSideTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RedisSideTableInfo redisSideTableInfo = new RedisSideTableInfo(); redisSideTableInfo.setName(tableName); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java index 5007868e9..17557f916 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.redis.table; import com.dtstack.flink.sql.side.ISideReqRow; -import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; @@ -40,9 +40,9 @@ public class RedisSideReqRow implements ISideReqRow, Serializable { private static final long serialVersionUID = 3751171828444748982L; - private SideInfo sideInfo; + private BaseSideInfo sideInfo; - public RedisSideReqRow(SideInfo sideInfo){ + public RedisSideReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -72,7 +72,7 @@ public Row fillData(Row input, Object sideInput) { return row; } - public void setRowField(Row row, Integer index, SideInfo sideInfo, String value) { + public void setRowField(Row row, Integer index, BaseSideInfo sideInfo, String value) { Integer keyIndex = sideInfo.getSideFieldIndex().get(index); String classType = sideInfo.getSideTableInfo().getFieldClassList().get(keyIndex).getName(); switch (classType){ diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java index 88cfcb8a2..5224134c8 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java @@ -18,10 +18,12 @@ package com.dtstack.flink.sql.side.redis.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; - -public class RedisSideTableInfo extends SideTableInfo { +/** + * @author yanxi + */ +public class RedisSideTableInfo extends AbstractSideTableInfo { private static final long serialVersionUID = -1L; diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index 0bd053473..ae4fe5a4b 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -18,14 +18,13 @@ package com.dtstack.flink.sql.sink.redis; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; - -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.HostAndPort; @@ -42,7 +41,10 @@ import java.util.List; import java.util.Set; -public class RedisOutputFormat extends DtRichOutputFormat { +/** + * @author yanxi + */ +public class RedisOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(RedisOutputFormat.class); private String url; @@ -140,7 +142,8 @@ private void establishConnection() { break; //集群 case 3: - jedis = new JedisCluster(addresses, timeout, timeout,10, password, poolConfig); + jedis = new JedisCluster(addresses, timeout, timeout, 10, password, poolConfig); + default: } } @@ -156,10 +159,10 @@ public void writeRecord(Tuple2 record) throws IOException { return; } - HashMap map = new HashMap<>(); - for (String primaryKey : primaryKeys){ - for (int i=0; i map = new HashMap<>(8); + for (String primaryKey : primaryKeys) { + for (int i = 0; i < fieldNames.length; i++) { + if (fieldNames[i].equals(primaryKey)) { map.put(primaryKey, i); } } @@ -167,10 +170,10 @@ public void writeRecord(Tuple2 record) throws IOException { List kvList = new LinkedList<>(); for (String primaryKey : primaryKeys){ - StringBuilder primaryKV = new StringBuilder(); + StringBuilder primaryKv = new StringBuilder(); int index = map.get(primaryKey).intValue(); - primaryKV.append(primaryKey).append(":").append(row.getField(index)); - kvList.add(primaryKV.toString()); + primaryKv.append(primaryKey).append(":").append(row.getField(index)); + kvList.add(primaryKv.toString()); } String perKey = String.join(":", kvList); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java index d2e28c01f..cc49a3ba8 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.redis.table.RedisTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -33,7 +33,9 @@ import org.apache.flink.types.Row; import java.util.List; - +/** + * @author yanxi + */ public class RedisSink implements RetractStreamTableSink, IStreamSinkGener { protected String[] fieldNames; @@ -67,7 +69,7 @@ public RedisSink(){ } @Override - public RedisSink genStreamSink(TargetTableInfo targetTableInfo) { + public RedisSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { RedisTableInfo redisTableInfo = (RedisTableInfo) targetTableInfo; this.url = redisTableInfo.getUrl(); this.database = redisTableInfo.getDatabase(); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 0c9b0a64a..e965eeecb 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.sink.redis.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; @@ -27,10 +27,12 @@ import java.util.Arrays; import java.util.List; import java.util.Map; - -public class RedisSinkParser extends AbsTableParser { +/** + * @author yanxi + */ +public class RedisSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RedisTableInfo redisTableInfo = new RedisTableInfo(); redisTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, redisTableInfo); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java index 60d6dd12f..2425f8c38 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java @@ -18,10 +18,12 @@ package com.dtstack.flink.sql.sink.redis.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; - -public class RedisTableInfo extends TargetTableInfo { +/** + * @author yanxi + */ +public class RedisTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "redis"; diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java index e46e0a76f..b032fdce6 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.format.DeserializationMetricWrapper; import com.dtstack.flink.sql.format.dtnest.DtNestRowDeserializationSchema; import com.dtstack.flink.sql.source.serversocket.table.ServersocketSourceTableInfo; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.flink.types.Row; @@ -67,7 +67,7 @@ public class CustomerSocketTextStreamFunction implements SourceFunction { ServersocketSourceTableInfo tableInfo; public CustomerSocketTextStreamFunction(ServersocketSourceTableInfo tableInfo, TypeInformation typeInfo, - Map rowAndFieldMapping, List fieldExtraInfos) { + Map rowAndFieldMapping, List fieldExtraInfos) { this.tableInfo = tableInfo; this.deserializationSchema = new DtNestRowDeserializationSchema(typeInfo, rowAndFieldMapping, fieldExtraInfos); this.deserializationMetricWrapper = new DeserializationMetricWrapper(typeInfo, deserializationSchema); @@ -94,7 +94,7 @@ public void run(SourceContext ctx) throws Exception { while (buffer.length() >= delimiter.length() && (delimPos = buffer.indexOf(delimiter)) != -1) { String record = buffer.substring(0, delimPos); // truncate trailing carriage return - if (delimiter.equals("\n") && record.endsWith("\r")) { + if ("\n".equals(delimiter) && "\r".endsWith(record)) { record = record.substring(0, record.length() - 1); } try { diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java index 3ab01fb42..b4dda5f3b 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.serversocket.table.ServersocketSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -37,7 +37,7 @@ */ public class ServersocketSource implements IStreamSourceGener
{ @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { ServersocketSourceTableInfo serversocketSourceTableInfo = (ServersocketSourceTableInfo) sourceTableInfo; String tableName = serversocketSourceTableInfo.getName(); diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java index b52e38e59..f2d861b17 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java @@ -17,8 +17,8 @@ */ package com.dtstack.flink.sql.source.serversocket.table; -import com.dtstack.flink.sql.table.AbsSourceParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSourceParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -30,9 +30,9 @@ * * @author maqi */ -public class ServersocketSourceParser extends AbsSourceParser { +public class ServersocketSourceParser extends AbstractSourceParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ServersocketSourceTableInfo serversocketSourceTableInfo = new ServersocketSourceTableInfo(); serversocketSourceTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, serversocketSourceTableInfo); diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java index 796728eb3..3123b477f 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java @@ -17,7 +17,7 @@ */ package com.dtstack.flink.sql.source.serversocket.table; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.google.common.base.Preconditions; /** @@ -27,8 +27,8 @@ * * @author maqi */ -public class ServersocketSourceTableInfo extends SourceTableInfo { - //version +public class ServersocketSourceTableInfo extends AbstractSourceTableInfo { + private static final String CURR_TYPE = "serversocket"; public static final String HOSTNAME_KEY = "host"; diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java index 961539fde..dbae271e4 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java @@ -20,39 +20,43 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.sql.Connection; import java.sql.DriverManager; import java.util.List; import java.util.Map; + /** - * side operator with cache for all(period reload) + * side operator with cache for all(period reload) + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi */ -public class SqlserverAllReqRow extends RdbAllReqRow { +public class SqlserverAllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(SqlserverAllReqRow.class); private static final String SQLSERVER_DRIVER = "net.sourceforge.jtds.jdbc.Driver"; - public SqlserverAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new SqlserverAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Class.forName(SQLSERVER_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); - //addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java index 2b9913e7e..a40af5697 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java @@ -20,14 +20,18 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; - +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverAllSideInfo extends RdbAllSideInfo { - public SqlserverAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index 9fc2c35cd..a63d28607 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -33,14 +33,18 @@ import java.util.List; - +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverAsyncReqRow extends RdbAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(SqlserverAsyncReqRow.class); private final static String SQLSERVER_DRIVER = "net.sourceforge.jtds.jdbc.Driver"; - public SqlserverAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new SqlserverAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -67,6 +71,6 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); } } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java index 347ca9bf4..e8a487721 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java @@ -20,16 +20,21 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import java.util.List; +import java.util.List; +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverAsyncSideInfo extends RdbAsyncSideInfo { - public SqlserverAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java index 3631dcd8a..e06c13898 100644 --- a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java +++ b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java @@ -18,16 +18,20 @@ package com.dtstack.flink.sql.side.sqlserver.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; - +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverSideParser extends RdbSideParser { private static final String CURR_TYPE = "sqlserver"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo sqlServerTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo sqlServerTableInfo = super.getTableInfo(tableName, fieldsInfo, props); sqlServerTableInfo.setType(CURR_TYPE); return sqlServerTableInfo; } diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java index dbbeb820e..3c331cdd0 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** @@ -29,7 +29,7 @@ * * @author maqi */ -public class SqlserverSink extends RdbSink implements IStreamSinkGener { +public class SqlserverSink extends AbstractRdbSink implements IStreamSinkGener { public SqlserverSink() { super(new SqlserverDialect()); @@ -38,7 +38,7 @@ public SqlserverSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java index a695d6c3b..5300884bc 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.sqlserver.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -33,8 +33,8 @@ public class SqlserverSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "sqlserver"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo sqlserverTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo sqlserverTableInfo = super.getTableInfo(tableName, fieldsInfo, props); sqlserverTableInfo.setType(CURR_TYPE); return sqlserverTableInfo; } From c5859aae1a45bbc93d86ed56959480ee8fd4aaa3 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 11 May 2020 16:04:27 +0800 Subject: [PATCH 306/523] =?UTF-8?q?hbase=E7=BB=93=E6=9E=9C=E8=A1=A8?= =?UTF-8?q?=E6=96=87=E6=A1=A3=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/cassandra/table/CassandraSideParser.java | 1 + docs/plugin/hbaseSink.md | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java index 62dd753b9..38b1d082a 100644 --- a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java +++ b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java @@ -99,6 +99,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Mon, 11 May 2020 16:05:00 +0800 Subject: [PATCH 307/523] =?UTF-8?q?hbase=E7=BB=93=E6=9E=9C=E8=A1=A8?= =?UTF-8?q?=E6=96=87=E6=A1=A3=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/hbaseSink.md | 52 ++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/docs/plugin/hbaseSink.md b/docs/plugin/hbaseSink.md index d151fc139..00c340b3d 100644 --- a/docs/plugin/hbaseSink.md +++ b/docs/plugin/hbaseSink.md @@ -41,19 +41,51 @@ hbase2.0 ## 5.样例： ``` - CREATE TABLE MyResult( - cf:info VARCHAR, - cf:name VARCHAR, - cf:channel varchar +CREATE TABLE MyTable( + name varchar, + channel varchar, + age int )WITH( - type ='hbase', - zookeeperQuorum ='xx:2181', - zookeeperParent ='/hbase', - tableName ='workerinfo01', - rowKey ='channel' + type ='kafka10', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest01', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + +CREATE TABLE MyResult( + cf:name varchar , + cf:channel varchar + )WITH( + type ='hbase', + zookeeperQuorum ='172.16.10.104:2181,172.16.10.224:2181,172.16.10.252:2181', + zookeeperParent ='/hbase', + tableName ='myresult', + partitionedJoin ='false', + parallelism ='1', + rowKey='name,channel' ); + +insert +into + MyResult + select + channel, + name + from + MyTable a ``` ## 6.hbase数据 ### 数据内容说明 -### 数据内容示例 \ No newline at end of file +hbase的rowkey 构建规则：以描述的rowkey字段值作为key，多个字段以'-'连接 +### 数据内容示例 +hbase(main):007:0> scan 'myresult' + ROW COLUMN+CELL + roc-daishu column=cf:channel, timestamp=1589183971724, value=daishu + roc-daishu column=cf:name, timestamp=1589183971724, value=roc \ No newline at end of file From 960660be24c86a1c44edab0a92e43bbabd197b53 Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 11 May 2020 16:20:22 +0800 Subject: [PATCH 308/523] =?UTF-8?q?=E5=9B=9E=E6=BB=9A=205f30a74b=20?= =?UTF-8?q?=EF=BC=8C=E8=A7=A3=E5=86=B3=20'=E5=9B=9E=E6=BB=9A=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=9A=84test=E5=88=86=E6=94=AF=E5=8F=98=E6=9B=B4?= =?UTF-8?q?=E5=86=85=E5=AE=B9'=E6=97=A0=E6=B3=95=E6=8F=90=E4=BA=A4?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java | 1 + .../main/java/com/dtstack/flink/sql/util/ParseUtils.java | 2 ++ .../dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 6 +++--- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java index b4240dbf3..8854ff4ec 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java @@ -27,6 +27,7 @@ import com.google.common.base.Strings; import java.io.Serializable; +import java.util.Map; /** * Join信息 diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java index fa06a6cc3..363d8bca0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java @@ -38,6 +38,7 @@ import com.google.common.collect.HashBasedTable; +import com.google.common.collect.HashBiMap; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlJoin; @@ -48,6 +49,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Set; import static org.apache.calcite.sql.SqlKind.AS; diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 9af6d02fd..6005146e2 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -130,7 +130,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce } } - rdbSQLClient.getConnection(conn -> { + rdbSqlClient.getConnection(conn -> { if (conn.failed()) { //Treatment failures resultFuture.completeExceptionally(conn.cause()); @@ -197,9 +197,9 @@ private Object convertDataType(Object val) { } else if (val instanceof Instant) { } else if (val instanceof Timestamp) { - val = DateUtil.getStringFromTimestamp((Timestamp) val); + val = DateUtil.timestampToString((Timestamp) val); } else if (val instanceof java.util.Date) { - val = DateUtil.getStringFromDate((java.sql.Date) val); + val = DateUtil.dateToString((java.sql.Date) val); } else { val = val.toString(); } From ac7bc46d95b7c70e6a893a3643b85c5b017c8dec Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 11 May 2020 16:59:50 +0800 Subject: [PATCH 309/523] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E6=9C=AC=E5=9C=B0log?= =?UTF-8?q?=E6=8F=92=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/pom.xml | 5 ----- .../dtstack/flink/sql/exec/ExecuteProcessHelper.java | 10 ---------- 2 files changed, 15 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index eb470a3c0..be1020b04 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -122,11 +122,6 @@ junit 4.12 - - com.aiweiergou - tools-logger - ${logger.tool.version} - diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 2831eabd5..60e3c8cd9 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -18,7 +18,6 @@ package com.dtstack.flink.sql.exec; -import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; import com.dtstack.flink.sql.parser.CreateFuncParser; import com.dtstack.flink.sql.parser.CreateTmpTableParser; import com.dtstack.flink.sql.parser.FlinkPlanner; @@ -354,13 +353,4 @@ public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperti return env; } - - public static void setLogLevel(ParamsInfo paramsInfo){ - String logLevel = paramsInfo.getConfProp().getProperty(ConfigConstrant.LOG_LEVEL_KEY); - if(org.apache.commons.lang3.StringUtils.isBlank(logLevel)){ - return; - } - ChangeLogLevelProcess logLevelProcess = new ChangeLogLevelProcess(); - logLevelProcess.process(logLevel); - } } \ No newline at end of file From 425bdc23c5748040a176d2b168bce2eed9192a1c Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 11 May 2020 17:00:36 +0800 Subject: [PATCH 310/523] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E6=9C=AC=E5=9C=B0log?= =?UTF-8?q?=E6=8F=92=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/main/java/com/dtstack/flink/sql/Main.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index fe925f9a6..5d7528869 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -38,7 +38,6 @@ public class Main { public static void main(String[] args) throws Exception { ParamsInfo paramsInfo = ExecuteProcessHelper.parseParams(args); - ExecuteProcessHelper.setLogLevel(paramsInfo); StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExecution(paramsInfo); env.execute(paramsInfo.getName()); LOG.info("program {} execution success", paramsInfo.getName()); From 3012c91c6e542c6dd1ab11049b004f96b09e8a5d Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 11 May 2020 21:29:52 +0800 Subject: [PATCH 311/523] =?UTF-8?q?mongo=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/mongoSink.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin/mongoSink.md b/docs/plugin/mongoSink.md index 8c32dc95e..bf4bbb233 100644 --- a/docs/plugin/mongoSink.md +++ b/docs/plugin/mongoSink.md @@ -102,7 +102,7 @@ on ``` - 数据结果： + ## 数据结果示例 向Topic mqTest03 发送数据 {"name":"maqi","id":1001} 插入 (1001,"maqi",null) From 0e21db667b3acbf630bea5ff56f5932086cda31f Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 12 May 2020 11:48:03 +0800 Subject: [PATCH 312/523] fix merge comflict bug --- .../sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index a01e0f71a..502f23f0a 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -128,7 +128,7 @@ public void onResponse(SearchResponse searchResponse) { dealCacheData(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); resultFuture.complete(rowList); } catch (Exception e) { - dealFillDataError(resultFuture, e, input); + dealFillDataError(input, resultFuture, e); } finally { if (tmpRhlClient != null) { try { From 42e83e7eb13f8353376456203d383bd0e018a600 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 12 May 2020 14:15:59 +0800 Subject: [PATCH 313/523] =?UTF-8?q?=E7=BC=96=E8=AF=91=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 71652042c..fbb0aa291 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -144,7 +144,7 @@ public void accept(Map values) { dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.MultiLine, values)); resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, input); + dealFillDataError(input, resultFuture, e); } } else { dealMissKey(input, resultFuture); From 811797dc454bd5504ec2caedf358bb8d8f11c818 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 12 May 2020 15:09:49 +0800 Subject: [PATCH 314/523] =?UTF-8?q?oracle=20=E6=96=87=E6=A1=A3=E6=9B=B4?= =?UTF-8?q?=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/oracleSink.md | 70 +++++++++++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 11 deletions(-) diff --git a/docs/plugin/oracleSink.md b/docs/plugin/oracleSink.md index c15a6109a..93771f3d6 100644 --- a/docs/plugin/oracleSink.md +++ b/docs/plugin/oracleSink.md @@ -37,19 +37,67 @@ CREATE TABLE tableName( | tableName | oracle表名称|是|| | schema | oracle 的schema|否|当前登录用户| | parallelism | 并行度设置|否|1| +| batchSize | flush的大小|否|100| +| batchWaitInterval | flush的时间间隔，单位ms|否|1000| +| allReplace | true:新值替换旧值|否|false| +| updateMode | APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|结果表设置主键则为UPSERT| + ## 5.样例： ``` -CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR +CREATE TABLE MyTable( + name varchar, + channel varchar, + id int )WITH( - type ='oracle', - url ='jdbc:oracle:thin:@xx.xx.xx.xx:1521:orcl', - userName ='dtstack', - password ='abc123', - tableName ='pv2', - schema = 'MQTEST', + type ='kafka10', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest01', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', parallelism ='1' - ) - ``` \ No newline at end of file + ); + + CREATE TABLE MyResult( + primarykey_id int , + name VARCHAR, + address VARCHAR + )WITH( + type ='oracle', + url ='jdbc:oracle:thin:@172.16.8.178', + userName ='system', + password ='oracle', + tableName ='YELUO_TEST_ORACLE_01', + updateMode ='append', + parallelism ='1', + batchSize ='100', + batchWaitInterval ='1000' + ); + +insert +into + MyResult + select + id as primarykey_id, + channel as address, + name + from + MyTable a + ``` + +## 6.数据示例 +### 输入数据 +``` +{"name":"roc","id":11,"channel":"daishuyun"} +``` +### 结果数据 +``` ++---------+------+------+-----------+ +| primarykey_id | name | address | ++---------+------+------+----------+ +| 11 | roc | daishuyun | +``` \ No newline at end of file From 2051383ef4ff48ff84512bb8ca5b81147f5b4a92 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 12 May 2020 16:19:48 +0800 Subject: [PATCH 315/523] =?UTF-8?q?rdb=20=E8=BF=9E=E6=8E=A5=E9=87=8D?= =?UTF-8?q?=E8=AF=95=E9=85=8D=E7=BD=AE=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/AbstractSideTableInfo.java | 15 ++++++++++++++- .../flink/sql/table/AbstractSideTableParser.java | 8 ++++---- .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index 75f507684..994674546 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -55,7 +55,9 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements public static final String ASYNC_TIMEOUT_KEY = "asyncTimeout"; public static final String ASYNC_FAIL_MAX_NUM_KEY = "asyncFailMaxNum"; - + + public static final String CONNECT_RETRY_MAX_NUM_KEY = "connectRetryMaxNum"; + public static final String ASYNC_REQ_POOL_KEY = "asyncPoolSize"; private String cacheType = "none"; @@ -79,6 +81,8 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements private Long asyncFailMaxNum; + private Integer connectRetryMaxNum; + private List predicateInfoes = Lists.newArrayList(); public RowTypeInfo getRowTypeInfo(){ @@ -164,6 +168,7 @@ public void setAsyncFailMaxNum(Long asyncFailMaxNum) { this.asyncFailMaxNum = asyncFailMaxNum; } + public int getAsyncPoolSize() { return asyncPoolSize; } @@ -172,6 +177,14 @@ public void setAsyncPoolSize(int asyncPoolSize) { this.asyncPoolSize = asyncPoolSize; } + + public Integer getConnectRetryMaxNum(Integer defaultValue) { + return Objects.isNull(connectRetryMaxNum) ? defaultValue : connectRetryMaxNum; + } + + public void setConnectRetryMaxNum(Integer connectRetryMaxNum) { + this.connectRetryMaxNum = connectRetryMaxNum; + } @Override public String toString() { return "Cache Info{" + diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 3aff6e2e6..276575c7e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -110,10 +110,10 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0){ - sideTableInfo.setAsyncFailMaxNum(asyncFailNum); + if(props.containsKey(AbstractSideTableInfo.CONNECT_RETRY_MAX_NUM_KEY.toLowerCase())){ + Integer connectRetryMaxNum = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.CONNECT_RETRY_MAX_NUM_KEY.toLowerCase())); + if (connectRetryMaxNum > 0){ + sideTableInfo.setConnectRetryMaxNum(connectRetryMaxNum); } } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index b7667ca0e..6b639bed2 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -137,7 +137,7 @@ private void connectWithRetry(Map inputParams, CRow input, Resul if(failCounter.getAndIncrement() % 1000 == 0){ LOG.error("getConnection error", conn.cause()); } - if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3L)){ + if(failCounter.get() >= sideInfo.getSideTableInfo().getConnectRetryMaxNum(3)){ dealFillDataError(input, resultFuture, conn.cause()); finishFlag.set(true); } From 95e7eaa84f55a976568b895c192248a7256ef173 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 12 May 2020 18:00:10 +0800 Subject: [PATCH 316/523] =?UTF-8?q?=E8=A7=A3=E6=9E=90columnName=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index f57956096..7e9937ad1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -182,7 +182,7 @@ private Map parseInputParam(CRow input){ if(equalObj == null){ return inputParams; } - String columnName = sideInfo.getEqualFieldList().get(conValIndex); + String columnName = sideInfo.getEqualFieldList().get(i); inputParams.put(columnName, equalObj); } return inputParams; From c80c4d38a06965690d100b0eda63e984a558ef55 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 13 May 2020 11:30:07 +0800 Subject: [PATCH 317/523] fix impalaDialect npe --- .../com/dtstack/flink/sql/sink/impala/ImpalaDialect.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java index 7bb24481a..3eaffc0ad 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java @@ -19,10 +19,12 @@ package com.dtstack.flink.sql.sink.impala; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -61,7 +63,7 @@ public String getInsertIntoStatement(String schema, String tableName, String[] f String schemaInfo = StringUtils.isEmpty(schema) ? "" : quoteIdentifier(schema) + "."; - List partitionFieldsList = Arrays.asList(partitionFields); + List partitionFieldsList = Objects.isNull(partitionFields) ? Lists.newArrayList() : Arrays.asList(partitionFields); String columns = Arrays.stream(fieldNames) .filter(f -> !partitionFieldsList.contains(f)) @@ -72,7 +74,7 @@ public String getInsertIntoStatement(String schema, String tableName, String[] f .map(f -> "?") .collect(Collectors.joining(", ")); - String partitionFieldStr = Arrays.stream(partitionFields) + String partitionFieldStr = partitionFieldsList.stream() .map(field -> field.replaceAll("\"", "'")) .collect(Collectors.joining(", ")); From ae6784f040bad10971d028cd129b255eda3d2707 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 13 May 2020 14:46:03 +0800 Subject: [PATCH 318/523] =?UTF-8?q?fix=20=E4=BB=A3=E7=A0=81merge=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 8fc1fbb02..6e7bb3408 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -787,10 +787,6 @@ private void joinFun(Object pollObj, Map localTableCache, throw new RuntimeException("can't not find side table:" + joinInfo.getRightTableName()); } -// if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ -// throw new RuntimeException("ON condition must contain all equal fields!!!"); -// } - rightScopeChild.setRowTypeInfo(sideTableInfo.getRowTypeInfo()); joinScope.addScope(leftScopeChild); From 323c6183dc9cd3fe4904c4cedaa4baba8f470040 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 13 May 2020 16:43:52 +0800 Subject: [PATCH 319/523] =?UTF-8?q?elasticsearch=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/elasticsearch6Sink.md | 90 ++++++++++++++++++++++++++----- docs/plugin/elasticsearchSink.md | 60 +++++++++++++++++---- 2 files changed, 128 insertions(+), 22 deletions(-) diff --git a/docs/plugin/elasticsearch6Sink.md b/docs/plugin/elasticsearch6Sink.md index 0bab12cc0..0f7a04280 100644 --- a/docs/plugin/elasticsearch6Sink.md +++ b/docs/plugin/elasticsearch6Sink.md @@ -42,21 +42,85 @@ CREATE TABLE tableName( |password | 密码 | 否，authMesh='true'时为必填 || |parallelism | 并行度设置|否|1| -## 5.样例： +## 5.完整样例： ``` +CREATE TABLE MyTable( + channel varchar, + pv INT, + xctime bigint + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='es_test', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + CREATE TABLE MyResult( - aa INT, - bb INT + pv varchar, + channel varchar )WITH( type ='elasticsearch6', - address ='172.16.10.47:9500', - cluster='es_47_menghan', - esType ='type1', - index ='xc_es_test', - authMesh = 'true', - userName = 'elastic', - password = 'abc123', - id ='0,1', + address ='172.16.8.193:9200', + authMesh='true', + username='elastic', + password='abc123', + estype ='external', + cluster ='docker-cluster', + index ='myresult', + id ='1', + updateMode ='append', parallelism ='1' - ) - ``` \ No newline at end of file + ); + +CREATE TABLE sideTable( + a varchar, + b varchar, + PRIMARY KEY(a) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='elasticsearch6', + address ='172.16.8.193:9200', + estype ='external', + cluster ='docker-cluster', + index ='sidetest', + authMesh='true', + username='elastic', + password='abc123', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + partitionedJoin ='false', + parallelism ='1' + ); + +insert +into + MyResult + select + w.b as pv, + s.channel as channel + from + MyTable s + join + sideTable w + on s.pv = w.a + where + w.a = '10' + and s.channel='xc'; + ``` +## 6.结果数据 +### 输入数据 +``` +{"channel":"xc26","pv":10,"xctime":1232312} +``` +### 输出数据 +``` +http://172.16.8.193:9200/myresult/_search +{"_index":"myresult","_type":"external","_id":"8aX_DHIBn3B7OBuqFl-i","_score":1.0,"_source":{"pv":"10","channel":"xc26"}} +``` \ No newline at end of file diff --git a/docs/plugin/elasticsearchSink.md b/docs/plugin/elasticsearchSink.md index 4908f8a05..9a2a32b6f 100644 --- a/docs/plugin/elasticsearchSink.md +++ b/docs/plugin/elasticsearchSink.md @@ -40,16 +40,58 @@ CREATE TABLE tableName( ## 5.样例： ``` +CREATE TABLE MyTable( + channel varchar, + pv varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='es_test', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + CREATE TABLE MyResult( - aa INT, - bb INT + pv varchar, + channel varchar )WITH( type ='elasticsearch', - address ='172.16.10.47:9500', - cluster='es_47_menghan', - estype ='type1', - index ='xc_es_test', - id ='0,1', + address ='172.16.8.193:9200', + authMesh='true', + username='elastic', + password='abc123', + estype ='external', + cluster ='docker-cluster', + index ='myresult', + id ='1', + updateMode ='append', parallelism ='1' - ) - ``` \ No newline at end of file + ); + + +insert +into + MyResult + select + s.pv as pv, + s.channel as channel + from + MyTable s + + ``` + +## 6.结果示例 +### 输入数据示例 +``` +{"channel":"xc26","pv":"10","xctime":1232312} +``` +### 输出数据示例 +``` +http://172.16.8.193:9200/myresult/_search +{"_index":"myresult","_type":"external","_id":"8aX_DHIBn3B7OBuqFl-i","_score":1.0,"_source":{"pv":"10","channel":"xc26"}} +``` \ No newline at end of file From 3ba019dfd7453242eb9d97537c43f57c35ddd90b Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 13 May 2020 17:33:48 +0800 Subject: [PATCH 320/523] fix --- .../dtstack/flink/sql/table/AbstractSideTableParser.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 276575c7e..0f0612c09 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -110,6 +110,14 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0."); + } + sideTableInfo.setAsyncFailMaxNum(asyncFailMaxNum); + } + if(props.containsKey(AbstractSideTableInfo.CONNECT_RETRY_MAX_NUM_KEY.toLowerCase())){ Integer connectRetryMaxNum = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.CONNECT_RETRY_MAX_NUM_KEY.toLowerCase())); if (connectRetryMaxNum > 0){ From b282ce6988c912a737797edba02902c7cb293301 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 14 May 2020 11:46:06 +0800 Subject: [PATCH 321/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dmongo=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5url=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/mongoSide.md | 3 +-- .../flink/sql/side/mongo/MongoAllReqRow.java | 18 +++++++++++++++--- .../flink/sql/side/mongo/MongoAsyncReqRow.java | 16 +++++++++++++--- .../sql/sink/mongo/MongoOutputFormat.java | 13 ++++++++++++- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/docs/plugin/mongoSide.md b/docs/plugin/mongoSide.md index 62d2d8698..002743256 100644 --- a/docs/plugin/mongoSide.md +++ b/docs/plugin/mongoSide.md @@ -62,8 +62,7 @@ create table sideTable( PERIOD FOR SYSTEM_TIME )WITH( type ='mongo', - //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] - address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', + address ='172.21.32.1:27017,172.21.32.1:27017', database ='test', tableName ='sidetest', cache ='LRU', diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index c1a32cb64..1857dbede 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -177,9 +177,10 @@ private String buildKey(Map val, List equalFieldList) { return sb.toString(); } - private MongoCollection getConn(String address, String database, String tableName) { + private MongoCollection getConn(String host, String userName, String password, String database, String tableName) { + MongoCollection dbCollection; - mongoClient = new MongoClient(new MongoClientURI(address)); + mongoClient = new MongoClient(new MongoClientURI(getConnectionUrl(host, userName, password))); db = mongoClient.getDatabase(database); dbCollection = db.getCollection(tableName, Document.class); return dbCollection; @@ -193,7 +194,8 @@ private void loadData(Map>> tmpCache) throws SQ try { for (int i = 0; i < CONN_RETRY_NUM; i++) { try { - dbCollection = getConn(tableInfo.getAddress(), tableInfo.getDatabase(), tableInfo.getTableName()); + dbCollection = getConn(tableInfo.getAddress(), tableInfo.getUserName(), tableInfo.getPassword(), + tableInfo.getDatabase(), tableInfo.getTableName()); break; } catch (Exception e) { if (i == CONN_RETRY_NUM - 1) { @@ -251,4 +253,14 @@ private void loadData(Map>> tmpCache) throws SQ } } } + private String getConnectionUrl(String address, String userName, String password){ + if(address.startsWith("mongodb://") || address.startsWith("mongodb+srv://")){ + return address; + } + if (StringUtils.isNotBlank(userName) && StringUtils.isNotBlank(password)) { + return String.format("mongodb://%s:%s@%s", userName, password, address); + } + return String.format("mongodb://%s", address); + } + } diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 6bea9e024..04f2ad9af 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.mongo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -84,9 +85,8 @@ public void open(Configuration parameters) throws Exception { } public void connMongoDb() throws Exception { - String address = mongoSideTableInfo.getAddress(); - ConnectionString connectionString = new ConnectionString(address); - + ConnectionString connectionString = new ConnectionString(getConnectionUrl(mongoSideTableInfo.getAddress(), + mongoSideTableInfo.getUserName(), mongoSideTableInfo.getPassword())); MongoClientSettings settings = MongoClientSettings.builder() .applyConnectionString(connectionString) .build(); @@ -193,4 +193,14 @@ public void close() throws Exception { } } + private String getConnectionUrl(String address, String userName, String password){ + if(address.startsWith("mongodb://") || address.startsWith("mongodb+srv://")){ + return address; + } + if (StringUtils.isNotBlank(userName) && StringUtils.isNotBlank(password)) { + return String.format("mongodb://%s:%s@%s", userName, password, address); + } + return String.format("mongodb://%s", address); + } + } diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index 37f7f5952..df8293522 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -25,6 +25,7 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.result.UpdateResult; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; @@ -119,7 +120,8 @@ public void close() { } private void establishConnection() { - mongoClient = new MongoClient(new MongoClientURI(address)); + + mongoClient = new MongoClient(new MongoClientURI(getConnectionUrl())); db = mongoClient.getDatabase(database); } @@ -197,5 +199,14 @@ public MongoOutputFormat finish() { } } + private String getConnectionUrl(){ + if(address.startsWith("mongodb://") || address.startsWith("mongodb+srv://")){ + return address; + } + if (StringUtils.isNotBlank(userName) && StringUtils.isNotBlank(password)) { + return String.format("mongodb://%s:%s@%s", userName, password, address); + } + return String.format("mongodb://%s", address); + } } From abfb0839a87f874eeb796f35ad6c9564f1e9c02a Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 14 May 2020 12:56:10 +0800 Subject: [PATCH 322/523] fix mongo url bug --- docs/mongoSide.md | 5 ++--- .../flink/sql/side/mongo/MongoAllReqRow.java | 18 +++++++++++++++--- .../flink/sql/side/mongo/MongoAsyncReqRow.java | 16 +++++++++++++--- .../sql/sink/mongo/MongoOutputFormat.java | 13 ++++++++++++- 4 files changed, 42 insertions(+), 10 deletions(-) diff --git a/docs/mongoSide.md b/docs/mongoSide.md index 73ce9644f..002743256 100644 --- a/docs/mongoSide.md +++ b/docs/mongoSide.md @@ -30,7 +30,7 @@ |----|---| | tableName | 注册到flink的表名称(可选填;不填默认和hbase对应的表名称相同)| | colName | 列名称| - | colType | 列类型 [colType支持的类型](colType.md)| + | colType | 列类型 [colType支持的类型](docs/colType.md)| | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| @@ -62,8 +62,7 @@ create table sideTable( PERIOD FOR SYSTEM_TIME )WITH( type ='mongo', - //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] - address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', + address ='172.21.32.1:27017,172.21.32.1:27017', database ='test', tableName ='sidetest', cache ='LRU', diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index c1a32cb64..1857dbede 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -177,9 +177,10 @@ private String buildKey(Map val, List equalFieldList) { return sb.toString(); } - private MongoCollection getConn(String address, String database, String tableName) { + private MongoCollection getConn(String host, String userName, String password, String database, String tableName) { + MongoCollection dbCollection; - mongoClient = new MongoClient(new MongoClientURI(address)); + mongoClient = new MongoClient(new MongoClientURI(getConnectionUrl(host, userName, password))); db = mongoClient.getDatabase(database); dbCollection = db.getCollection(tableName, Document.class); return dbCollection; @@ -193,7 +194,8 @@ private void loadData(Map>> tmpCache) throws SQ try { for (int i = 0; i < CONN_RETRY_NUM; i++) { try { - dbCollection = getConn(tableInfo.getAddress(), tableInfo.getDatabase(), tableInfo.getTableName()); + dbCollection = getConn(tableInfo.getAddress(), tableInfo.getUserName(), tableInfo.getPassword(), + tableInfo.getDatabase(), tableInfo.getTableName()); break; } catch (Exception e) { if (i == CONN_RETRY_NUM - 1) { @@ -251,4 +253,14 @@ private void loadData(Map>> tmpCache) throws SQ } } } + private String getConnectionUrl(String address, String userName, String password){ + if(address.startsWith("mongodb://") || address.startsWith("mongodb+srv://")){ + return address; + } + if (StringUtils.isNotBlank(userName) && StringUtils.isNotBlank(password)) { + return String.format("mongodb://%s:%s@%s", userName, password, address); + } + return String.format("mongodb://%s", address); + } + } diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 6bea9e024..04f2ad9af 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.mongo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -84,9 +85,8 @@ public void open(Configuration parameters) throws Exception { } public void connMongoDb() throws Exception { - String address = mongoSideTableInfo.getAddress(); - ConnectionString connectionString = new ConnectionString(address); - + ConnectionString connectionString = new ConnectionString(getConnectionUrl(mongoSideTableInfo.getAddress(), + mongoSideTableInfo.getUserName(), mongoSideTableInfo.getPassword())); MongoClientSettings settings = MongoClientSettings.builder() .applyConnectionString(connectionString) .build(); @@ -193,4 +193,14 @@ public void close() throws Exception { } } + private String getConnectionUrl(String address, String userName, String password){ + if(address.startsWith("mongodb://") || address.startsWith("mongodb+srv://")){ + return address; + } + if (StringUtils.isNotBlank(userName) && StringUtils.isNotBlank(password)) { + return String.format("mongodb://%s:%s@%s", userName, password, address); + } + return String.format("mongodb://%s", address); + } + } diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index 37f7f5952..df8293522 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -25,6 +25,7 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.result.UpdateResult; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; @@ -119,7 +120,8 @@ public void close() { } private void establishConnection() { - mongoClient = new MongoClient(new MongoClientURI(address)); + + mongoClient = new MongoClient(new MongoClientURI(getConnectionUrl())); db = mongoClient.getDatabase(database); } @@ -197,5 +199,14 @@ public MongoOutputFormat finish() { } } + private String getConnectionUrl(){ + if(address.startsWith("mongodb://") || address.startsWith("mongodb+srv://")){ + return address; + } + if (StringUtils.isNotBlank(userName) && StringUtils.isNotBlank(password)) { + return String.format("mongodb://%s:%s@%s", userName, password, address); + } + return String.format("mongodb://%s", address); + } } From c52ffa9a164540f3afba772e0ec6666e318d71ec Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 14 May 2020 15:18:56 +0800 Subject: [PATCH 323/523] =?UTF-8?q?hbase=20=E8=A7=A3=E6=9E=90=E5=88=97?= =?UTF-8?q?=E6=97=8F=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/hbase/table/HbaseSinkParser.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index bd1b7cac3..001443daa 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -26,6 +26,7 @@ import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.MathUtil; +import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import java.util.LinkedHashMap; @@ -98,14 +99,23 @@ public void parseFieldsInfo(String fieldsInfo, HbaseTableInfo tableInfo){ String fieldType = filedInfoArr[filedInfoArr.length - 1 ].trim(); Class fieldClass = dbTypeConvertToJavaType(fieldType); String[] columnFamily = StringUtils.split(fieldName.trim(), ":"); - columnFamilies.put(fieldName.trim(),columnFamily[1]); tableInfo.addPhysicalMappings(filedInfoArr[0],filedInfoArr[0]); tableInfo.addField(columnFamily[1]); tableInfo.addFieldClass(fieldClass); tableInfo.addFieldType(fieldType); tableInfo.addFieldExtraInfo(null); } - tableInfo.setColumnNameFamily(columnFamilies); + tableInfo.setColumnNameFamily(parseColumnFamily(tableInfo.getPhysicalFields())); tableInfo.finish(); } + + private Map parseColumnFamily(Map physicalFieldMap){ + Map columnFamiles = Maps.newHashMap(); + physicalFieldMap.values().forEach(x -> { + String[] columnFamily = StringUtils.split(x.trim(), ":"); + columnFamiles.put(x, columnFamily[1]); + }); + + return columnFamiles; + } } From 15c86353dcf954fb48d0f11eb05703204e18ec02 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 14 May 2020 16:20:03 +0800 Subject: [PATCH 324/523] =?UTF-8?q?fix=20mongo=20=E5=BC=82=E6=AD=A5?= =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E6=B2=A1=E6=9C=89=E8=A7=A3=E6=9E=90equalFiel?= =?UTF-8?q?d=20bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/mongo/MongoAsyncReqRow.java | 2 +- .../flink/sql/side/mongo/MongoAsyncSideInfo.java | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 04f2ad9af..0cf8158c7 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -96,7 +96,7 @@ public void connMongoDb() throws Exception { @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { - CRow inputCopy = new CRow(input.row(), input.change()); + CRow inputCopy = new CRow(Row.copy(input.row()), input.change()); BasicDBObject basicDbObject = new BasicDBObject(); try { basicDbObject.putAll(inputParams); diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java index 19534d4e1..37ec8d8c6 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java @@ -22,6 +22,8 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.util.ParseUtils; +import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; @@ -46,6 +48,16 @@ public MongoAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List sqlNodeList = Lists.newArrayList(); + ParseUtils.parseAnd(conditionNode, sqlNodeList); + + for(SqlNode sqlNode : sqlNodeList){ + dealOneEqualCon(sqlNode, sideTableName); + } } From 818c3203b5fb4d77a550c8ed9a4c1516dc2999f8 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 14 May 2020 21:51:16 +0800 Subject: [PATCH 325/523] fix mongo and async block bug --- .../flink/sql/side/mongo/utils/MongoUtil.java | 19 +++++++++++-------- .../sql/side/rdb/async/RdbAsyncReqRow.java | 12 +++++++++--- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java index cbcba8f97..1b3ce7194 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.side.PredicateInfo; import com.mongodb.BasicDBObject; import org.apache.commons.lang3.StringUtils; +import org.bson.Document; import java.util.Arrays; import java.util.stream.Collectors; @@ -32,25 +33,27 @@ */ public class MongoUtil { public static BasicDBObject buildFilterObject(PredicateInfo info) { + + String value = info.getCondition().replaceAll("'", ""); switch (info.getOperatorName()) { case "=": - return new BasicDBObject("$eq", info.getCondition()); + return new BasicDBObject("$eq", value); case ">": - return new BasicDBObject("$gt", info.getCondition()); + return new BasicDBObject("$gt", value); case ">=": - return new BasicDBObject("$gte", info.getCondition()); + return new BasicDBObject("$gte", value); case "<": - return new BasicDBObject("$lt", info.getCondition()); + return new BasicDBObject("$lt", value); case "<=": - return new BasicDBObject("$lte", info.getCondition()); + return new BasicDBObject("$lte", value); case "<>": - return new BasicDBObject("$ne", info.getCondition()); + return new BasicDBObject("$ne", value); case "IN": - Object[] values = Arrays.stream(StringUtils.split(info.getCondition(), ",")).map(String::trim) + Object[] values = Arrays.stream(StringUtils.split(value, ",")).map(String::trim) .collect(Collectors.toList()).toArray(); return new BasicDBObject("$in", values); case "NOT IN": - return new BasicDBObject("$nin", StringUtils.split(info.getCondition(), ",")); + return new BasicDBObject("$nin", StringUtils.split(value, ",")); case "IS NOT NULL": return new BasicDBObject("$exists", true); case "IS NULL": diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 6b639bed2..e9d534df2 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -137,8 +137,8 @@ private void connectWithRetry(Map inputParams, CRow input, Resul if(failCounter.getAndIncrement() % 1000 == 0){ LOG.error("getConnection error", conn.cause()); } - if(failCounter.get() >= sideInfo.getSideTableInfo().getConnectRetryMaxNum(3)){ - dealFillDataError(input, resultFuture, conn.cause()); + if(failCounter.get() >= sideInfo.getSideTableInfo().getConnectRetryMaxNum(100)){ + resultFuture.completeExceptionally(conn.cause()); finishFlag.set(true); } conn.result().close(); @@ -155,12 +155,18 @@ private void connectWithRetry(Map inputParams, CRow input, Resul latch.countDown(); } }); - //主线程阻塞 try { latch.wait(); } catch (InterruptedException e) { LOG.error("", e); } + if(!finishFlag.get()){ + try { + Thread.sleep(100); + } catch (Exception e){ + LOG.error("", e); + } + } } } From 9435152893df7a8ac1a99b3282119592ee12f484 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 15 May 2020 10:10:48 +0800 Subject: [PATCH 326/523] =?UTF-8?q?kudu=E6=96=87=E6=A1=A3=E6=9B=B4?= =?UTF-8?q?=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/kafkaSource.md | 5 +++ docs/plugin/kuduSink.md | 80 ++++++++++++++++++++++++++++++++++---- 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/docs/plugin/kafkaSource.md b/docs/plugin/kafkaSource.md index 035e39ffb..1ad247398 100644 --- a/docs/plugin/kafkaSource.md +++ b/docs/plugin/kafkaSource.md @@ -78,6 +78,11 @@ kafka.dual.commit.enabled kafka.partition.assignment.strategy kafka.socket.receive.buffer.bytes kafka.fetch.min.bytes + +###kerberos认证相关参数 +kafka.security.protocal +kafka.sasl.mechanism +kafka.sasl.kerberos.service.name ``` ## 5.样例： diff --git a/docs/plugin/kuduSink.md b/docs/plugin/kuduSink.md index 25f44ede1..b39ba1d67 100644 --- a/docs/plugin/kuduSink.md +++ b/docs/plugin/kuduSink.md @@ -46,17 +46,81 @@ kudu 1.9.0+cdh6.2.0 ## 5.样例： ``` +CREATE TABLE MyTable( + channel varchar, + name varchar, + pv varchar, + a varchar, + b varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='es_test', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + CREATE TABLE MyResult( - id int, - title VARCHAR, - amount decimal, - tablename1 VARCHAR + a string, + b string, + c string, + d string )WITH( type ='kudu', - kuduMasters ='localhost1,localhost2,localhost3', - tableName ='impala::default.test', - writeMode='upsert', + kuduMasters ='cdh03.cdhsite:7051', + tableName ='myresult', + writeMode='insert', parallelism ='1' ); - ``` \ No newline at end of file +CREATE TABLE sideTable( + c string, + d string, + PRIMARY KEY(c) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='kudu', + kuduMasters ='cdh03.cdhsite:7051', + tableName ='sidetest4', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1', + primaryKey ='c', + isFaultTolerant ='false' + ); + +insert +into + MyResult + select + MyTable.a, + MyTable.b, + s.c, + s.d + from + MyTable + join + sideTable s + on MyTable.a = s.c + where + MyTable.a='2' + and s.d='2' + + ``` + +## 6.数据示例 +### 输入数据 +``` +{"channel":"daishuyun","name":"roc","pv":"10","a":"2","b":"2"} +``` +### 结果数据 +``` +{"a":"2","b":"2","c":"3","d":"4"} +``` \ No newline at end of file From bbd8cf935b12d218569bc9174c86418c2ec160d0 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 15 May 2020 11:52:56 +0800 Subject: [PATCH 327/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=85=A8=E5=B1=80?= =?UTF-8?q?=E7=9A=84=E8=BF=9E=E6=8E=A5=E7=8A=B6=E6=80=81=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index e9d534df2..7340357a9 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -82,7 +82,7 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private transient SQLClient rdbSqlClient; - private final static AtomicBoolean CONN_STATUS = new AtomicBoolean(true); + private AtomicBoolean connectionStatus = new AtomicBoolean(true); public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); @@ -105,7 +105,7 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { AtomicLong networkLogCounter = new AtomicLong(0L); - while (!CONN_STATUS.get()){//network is unhealth + while (!connectionStatus.get()){//network is unhealth if(networkLogCounter.getAndIncrement() % 1000 == 0){ LOG.info("network unhealth to block task"); } @@ -114,11 +114,11 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul Map params = formatInputParam(inputParams); rdbSqlClient.getConnection(conn -> { if(conn.failed()){ - CONN_STATUS.set(false); + connectionStatus.set(false); connectWithRetry(params, input, resultFuture, rdbSqlClient); return; } - CONN_STATUS.set(true); + connectionStatus.set(true); ScheduledFuture timerFuture = registerTimer(input, resultFuture); cancelTimerWhenComplete(resultFuture, timerFuture); handleQuery(conn.result(), params, input, resultFuture); @@ -141,10 +141,9 @@ private void connectWithRetry(Map inputParams, CRow input, Resul resultFuture.completeExceptionally(conn.cause()); finishFlag.set(true); } - conn.result().close(); return; } - CONN_STATUS.set(true); + connectionStatus.set(true); ScheduledFuture timerFuture = registerTimer(input, resultFuture); cancelTimerWhenComplete(resultFuture, timerFuture); handleQuery(conn.result(), inputParams, input, resultFuture); From 72b4ec2de72f0130687a81d84fbe4bf42880c074 Mon Sep 17 00:00:00 2001 From: xuchao Date: Fri, 15 May 2020 14:24:43 +0800 Subject: [PATCH 328/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=B9=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E4=BD=BF=E7=94=A8=E7=BB=B4=E8=A1=A8=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E7=9A=84=E6=95=B0=E6=8D=AE=E9=A2=84=E6=9C=9F=E4=B8=8D=E4=B8=80?= =?UTF-8?q?=E8=87=B4=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../environment/MyLocalStreamEnvironment.java | 33 ++++++++++++++----- .../flink/sql/exec/ExecuteProcessHelper.java | 12 +++++-- .../com/dtstack/flink/sql/side/JoinInfo.java | 17 ++++++++-- .../flink/sql/side/JoinNodeDealer.java | 33 ++++++++++++------- .../dtstack/flink/sql/side/SideSQLParser.java | 25 +++++++------- .../dtstack/flink/sql/side/SideSqlExec.java | 9 ++--- .../dtstack/flink/sql/util/TableUtils.java | 14 +++++--- 7 files changed, 98 insertions(+), 45 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java index 8ac1edd41..6aec8475e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestOptions; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.minicluster.MiniCluster; @@ -93,34 +94,48 @@ public JobExecutionResult execute(String jobName) throws Exception { // transform the streaming program into a JobGraph StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); + return execute(streamGraph); + } + + public JobExecutionResult execute(StreamGraph streamGraph) throws Exception { JobGraph jobGraph = streamGraph.getJobGraph(); jobGraph.setClasspaths(classpaths); + jobGraph.setAllowQueuedScheduling(true); Configuration configuration = new Configuration(); configuration.addAll(jobGraph.getJobConfiguration()); - - configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "512M"); - configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); + configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0"); // add (and override) the settings with what the user defined configuration.addAll(this.conf); - MiniClusterConfiguration.Builder configBuilder = new MiniClusterConfiguration.Builder(); - configBuilder.setConfiguration(configuration); + if (!configuration.contains(RestOptions.BIND_PORT)) { + configuration.setString(RestOptions.BIND_PORT, "0"); + } + + int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); + + MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder() + .setConfiguration(configuration) + .setNumSlotsPerTaskManager(numSlotsPerTaskManager) + .build(); if (LOG.isInfoEnabled()) { LOG.info("Running job on local embedded Flink mini cluster"); } - MiniCluster exec = new MiniCluster(configBuilder.build()); + MiniCluster miniCluster = new MiniCluster(cfg); + try { - exec.start(); - return exec.executeJobBlocking(jobGraph); + miniCluster.start(); + configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort()); + + return miniCluster.executeJobBlocking(jobGraph); } finally { transformations.clear(); - exec.closeAsync(); + miniCluster.close(); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 3c9f37811..01b3990de 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -194,14 +194,18 @@ private static void sqlTranslation(String localSqlPluginPath, SideSqlExec sideSqlExec = new SideSqlExec(); sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); + + int scope = 0; for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { - sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, result); + sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, result, scope + ""); + scope++; } for (InsertSqlParser.SqlParseResult result : sqlTree.getExecSqlList()) { if (LOG.isInfoEnabled()) { LOG.info("exe-sql:\n" + result.getExecSql()); } + boolean isSide = false; for (String tableName : result.getTargetTableList()) { if (sqlTree.getTmpTableMap().containsKey(tableName)) { @@ -212,7 +216,7 @@ private static void sqlTranslation(String localSqlPluginPath, SqlNode sqlNode = flinkPlanner.parse(realSql); String tmpSql = ((SqlInsert) sqlNode).getSource().toString(); tmp.setExecSql(tmpSql); - sideSqlExec.exec(tmp.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, tmp); + sideSqlExec.exec(tmp.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, tmp, scope + ""); } else { for (String sourceTable : result.getSourceTableList()) { if (sideTableMap.containsKey(sourceTable)) { @@ -222,7 +226,7 @@ private static void sqlTranslation(String localSqlPluginPath, } if (isSide) { //sql-dimensional table contains the dimension table of execution - sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null); + sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null, null); } else { System.out.println("----------exec sql without dimension join-----------"); System.out.println("----------real sql exec is--------------------------"); @@ -233,6 +237,8 @@ private static void sqlTranslation(String localSqlPluginPath, } } } + + scope++; } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java index 8854ff4ec..997ed417e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.side; +import com.dtstack.flink.sql.util.TableUtils; import com.google.common.collect.HashBasedTable; import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; @@ -66,6 +67,8 @@ public class JoinInfo implements Serializable { private JoinType joinType; + private String scope = ""; + /** * 左表需要查询的字段信息和output的时候对应的列名称 */ @@ -96,12 +99,14 @@ public String getNewTableName(){ //兼容左边表是as 的情况 String leftStr = leftTableName; leftStr = Strings.isNullOrEmpty(leftStr) ? leftTableAlias : leftStr; - return leftStr + "_" + rightTableName; + String newName = leftStr + "_" + rightTableName; + return TableUtils.buildTableNameWithScope(newName, scope); } public String getNewTableAlias(){ - return leftTableAlias + "_" + rightTableAlias; + String newName = leftTableAlias + "_" + rightTableAlias; + return TableUtils.buildTableNameWithScope(newName, scope); } public boolean isLeftIsSideTable() { @@ -233,6 +238,14 @@ public HashBasedTable getTableFieldRef(){ return mappingTable; } + public String getScope() { + return scope; + } + + public void setScope(String scope) { + this.scope = scope; + } + @Override public String toString() { return "JoinInfo{" + diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index 544e8ebb0..5a3111ffa 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -89,7 +89,8 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, SqlNodeList parentGroupByList, Set> joinFieldSet, Map tableRef, - Map fieldRef) { + Map fieldRef, + String scope) { SqlNode leftNode = joinNode.getLeft(); SqlNode rightNode = joinNode.getRight(); @@ -105,13 +106,14 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, if (leftNode.getKind() == JOIN) { //处理连续join - dealNestJoin(joinNode, sideTableSet, - queueInfo, parentWhere, parentSelectList, parentGroupByList, joinFieldSet, tableRef, fieldRef); + dealNestJoin(joinNode, sideTableSet, queueInfo, parentWhere, parentSelectList, + parentGroupByList, joinFieldSet, tableRef, fieldRef, scope); leftNode = joinNode.getLeft(); } if (leftNode.getKind() == AS) { - AliasInfo aliasInfo = (AliasInfo) sideSQLParser.parseSql(leftNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + AliasInfo aliasInfo = (AliasInfo) sideSQLParser.parseSql(leftNode, sideTableSet, queueInfo, + parentWhere, parentSelectList, parentGroupByList, scope); leftTbName = aliasInfo.getName(); leftTbAlias = aliasInfo.getAlias(); } else if(leftNode.getKind() == IDENTIFIER){ @@ -122,7 +124,8 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, boolean leftIsSide = checkIsSideTable(leftTbName, sideTableSet); Preconditions.checkState(!leftIsSide, "side-table must be at the right of join operator"); - Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, + parentWhere, parentSelectList, parentGroupByList, scope); rightTableName = rightTableNameAndAlias.f0; rightTableAlias = rightTableNameAndAlias.f1; @@ -145,6 +148,8 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, tableInfo.setRightNode(rightNode); tableInfo.setJoinType(joinType); tableInfo.setCondition(joinNode.getCondition()); + tableInfo.setScope(scope); + TableUtils.replaceJoinFieldRefTableName(joinNode.getCondition(), fieldRef); //extract 需要查询的字段信息 @@ -255,16 +260,21 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, SqlNodeList parentGroupByList, Set> joinFieldSet, Map tableRef, - Map fieldRef){ + Map fieldRef, + String scope){ SqlJoin leftJoinNode = (SqlJoin) joinNode.getLeft(); SqlNode parentRightJoinNode = joinNode.getRight(); SqlNode rightNode = leftJoinNode.getRight(); - Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); - Tuple2 parentRightJoinInfo = parseRightNode(parentRightJoinNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + + Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, + parentWhere, parentSelectList, parentGroupByList, scope); + Tuple2 parentRightJoinInfo = parseRightNode(parentRightJoinNode, sideTableSet, + queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); boolean parentRightIsSide = checkIsSideTable(parentRightJoinInfo.f0, sideTableSet); - JoinInfo joinInfo = dealJoinNode(leftJoinNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, joinFieldSet, tableRef, fieldRef); + JoinInfo joinInfo = dealJoinNode(leftJoinNode, sideTableSet, queueInfo, parentWhere, parentSelectList, + parentGroupByList, joinFieldSet, tableRef, fieldRef, scope); String rightTableName = rightTableNameAndAlias.f0; boolean rightIsSide = checkIsSideTable(rightTableName, sideTableSet); @@ -616,12 +626,13 @@ private void extractSelectField(SqlNode selectNode, private Tuple2 parseRightNode(SqlNode sqlNode, Set sideTableSet, Queue queueInfo, - SqlNode parentWhere, SqlNodeList selectList, SqlNodeList parentGroupByList) { + SqlNode parentWhere, SqlNodeList selectList, SqlNodeList parentGroupByList, + String scope) { Tuple2 tabName = new Tuple2<>("", ""); if(sqlNode.getKind() == IDENTIFIER){ tabName.f0 = sqlNode.toString(); }else{ - AliasInfo aliasInfo = (AliasInfo)sideSQLParser.parseSql(sqlNode, sideTableSet, queueInfo, parentWhere, selectList, parentGroupByList); + AliasInfo aliasInfo = (AliasInfo)sideSQLParser.parseSql(sqlNode, sideTableSet, queueInfo, parentWhere, selectList, parentGroupByList, scope); tabName.f0 = aliasInfo.getName(); tabName.f1 = aliasInfo.getAlias(); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index f37c3f78c..8187e2c02 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -61,7 +61,7 @@ public class SideSQLParser { private Map localTableCache = Maps.newHashMap(); - public Queue getExeQueue(String exeSql, Set sideTableSet) throws SqlParseException { + public Queue getExeQueue(String exeSql, Set sideTableSet, String scope) throws SqlParseException { System.out.println("----------exec original Sql----------"); System.out.println(exeSql); LOG.info("----------exec original Sql----------"); @@ -71,7 +71,7 @@ public Queue getExeQueue(String exeSql, Set sideTableSet) throws FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); SqlNode sqlNode = flinkPlanner.parse(exeSql); - parseSql(sqlNode, sideTableSet, queueInfo, null, null, null); + parseSql(sqlNode, sideTableSet, queueInfo, null, null, null, scope); queueInfo.offer(sqlNode); return queueInfo; } @@ -91,7 +91,8 @@ public Object parseSql(SqlNode sqlNode, Queue queueInfo, SqlNode parentWhere, SqlNodeList parentSelectList, - SqlNodeList parentGroupByList){ + SqlNodeList parentGroupByList, + String scope){ SqlKind sqlKind = sqlNode.getKind(); switch (sqlKind){ case WITH: { @@ -99,15 +100,15 @@ public Object parseSql(SqlNode sqlNode, SqlNodeList sqlNodeList = sqlWith.withList; for (SqlNode withAsTable : sqlNodeList) { SqlWithItem sqlWithItem = (SqlWithItem) withAsTable; - parseSql(sqlWithItem.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + parseSql(sqlWithItem.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); queueInfo.add(sqlWithItem); } - parseSql(sqlWith.body, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + parseSql(sqlWith.body, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); break; } case INSERT: SqlNode sqlSource = ((SqlInsert)sqlNode).getSource(); - return parseSql(sqlSource, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + return parseSql(sqlSource, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); case SELECT: SqlNode sqlFrom = ((SqlSelect)sqlNode).getFrom(); SqlNode sqlWhere = ((SqlSelect)sqlNode).getWhere(); @@ -115,7 +116,7 @@ public Object parseSql(SqlNode sqlNode, SqlNodeList groupByList = ((SqlSelect) sqlNode).getGroup(); if(sqlFrom.getKind() != IDENTIFIER){ - Object result = parseSql(sqlFrom, sideTableSet, queueInfo, sqlWhere, selectList, groupByList); + Object result = parseSql(sqlFrom, sideTableSet, queueInfo, sqlWhere, selectList, groupByList, scope); if(result instanceof JoinInfo){ return TableUtils.dealSelectResultWithJoinInfo((JoinInfo) result, (SqlSelect) sqlNode, queueInfo); }else if(result instanceof AliasInfo){ @@ -137,7 +138,7 @@ public Object parseSql(SqlNode sqlNode, Map tableRef = Maps.newHashMap(); Map fieldRef = Maps.newHashMap(); return joinNodeDealer.dealJoinNode((SqlJoin) sqlNode, sideTableSet, queueInfo, - parentWhere, parentSelectList, parentGroupByList, joinFieldSet, tableRef, fieldRef); + parentWhere, parentSelectList, parentGroupByList, joinFieldSet, tableRef, fieldRef, scope); case AS: SqlNode info = ((SqlBasicCall)sqlNode).getOperands()[0]; SqlNode alias = ((SqlBasicCall) sqlNode).getOperands()[1]; @@ -146,7 +147,7 @@ public Object parseSql(SqlNode sqlNode, if(info.getKind() == IDENTIFIER){ infoStr = info.toString(); } else { - infoStr = parseSql(info, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList).toString(); + infoStr = parseSql(info, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope).toString(); } AliasInfo aliasInfo = new AliasInfo(); @@ -159,12 +160,12 @@ public Object parseSql(SqlNode sqlNode, SqlNode unionLeft = ((SqlBasicCall)sqlNode).getOperands()[0]; SqlNode unionRight = ((SqlBasicCall)sqlNode).getOperands()[1]; - parseSql(unionLeft, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); - parseSql(unionRight, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + parseSql(unionLeft, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); + parseSql(unionRight, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); break; case ORDER_BY: SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode; - parseSql(sqlOrderBy.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + parseSql(sqlOrderBy.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); case LITERAL: return LITERAL.toString(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index f90138b2a..9de8af04f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -101,7 +101,8 @@ public void exec(String sql, StreamTableEnvironment tableEnv, Map tableCache, StreamQueryConfig queryConfig, - CreateTmpTableParser.SqlParserResult createView) throws Exception { + CreateTmpTableParser.SqlParserResult createView, + String scope) throws Exception { if(localSqlPluginPath == null){ throw new RuntimeException("need to set localSqlPluginPath"); } @@ -121,7 +122,7 @@ public void exec(String sql, SideSQLParser sideSQLParser = new SideSQLParser(); sideSQLParser.setLocalTableCache(localTableCache); - Queue exeQueue = sideSQLParser.getExeQueue(sql, sideTableMap.keySet()); + Queue exeQueue = sideSQLParser.getExeQueue(sql, sideTableMap.keySet(), scope); Object pollObj = null; while((pollObj = exeQueue.poll()) != null){ @@ -452,9 +453,9 @@ private void joinFun(Object pollObj, replaceInfo.setTargetTableName(targetTableName); replaceInfo.setTargetTableAlias(targetTableAlias); - if (!tableEnv.isRegistered(joinInfo.getNewTableName())){ + if (!tableEnv.isRegistered(targetTableName)){ Table joinTable = tableEnv.fromDataStream(dsOut); - tableEnv.registerTable(joinInfo.getNewTableName(), joinTable); + tableEnv.registerTable(targetTableName, joinTable); localTableCache.put(joinInfo.getNewTableName(), joinTable); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 4a4973901..3075eb1df 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -198,11 +198,9 @@ public static SqlBasicCall buildAsNodeByJoinInfo(JoinInfo joinInfo, SqlNode sqlN SqlOperator operator = new SqlAsOperator(); SqlParserPos sqlParserPos = new SqlParserPos(0, 0); - String joinLeftTableName = joinInfo.getLeftTableName(); - String joinLeftTableAlias = joinInfo.getLeftTableAlias(); - joinLeftTableName = Strings.isNullOrEmpty(joinLeftTableName) ? joinLeftTableAlias : joinLeftTableName; - String newTableName = buildInternalTableName(joinLeftTableName, SPLIT, joinInfo.getRightTableName()); + String newTableName = joinInfo.getNewTableName(); String lefTbAlias = joinInfo.getLeftTableAlias(); + if(Strings.isNullOrEmpty(lefTbAlias)){ Set fromTableSet = Sets.newHashSet(); TableUtils.getFromTableInfo(joinInfo.getLeftNode(), fromTableSet); @@ -661,4 +659,12 @@ public static String buildTableField(String tableName, String fieldName){ } + public static String buildTableNameWithScope(String tableName, String scope){ + if(StringUtils.isEmpty(scope)){ + return tableName; + } + + return tableName + "_" + scope; + } + } From 7abe4dfd07757d026052915834329efbf6b53e6a Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 15 May 2020 16:22:16 +0800 Subject: [PATCH 329/523] fix kudu and async quest --- .../sql/side/kudu/KuduAsyncSideInfo.java | 19 +++++++++++++++++++ .../sql/side/rdb/async/RdbAsyncReqRow.java | 4 ++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java index 5ace515f7..91b380594 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java @@ -4,6 +4,9 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; +import com.dtstack.flink.sql.util.ParseUtils; +import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; @@ -21,6 +24,22 @@ public KuduAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List sqlNodeList = Lists.newArrayList(); + ParseUtils.parseAnd(conditionNode, sqlNodeList); + + for (SqlNode sqlNode : sqlNodeList) { + dealOneEqualCon(sqlNode, sideTableName); + } + + sqlCondition = "select ${selectField} from ${tableName} "; + sqlCondition = sqlCondition.replace("${tableName}", kuduSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields); + System.out.println("---------side_exe_sql-----\n" + sqlCondition); } @Override diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 7340357a9..7a6b45be6 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -155,13 +155,13 @@ private void connectWithRetry(Map inputParams, CRow input, Resul } }); try { - latch.wait(); + latch.await(); } catch (InterruptedException e) { LOG.error("", e); } if(!finishFlag.get()){ try { - Thread.sleep(100); + Thread.sleep(3000); } catch (Exception e){ LOG.error("", e); } From 3ca492b90a94a91749deaef71a91748fbd84bfea Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 15 May 2020 18:20:23 +0800 Subject: [PATCH 330/523] =?UTF-8?q?fix=20row=20=E7=9A=84=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E9=94=99=E4=B9=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java | 3 ++- .../com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 7e9937ad1..11240b973 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -160,7 +160,8 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ } @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + public void asyncInvoke(CRow row, ResultFuture resultFuture) throws Exception { + CRow input = new CRow(Row.copy(row.row()), row.change()); preInvoke(input, resultFuture); Map inputParams = parseInputParam(input); if(MapUtils.isEmpty(inputParams)){ diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 71652042c..fbb0aa291 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -144,7 +144,7 @@ public void accept(Map values) { dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.MultiLine, values)); resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, input); + dealFillDataError(input, resultFuture, e); } } else { dealMissKey(input, resultFuture); From 46598c486160b88ecee2584db8be3ef9a6bba459 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 18 May 2020 10:43:31 +0800 Subject: [PATCH 331/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=BC=82=E6=AD=A5?= =?UTF-8?q?=E9=98=BB=E5=A1=9E=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clickhouse/ClickhouseAsyncReqRow.java | 6 ++++ .../flink/sql/side/db2/Db2AsyncReqRow.java | 6 ++++ .../sql/side/impala/ImpalaAsyncReqRow.java | 6 ++++ .../sql/side/mysql/MysqlAsyncReqRow.java | 6 ++++ .../sql/side/oracle/OracleAsyncReqRow.java | 6 ++++ .../sql/side/polardb/PolardbAsyncReqRow.java | 6 ++++ .../postgresql/PostgresqlAsyncReqRow.java | 6 ++++ .../sql/side/rdb/async/RdbAsyncReqRow.java | 28 +++++++++---------- .../side/sqlserver/SqlserverAsyncReqRow.java | 6 ++++ 9 files changed, 62 insertions(+), 14 deletions(-) diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index 9da7d8eaa..d279973dc 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.clickhouse; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; @@ -32,6 +33,9 @@ import org.apache.flink.configuration.Configuration; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; public class ClickhouseAsyncReqRow extends RdbAsyncReqRow { @@ -63,6 +67,8 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("clickhouseAsyncExec"))); } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index 8e7275977..3a348df29 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.db2; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; @@ -33,6 +34,9 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * Reason: @@ -75,6 +79,8 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, db2lientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("dbAsyncExec"))); } } diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index b5e0ed5fb..3f2172f6d 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.impala; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; @@ -35,6 +36,9 @@ import java.io.IOException; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * Date: 2019/11/12 @@ -79,6 +83,8 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("impalaAsyncExec"))); } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index d52f1918f..4ac7e8ba5 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.mysql; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; @@ -32,6 +33,9 @@ import org.apache.flink.configuration.Configuration; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * Mysql dim table @@ -72,6 +76,8 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("mysqlAsyncExec"))); } } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index 938fd870c..53d6ff896 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.oracle; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; @@ -32,6 +33,9 @@ import org.apache.flink.configuration.Configuration; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; public class OracleAsyncReqRow extends RdbAsyncReqRow { @@ -64,5 +68,7 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("oracleAsyncExec"))); } } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 13d25b111..63b80ef0d 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -17,6 +17,7 @@ */ package com.dtstack.flink.sql.side.polardb; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; @@ -32,6 +33,9 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * Date: 2019/12/20 @@ -71,5 +75,7 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("polardbAsyncExec"))); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index 42aa39a51..cedcb9eca 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.postgresql; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; @@ -34,6 +35,9 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * Date: 2019-08-11 @@ -72,6 +76,8 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, pgClientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("postgresqlAsyncExec"))); } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 7a6b45be6..e92ab282b 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.side.rdb.async; import com.dtstack.flink.sql.enums.ECacheContentType; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.CacheMissVal; @@ -46,8 +47,7 @@ import java.time.Instant; import java.util.List; import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -84,6 +84,8 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private AtomicBoolean connectionStatus = new AtomicBoolean(true); + private transient ThreadPoolExecutor executor; + public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); init(sideInfo); @@ -112,18 +114,7 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul Thread.sleep(100); } Map params = formatInputParam(inputParams); - rdbSqlClient.getConnection(conn -> { - if(conn.failed()){ - connectionStatus.set(false); - connectWithRetry(params, input, resultFuture, rdbSqlClient); - return; - } - connectionStatus.set(true); - ScheduledFuture timerFuture = registerTimer(input, resultFuture); - cancelTimerWhenComplete(resultFuture, timerFuture); - handleQuery(conn.result(), params, input, resultFuture); - }); - + executor.execute(() -> connectWithRetry(params, input, resultFuture, rdbSqlClient)); } private void connectWithRetry(Map inputParams, CRow input, ResultFuture resultFuture, SQLClient rdbSqlClient) { @@ -134,6 +125,7 @@ private void connectWithRetry(Map inputParams, CRow input, Resul rdbSqlClient.getConnection(conn -> { try { if(conn.failed()){ + connectionStatus.set(false); if(failCounter.getAndIncrement() % 1000 == 0){ LOG.error("getConnection error", conn.cause()); } @@ -210,12 +202,20 @@ public void close() throws Exception { rdbSqlClient.close(); } + if(executor != null){ + executor.shutdown(); + } + } public void setRdbSqlClient(SQLClient rdbSqlClient) { this.rdbSqlClient = rdbSqlClient; } + public void setExecutor(ThreadPoolExecutor executor) { + this.executor = executor; + } + private void handleQuery(SQLConnection connection, Map inputParams, CRow input, ResultFuture resultFuture){ String key = buildCacheKey(inputParams); JsonArray params = new JsonArray(Lists.newArrayList(inputParams.values())); diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index 1f4fbf13a..e8a22dfe8 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.sqlserver; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; @@ -34,6 +35,9 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * Date: 2019/11/26 @@ -74,5 +78,7 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("sqlServerAsyncExec"))); } } From 34cc5825fcd4b950cbcde4fa04376868365318df Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 19 May 2020 17:28:20 +0800 Subject: [PATCH 332/523] =?UTF-8?q?fix=20=E9=87=8D=E6=9E=84=E4=B9=8B?= =?UTF-8?q?=E5=90=8E=E7=9A=84=E7=BC=93=E5=AD=98=E6=A0=BC=E5=BC=8F=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 0d8691d1c..094f5a6ea 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -203,7 +203,7 @@ private void invokeWithCache(Map inputParams, CRow input, Result return; }else if(ECacheContentType.SingleLine == val.getType()){ try { - Row row = fillData(input.row(), val); + Row row = fillData(input.row(), val.getContent()); resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { dealFillDataError(input, resultFuture, e); From 312988833605330ac00c8461acd0639d86558239 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 21 May 2020 14:44:44 +0800 Subject: [PATCH 333/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E9=93=BE=E6=8E=A5=E5=92=8Credis=E6=96=87=E6=A1=A3=E8=AF=B4?= =?UTF-8?q?=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/redisSink.md | 3 ++- docs/pluginsInfo.md | 52 ++++++++++++++++++++-------------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/docs/plugin/redisSink.md b/docs/plugin/redisSink.md index 224c26b2f..103cb4997 100644 --- a/docs/plugin/redisSink.md +++ b/docs/plugin/redisSink.md @@ -25,7 +25,6 @@ redis5.0 | tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称 | colName | 列名称，redis中存储为 表名:主键名:主键值:列名]| | colType | 列类型，当前只支持varchar| -| PRIMARY KEY(keyInfo) | 结果表主键定义;多个列之间用逗号隔开| ## 4.参数： @@ -44,6 +43,7 @@ redis5.0 |maxIdle|最大空闲连接数|否|8| |minIdle|最小空闲连接数|否||0| |masterName| 哨兵模式下的masterName|否|| +|primarykeys|主键字段，多个字段以逗号分割|是|| ## 5.样例： @@ -69,6 +69,7 @@ redis5.0 pv VARCHAR )WITH( type ='redis', + primarykeys='name', redisType ='1', url ='172.16.8.109:6379', tableName ='resultTable', diff --git a/docs/pluginsInfo.md b/docs/pluginsInfo.md index 6baec3faf..60bfc7fc1 100644 --- a/docs/pluginsInfo.md +++ b/docs/pluginsInfo.md @@ -1,32 +1,32 @@ ### 1 插件列表 #### 1.1 源表插件 -* [kafka 源表插件](docs/plugin/kafkaSource.md) +* [kafka 源表插件](plugin/kafkaSource.md) #### 1.2 结果表插件 -* [elasticsearch 结果表插件](docs/plugin/elasticsearchSink.md) -* [hbase 结果表插件](docs/plugin/hbaseSink.md) -* [mysql 结果表插件](docs/plugin/mysqlSink.md) -* [oracle 结果表插件](docs/plugin/oracleSink.md) -* [mongo 结果表插件](docs/plugin/mongoSink.md) -* [redis 结果表插件](docs/plugin/redisSink.md) -* [cassandra 结果表插件](docs/plugin/cassandraSink.md) -* [kudu 结果表插件](docs/plugin/kuduSink.md) -* [postgresql 结果表插件](docs/plugin/postgresqlSink.md) -* [clickhouse 结果表插件](docs/plugin/clickhouseSink.md) -* [impala 结果表插件](docs/plugin/impalaSink.md) -* [db2 结果表插件](docs/plugin/db2Sink.md) -* [sqlserver 结果表插件](docs/plugin/sqlserverSink.md) +* [elasticsearch 结果表插件](plugin/elasticsearchSink.md) +* [hbase 结果表插件](plugin/hbaseSink.md) +* [mysql 结果表插件](plugin/mysqlSink.md) +* [oracle 结果表插件](plugin/oracleSink.md) +* [mongo 结果表插件](plugin/mongoSink.md) +* [redis 结果表插件](plugin/redisSink.md) +* [cassandra 结果表插件](plugin/cassandraSink.md) +* [kudu 结果表插件](plugin/kuduSink.md) +* [postgresql 结果表插件](plugin/postgresqlSink.md) +* [clickhouse 结果表插件](plugin/clickhouseSink.md) +* [impala 结果表插件](plugin/impalaSink.md) +* [db2 结果表插件](plugin/db2Sink.md) +* [sqlserver 结果表插件](plugin/sqlserverSink.md) #### 1.3 维表插件 -* [hbase 维表插件](docs/plugin/hbaseSide.md) -* [mysql 维表插件](docs/plugin/mysqlSide.md) -* [oracle 维表插件](docs/plugin/oracleSide.md) -* [mongo 维表插件](docs/plugin/mongoSide.md) -* [redis 维表插件](docs/plugin/redisSide.md) -* [cassandra 维表插件](docs/plugin/cassandraSide.md) -* [kudu 维表插件](docs/plugin/kuduSide.md) -* [postgresql 维表插件](docs/plugin/postgresqlSide.md) -* [clickhouse 维表插件](docs/plugin/clickhouseSide.md) -* [impala 维表插件](docs/plugin/impalaSide.md) -* [db2 维表插件](docs/plugin/db2Side.md) -* [sqlserver 维表插件](docs/plugin/sqlserverSide.md) +* [hbase 维表插件](plugin/hbaseSide.md) +* [mysql 维表插件](plugin/mysqlSide.md) +* [oracle 维表插件](plugin/oracleSide.md) +* [mongo 维表插件](plugin/mongoSide.md) +* [redis 维表插件](plugin/redisSide.md) +* [cassandra 维表插件](plugin/cassandraSide.md) +* [kudu 维表插件](plugin/kuduSide.md) +* [postgresql 维表插件](plugin/postgresqlSide.md) +* [clickhouse 维表插件](plugin/clickhouseSide.md) +* [impala 维表插件](plugin/impalaSide.md) +* [db2 维表插件](plugin/db2Side.md) +* [sqlserver 维表插件](plugin/sqlserverSide.md) From 85285e2a1cec9fc857c929b9b12b549e62101833 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 21 May 2020 15:03:38 +0800 Subject: [PATCH 334/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E5=86=85=E9=83=A8=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/cassandraSink.md | 2 +- docs/plugin/clickhouseSide.md | 2 +- docs/plugin/consoleSink.md | 2 +- docs/plugin/db2Side.md | 2 +- docs/plugin/db2Sink.md | 2 +- docs/plugin/elasticsearch6Side.md | 2 +- docs/plugin/elasticsearch6Sink.md | 2 +- docs/plugin/elasticsearchSink.md | 2 +- docs/plugin/hbaseSide.md | 6 +++--- docs/plugin/hbaseSink.md | 2 +- docs/plugin/impalaSide.md | 2 +- docs/plugin/impalaSink.md | 2 +- docs/plugin/kafkaSink.md | 2 +- docs/plugin/kafkaSource.md | 4 ++-- docs/plugin/kuduSide.md | 4 ++-- docs/plugin/kuduSink.md | 2 +- docs/plugin/mongoSide.md | 2 +- docs/plugin/mongoSink.md | 2 +- docs/plugin/mysqlSide.md | 2 +- docs/plugin/mysqlSink.md | 2 +- docs/plugin/oracleSide.md | 2 +- docs/plugin/oracleSink.md | 2 +- docs/plugin/polardbSide.md | 2 +- docs/plugin/polardbSink.md | 2 +- docs/plugin/postgresqlSide.md | 2 +- docs/plugin/postgresqlSink.md | 2 +- docs/plugin/redisSide.md | 2 +- docs/plugin/sqlserverSide.md | 2 +- docs/plugin/sqlserverSink.md | 2 +- 29 files changed, 33 insertions(+), 33 deletions(-) diff --git a/docs/plugin/cassandraSink.md b/docs/plugin/cassandraSink.md index 8702acb7a..a4ffaf818 100644 --- a/docs/plugin/cassandraSink.md +++ b/docs/plugin/cassandraSink.md @@ -25,7 +25,7 @@ CREATE TABLE tableName( |----|---| | tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数 diff --git a/docs/plugin/clickhouseSide.md b/docs/plugin/clickhouseSide.md index df050ca8b..c41226973 100644 --- a/docs/plugin/clickhouseSide.md +++ b/docs/plugin/clickhouseSide.md @@ -29,7 +29,7 @@ ## 3.表结构定义 - [维表参数信息](docs/plugin/sideParams.md) + [维表参数信息](sideParams.md) clinkhose独有的参数信息 diff --git a/docs/plugin/consoleSink.md b/docs/plugin/consoleSink.md index d756f5a36..5a923236c 100644 --- a/docs/plugin/consoleSink.md +++ b/docs/plugin/consoleSink.md @@ -20,7 +20,7 @@ CREATE TABLE tableName( |----|---| | tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： diff --git a/docs/plugin/db2Side.md b/docs/plugin/db2Side.md index af0c29c02..faf1b3060 100644 --- a/docs/plugin/db2Side.md +++ b/docs/plugin/db2Side.md @@ -28,7 +28,7 @@ ## 3.表结构定义 - [维表参数信息](docs/plugin/sideParams.md) + [维表参数信息](sideParams.md) db2独有的参数配置： |参数名称|含义| diff --git a/docs/plugin/db2Sink.md b/docs/plugin/db2Sink.md index 9544e44f1..9994bc4a5 100644 --- a/docs/plugin/db2Sink.md +++ b/docs/plugin/db2Sink.md @@ -25,7 +25,7 @@ CREATE TABLE tableName( |----|---| | tableName| db2表名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| | primary key | updateMode为UPSERT时，需要指定的主键信息，不需要和数据库一致| diff --git a/docs/plugin/elasticsearch6Side.md b/docs/plugin/elasticsearch6Side.md index 62da6ca62..826d850b0 100644 --- a/docs/plugin/elasticsearch6Side.md +++ b/docs/plugin/elasticsearch6Side.md @@ -32,7 +32,7 @@ |----|---| | tableName | elasticsearch表名称| | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| + | colType | 列类型 [colType支持的类型](../colType.md)| | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| diff --git a/docs/plugin/elasticsearch6Sink.md b/docs/plugin/elasticsearch6Sink.md index 0f7a04280..4ecfd99f6 100644 --- a/docs/plugin/elasticsearch6Sink.md +++ b/docs/plugin/elasticsearch6Sink.md @@ -25,7 +25,7 @@ CREATE TABLE tableName( |----|---| |tableName|在 sql 中使用的名称;即注册到flink-table-env上的名称| |colName|列名称| -|colType|列类型 [colType支持的类型](docs/colType.md)| +|colType|列类型 [colType支持的类型](../colType.md)| ## 4.参数： |参数名称|含义|是否必填|默认值| diff --git a/docs/plugin/elasticsearchSink.md b/docs/plugin/elasticsearchSink.md index 9a2a32b6f..99ac87ca5 100644 --- a/docs/plugin/elasticsearchSink.md +++ b/docs/plugin/elasticsearchSink.md @@ -22,7 +22,7 @@ CREATE TABLE tableName( |----|---| |tableName|在 sql 中使用的名称;即注册到flink-table-env上的名称| |colName|列名称| -|colType|列类型 [colType支持的类型](docs/colType.md)| +|colType|列类型 [colType支持的类型](../colType.md)| ## 4.参数： |参数名称|含义|是否必填|默认值| diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index 25af638d6..ba1465aba 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -33,7 +33,7 @@ ## 4.参数 -参数详细说明请看[参数详细说明](./sideParams.md) +参数详细说明请看[参数详细说明](sideParams.md) |参数名称|含义|是否必填|默认值| |----|---|---|----| @@ -138,7 +138,7 @@ CREATE TABLE MyResult( CREATE TABLE sideTable ( wtz:message varchar as message, wtz:info varchar as info , - PRIMARY KEY (rowkey), + PRIMARY KEY (md5(key1) + key2), PERIOD FOR SYSTEM_TIME ) WITH ( type = 'hbase', @@ -166,6 +166,6 @@ into MyTable a left join sideTable b - on a.id=b.rowkey; + on a.id=b.key1 and a.xx = b.key2; ``` diff --git a/docs/plugin/hbaseSink.md b/docs/plugin/hbaseSink.md index 00c340b3d..ef1be339b 100644 --- a/docs/plugin/hbaseSink.md +++ b/docs/plugin/hbaseSink.md @@ -24,7 +24,7 @@ hbase2.0 |----|---| | tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称 | colFamily:colName | hbase中的列族名称和列名称 -| colType | 列类型 [colType支持的类型](docs/colType.md) +| colType | 列类型 [colType支持的类型](../colType.md) ## 4.参数： diff --git a/docs/plugin/impalaSide.md b/docs/plugin/impalaSide.md index dce959388..64c22b498 100644 --- a/docs/plugin/impalaSide.md +++ b/docs/plugin/impalaSide.md @@ -26,7 +26,7 @@ ## 3.表结构定义 - [维表参数信息](docs/plugin/sideParams.md) + [维表参数信息](sideParams.md) impala独有的参数配置 diff --git a/docs/plugin/impalaSink.md b/docs/plugin/impalaSink.md index 398ec6ef2..4847cf2e2 100644 --- a/docs/plugin/impalaSink.md +++ b/docs/plugin/impalaSink.md @@ -24,7 +24,7 @@ CREATE TABLE tableName( |----|---| | tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/plugin/impalaColType.md)| +| colType | 列类型 [colType支持的类型](impalaColType.md)| ## 4.参数： diff --git a/docs/plugin/kafkaSink.md b/docs/plugin/kafkaSink.md index 8b50c9e3f..1d8d8ce5f 100644 --- a/docs/plugin/kafkaSink.md +++ b/docs/plugin/kafkaSink.md @@ -27,7 +27,7 @@ CREATE TABLE tableName( |----|---| | tableName| 结果表名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： diff --git a/docs/plugin/kafkaSource.md b/docs/plugin/kafkaSource.md index bfc182b79..9e628f287 100644 --- a/docs/plugin/kafkaSource.md +++ b/docs/plugin/kafkaSource.md @@ -29,7 +29,7 @@ CREATE TABLE tableName( |----|---| | tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| | function(colNameX) as aliasName | 支持在定义列信息的时候根据已有列类型生成新的列(函数可以使用系统函数和已经注册的UDF)| | WATERMARK FOR colName AS withOffset( colName , delayTime ) | 标识输入流生的watermake生成规则,根据指定的colName(当前支持列的类型为Long | Timestamp) 和delayTime生成waterMark 同时会在注册表的使用附带上rowtime字段(如果未指定则默认添加proctime字段);注意：添加该标识的使用必须设置系统参数 time.characteristic:EventTime; delayTime: 数据最大延迟时间(ms)| @@ -48,7 +48,7 @@ CREATE TABLE tableName( |sourcedatatype | 数据类型,avro,csv,json,dt_nest。dt_nest为默认JSON解析器，能够解析嵌套JSON数据类型，其他仅支持非嵌套格式|否|dt_nest| |schemaInfo | avro类型使用的schema信息|否|| |fieldDelimiter |csv类型使用的数据分隔符|否| | | -|timezone|时区设置[timezone支持的参数](timeZone.md)|否|'Asia/Shanghai' +|timezone|时区设置[timezone支持的参数](../timeZone.md)|否|'Asia/Shanghai' **kafka相关参数可以自定义，使用kafka.开头即可。** ``` kafka.consumer.id diff --git a/docs/plugin/kuduSide.md b/docs/plugin/kuduSide.md index a45300deb..a2636d4aa 100644 --- a/docs/plugin/kuduSide.md +++ b/docs/plugin/kuduSide.md @@ -36,13 +36,13 @@ |----|---| | tableName | 注册到flink的表名称(可选填;不填默认和hbase对应的表名称相同)| | colName | 列名称| - | colType | 列类型 [colType支持的类型](colType.md)| + | colType | 列类型 [colType支持的类型](../colType.md)| | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| ## 4.参数 -参数详细说明请看[参数详细说明](./sideParams.md) +参数详细说明请看[参数详细说明](sideParams.md) |参数名称|含义|是否必填|默认值| |----|---|---|-----| diff --git a/docs/plugin/kuduSink.md b/docs/plugin/kuduSink.md index b39ba1d67..cc8f8535a 100644 --- a/docs/plugin/kuduSink.md +++ b/docs/plugin/kuduSink.md @@ -27,7 +27,7 @@ kudu 1.9.0+cdh6.2.0 |----|---| | tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称 | colName | 列名称，redis中存储为 表名:主键名:主键值:列名]| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： diff --git a/docs/plugin/mongoSide.md b/docs/plugin/mongoSide.md index 92362a752..9ff52b85c 100644 --- a/docs/plugin/mongoSide.md +++ b/docs/plugin/mongoSide.md @@ -26,7 +26,7 @@ ## 3.表结构定义 - [通用维表参数信息](docs/plugin/sideParams.md) + [通用维表参数信息](sideParams.md) mongo相关参数配置： diff --git a/docs/plugin/mongoSink.md b/docs/plugin/mongoSink.md index bf4bbb233..9eea6fd89 100644 --- a/docs/plugin/mongoSink.md +++ b/docs/plugin/mongoSink.md @@ -25,7 +25,7 @@ CREATE TABLE tableName( |----|---| | tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： diff --git a/docs/plugin/mysqlSide.md b/docs/plugin/mysqlSide.md index 834183ed2..a4247ef0a 100644 --- a/docs/plugin/mysqlSide.md +++ b/docs/plugin/mysqlSide.md @@ -29,7 +29,7 @@ ## 3.表结构定义 - [维表参数信息](docs/plugin/sideParams.md) + [维表参数信息](sideParams.md) mysql独有的参数配置： diff --git a/docs/plugin/mysqlSink.md b/docs/plugin/mysqlSink.md index ffc62792c..d54c36b28 100644 --- a/docs/plugin/mysqlSink.md +++ b/docs/plugin/mysqlSink.md @@ -25,7 +25,7 @@ CREATE TABLE tableName( |----|---| | tableName| 结果表名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： diff --git a/docs/plugin/oracleSide.md b/docs/plugin/oracleSide.md index b681536f8..feb99a1ee 100644 --- a/docs/plugin/oracleSide.md +++ b/docs/plugin/oracleSide.md @@ -31,7 +31,7 @@ 10g 11g ## 3. 表结构定义 - [维表参数](docs/plugin/sideParams.md) + [维表参数](sideParams.md) oracle独有的参数配置： diff --git a/docs/plugin/oracleSink.md b/docs/plugin/oracleSink.md index 93771f3d6..40c5836e9 100644 --- a/docs/plugin/oracleSink.md +++ b/docs/plugin/oracleSink.md @@ -24,7 +24,7 @@ CREATE TABLE tableName( |----|---| | tableName| oracle表名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： diff --git a/docs/plugin/polardbSide.md b/docs/plugin/polardbSide.md index 625613f98..eaef0acb9 100644 --- a/docs/plugin/polardbSide.md +++ b/docs/plugin/polardbSide.md @@ -28,7 +28,7 @@ |----|---| | tableName | polardb表名称| | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| + | colType | 列类型 [colType支持的类型](../colType.md)| | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| diff --git a/docs/plugin/polardbSink.md b/docs/plugin/polardbSink.md index d252dd712..4a17b72c6 100644 --- a/docs/plugin/polardbSink.md +++ b/docs/plugin/polardbSink.md @@ -24,7 +24,7 @@ CREATE TABLE tableName( |----|---| | tableName| polardb表名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： diff --git a/docs/plugin/postgresqlSide.md b/docs/plugin/postgresqlSide.md index b50a2de52..43fdffeee 100644 --- a/docs/plugin/postgresqlSide.md +++ b/docs/plugin/postgresqlSide.md @@ -29,7 +29,7 @@ ## 3.表结构定义 - [维表参数信息](docs/plugin/sideParams.md) + [维表参数信息](sideParams.md) postgresql独有的参数配置 diff --git a/docs/plugin/postgresqlSink.md b/docs/plugin/postgresqlSink.md index eb055cc04..bc9f98fc0 100644 --- a/docs/plugin/postgresqlSink.md +++ b/docs/plugin/postgresqlSink.md @@ -25,7 +25,7 @@ CREATE TABLE tableName( |----|---| | tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| | primary key | updateMode为UPSERT时，需要指定的主键信息| diff --git a/docs/plugin/redisSide.md b/docs/plugin/redisSide.md index c63c542cd..5a037c763 100644 --- a/docs/plugin/redisSide.md +++ b/docs/plugin/redisSide.md @@ -36,7 +36,7 @@ ## 4.参数 -参数详细说明请看[参数详细说明](./sideParams.md) +参数详细说明请看[参数详细说明](sideParams.md) |参数名称|含义|是否必填|默认值| |----|---|---|----| diff --git a/docs/plugin/sqlserverSide.md b/docs/plugin/sqlserverSide.md index f95d8bba5..90d036fd2 100644 --- a/docs/plugin/sqlserverSide.md +++ b/docs/plugin/sqlserverSide.md @@ -29,7 +29,7 @@ ## 3.表结构定义 - [维表参数信息](docs/plugin/sideParams.md) + [维表参数信息](sideParams.md) sqlserver独有的参数配置 diff --git a/docs/plugin/sqlserverSink.md b/docs/plugin/sqlserverSink.md index 0ab9431e9..4b82898fd 100644 --- a/docs/plugin/sqlserverSink.md +++ b/docs/plugin/sqlserverSink.md @@ -25,7 +25,7 @@ CREATE TABLE tableName( |----|---| | tableName| sqlserver表名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| | primary key | updateMode为UPSERT时，需要指定的主键信息| ## 4.参数： From c83dc91a9c68ae087203371dac62ab1393074b1c Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 21 May 2020 16:19:44 +0800 Subject: [PATCH 335/523] =?UTF-8?q?hbase=20=E8=AF=A6=E7=BB=86=E7=A4=BA?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/hbaseSide.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index ba1465aba..e590b02e7 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -98,12 +98,12 @@ CREATE TABLE sideTable ( ``` hbase(main):002:0> scan 'testFlinkStreamSql' ROW COLUMN+CELL - 0 column=wtz:info, timestamp=1587089266719, value=hadoop - 0 column=wtz:message, timestamp=1587089245780, value=hbase - 1 column=wtz:info, timestamp=1587088818432, value=flink - 1 column=wtz:message, timestamp=1587088796633, value=dtstack - 2 column=wtz:info, timestamp=1587088858564, value=sql - 2 column=wtz:message, timestamp=1587088840507, value=stream + cfcd208495d565ef66e7dff9f98764dazhangsantest column=wtz:info, timestamp=1587089266719, value=hadoop + cfcd208495d565ef66e7dff9f98764dazhangsantest column=wtz:message, timestamp=1587089245780, value=hbase + c4ca4238a0b923820dcc509a6f75849blisitest column=wtz:info, timestamp=1587088818432, value=flink + c4ca4238a0b923820dcc509a6f75849blisitest column=wtz:message, timestamp=1587088796633, value=dtstack + c81e728d9d4c2f636f067f89cc14862cwangwutest column=wtz:info, timestamp=1587088858564, value=sql + c81e728d9d4c2f636f067f89cc14862cwangwutest column=wtz:message, timestamp=1587088840507, value=stream ``` 在hbase中，rowKey是一个二进制码流，可以为任意字符串，flinkStreamSql读取rowKey并通过rowKey唯一确定数据，对rowKey没有任何限制,对rowKey可选择的构造包括 md5(alias + alias), '常量',也可以它们的自由组合。 在本次案例中，rowKey为了简单，设置成了"0,1,2"这样的数值型字符，若有必要，也可以设计得更为复杂。 @@ -138,7 +138,7 @@ CREATE TABLE MyResult( CREATE TABLE sideTable ( wtz:message varchar as message, wtz:info varchar as info , - PRIMARY KEY (md5(key1) + key2), + PRIMARY KEY (md5(rowkey1) + rowkey2 + 'test'), PERIOD FOR SYSTEM_TIME ) WITH ( type = 'hbase', @@ -166,6 +166,6 @@ into MyTable a left join sideTable b - on a.id=b.key1 and a.xx = b.key2; + on a.id=b.rowkey1 and a.name = b.rowkey2; ``` From 33047f6a8f2cee0ed59b9a65e8bf5b7a1f15f8a1 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 21 May 2020 20:00:40 +0800 Subject: [PATCH 336/523] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E4=B8=AD=E6=96=87?= =?UTF-8?q?=E4=B9=B1=E5=90=97=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/impala/ImpalaDialect.java | 16 ++++++++++++++-- .../flink/sql/sink/impala/ImpalaSink.java | 4 ++-- .../sql/sink/impala/table/ImpalaSinkParser.java | 1 + 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java index 3eaffc0ad..89fb769c3 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; import java.util.Arrays; import java.util.List; @@ -38,6 +39,12 @@ public class ImpalaDialect implements JDBCDialect { private static final String IMPALA_PARTITION_KEYWORD = "partition"; + private TypeInformation[] fieldTypes; + + public ImpalaDialect(TypeInformation[] fieldTypes){ + this.fieldTypes = fieldTypes; + } + @Override public boolean canHandle(String url) { return url.startsWith("jdbc:impala:"); @@ -70,8 +77,13 @@ public String getInsertIntoStatement(String schema, String tableName, String[] f .map(this::quoteIdentifier) .collect(Collectors.joining(", ")); - String placeholders = Arrays.stream(fieldNames) - .map(f -> "?") + String placeholders = Arrays.stream(fieldTypes) + .map(f -> { + if(String.class.getName().equals(f.getTypeClass().getName())){ + return "cast( ? as string)"; + } + return "?"; + }) .collect(Collectors.joining(", ")); String partitionFieldStr = partitionFieldsList.stream() diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index cd1e1e945..910f6f4c5 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -41,14 +41,14 @@ public class ImpalaSink extends AbstractRdbSink implements IStreamSinkGener Date: Mon, 25 May 2020 14:51:51 +0800 Subject: [PATCH 337/523] add hbase kerbos --- .../flink/sql/side/hbase/HbaseAllReqRow.java | 85 ++++++++++++-- .../sql/side/hbase/HbaseAsyncReqRow.java | 40 ++++++- .../side/hbase/table/HbaseSideTableInfo.java | 48 ++++++++ .../side/hbase/utils/HbaseConfigUtils.java | 108 ++++++++++++++++++ 4 files changed, 272 insertions(+), 9 deletions(-) create mode 100644 hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 63d26d5dd..401ef00c0 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -25,8 +25,10 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; +import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.map.HashedMap; +import org.apache.commons.lang.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; import org.apache.flink.table.runtime.types.CRow; @@ -36,6 +38,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; @@ -44,10 +47,13 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; +import java.security.PrivilegedAction; import java.sql.SQLException; import java.sql.Timestamp; import java.util.Calendar; @@ -166,13 +172,44 @@ public void flatMap(CRow input, Collector out) throws Exception { private void loadData(Map> tmpCache) throws SQLException { AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; - Configuration conf = new Configuration(); - conf.set("hbase.zookeeper.quorum", hbaseSideTableInfo.getHost()); - Connection conn = null; - Table table = null; - ResultScanner resultScanner = null; + boolean openKerberos = hbaseSideTableInfo.isKerberosAuthEnable(); + int loadDataCount = 0; try { - conn = ConnectionFactory.createConnection(conf); + conf = HBaseConfiguration.create(); + if (openKerberos) { + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_SYNC, hbaseSideTableInfo.getParent()); + + fillSyncKerberosConfig(conf,hbaseSideTableInfo); + LOG.info("hbase.security.authentication:{}", conf.get("hbase.security.authentication")); + LOG.info("hbase.security.authorization:{}", conf.get("hbase.security.authorization")); + LOG.info("hbase.master.keytab.file:{}", conf.get("hbase.master.keytab.file")); + LOG.info("hbase.master.kerberos.principal:{}", conf.get("hbase.master.kerberos.principal")); + LOG.info("hbase.regionserver.keytab.file:{}", conf.get("hbase.regionserver.keytab.file")); + LOG.info("hbase.regionserver.kerberos.principal:{}", conf.get("hbase.regionserver.kerberos.principal")); + + UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, hbaseSideTableInfo.getRegionserverPrincipal(), + hbaseSideTableInfo.getRegionserverKeytabFile()); + + Configuration finalConf = conf; + conn = userGroupInformation.doAs(new PrivilegedAction() { + @Override + public Connection run() { + try { + return ConnectionFactory.createConnection(finalConf); + } catch (IOException e) { + LOG.error("Get connection fail with config:{}", finalConf); + throw new RuntimeException(e); + } + } + }); + + } else { + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_SYNC, hbaseSideTableInfo.getParent()); + conn = ConnectionFactory.createConnection(conf); + } + table = conn.getTable(TableName.valueOf(tableName)); resultScanner = table.getScanner(new Scan()); for (Result r : resultScanner) { @@ -187,13 +224,15 @@ private void loadData(Map> tmpCache) throws SQLExcep kv.put(aliasNameInversion.get(key.toString()), value); } + loadDataCount++; tmpCache.put(new String(r.getRow()), kv); } } catch (IOException e) { - LOG.error("", e); + throw new RuntimeException(e); } finally { + LOG.info("load Data count: {}", loadDataCount); try { - if (null != conn && !conn.isClosed()) { + if (null != conn) { conn.close(); } @@ -209,4 +248,34 @@ private void loadData(Map> tmpCache) throws SQLExcep } } } + + private void fillSyncKerberosConfig(Configuration config, HbaseSideTableInfo hbaseSideTableInfo) throws IOException { + String regionserverKeytabFile = hbaseSideTableInfo.getRegionserverKeytabFile(); + if (StringUtils.isEmpty(regionserverKeytabFile)) { + throw new IllegalArgumentException("Must provide regionserverKeytabFile when authentication is Kerberos"); + } + String regionserverKeytabFilePath = System.getProperty("user.dir") + File.separator + regionserverKeytabFile; + LOG.info("regionserverKeytabFilePath:{}", regionserverKeytabFilePath); + config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KEYTAB_FILE, regionserverKeytabFilePath); + config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KEYTAB_FILE, regionserverKeytabFilePath); + + String regionserverPrincipal = hbaseSideTableInfo.getRegionserverPrincipal(); + if (StringUtils.isEmpty(regionserverPrincipal)) { + throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); + } + config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KERBEROS_PRINCIPAL, regionserverPrincipal); + config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL, regionserverPrincipal); + config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHORIZATION, "true"); + config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHENTICATION, "kerberos"); + + if (!StringUtils.isEmpty(hbaseSideTableInfo.getZookeeperSaslClient())) { + System.setProperty(HbaseConfigUtils.KEY_ZOOKEEPER_SASL_CLIENT, hbaseSideTableInfo.getZookeeperSaslClient()); + } + + if (!StringUtils.isEmpty(hbaseSideTableInfo.getSecurityKrb5Conf())) { + String krb5ConfPath = System.getProperty("user.dir") + File.separator + hbaseSideTableInfo.getSecurityKrb5Conf(); + LOG.info("krb5ConfPath:{}", krb5ConfPath); + System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_KRB5_CONF, krb5ConfPath); + } + } } \ No newline at end of file diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 56f50e27c..d43d1408a 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -31,6 +31,7 @@ import com.dtstack.flink.sql.side.hbase.rowkeydealer.RowKeyEqualModeDealer; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; import com.dtstack.flink.sql.factory.DTThreadFactory; +import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; import com.google.common.collect.Maps; import com.stumbleupon.async.Deferred; import org.apache.commons.lang3.StringUtils; @@ -40,10 +41,13 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; +import org.hbase.async.Config; import org.hbase.async.HBaseClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.IOException; import java.sql.Timestamp; import java.util.Collections; import java.util.List; @@ -93,11 +97,19 @@ public void open(Configuration parameters) throws Exception { super.open(parameters); AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; + ExecutorService executorService =new ThreadPoolExecutor(DEFAULT_POOL_SIZE, DEFAULT_POOL_SIZE, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new DTThreadFactory("hbase-aysnc")); - hBaseClient = new HBaseClient(hbaseSideTableInfo.getHost(), hbaseSideTableInfo.getParent(), executorService); + Config config = new Config(); + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_ASYNC, hbaseSideTableInfo.getParent()); + + if (hbaseSideTableInfo.isKerberosAuthEnable()) { + fillAsyncKerberosConfig(config, hbaseSideTableInfo); + } + hBaseClient = new HBaseClient(config, executorService); try { Deferred deferred = hBaseClient.ensureTableExists(tableName) @@ -166,6 +178,32 @@ public void close() throws Exception { hBaseClient.shutdown(); } + private void fillAsyncKerberosConfig(Config config, HbaseSideTableInfo hbaseSideTableInfo) throws IOException { + AuthUtil.JAASConfig jaasConfig = HbaseConfigUtils.buildJaasConfig(hbaseSideTableInfo); + LOG.info("jaasConfig file:\n {}", jaasConfig.toString()); + String jaasFilePath = AuthUtil.creatJaasFile("JAAS", ".conf", jaasConfig); + config.overrideConfig(HbaseConfigUtils.KEY_JAVA_SECURITY_AUTH_LOGIN_CONF, jaasFilePath); + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTH_ENABLE, "true"); + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_SASL_CLIENTCONFIG, "Client"); + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHENTICATION, "kerberos"); + + String regionserverPrincipal = hbaseSideTableInfo.getRegionserverPrincipal(); + if (StringUtils.isEmpty(regionserverPrincipal)) { + throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); + } + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL, regionserverPrincipal); + + if (!StringUtils.isEmpty(hbaseSideTableInfo.getZookeeperSaslClient())) { + System.setProperty(HbaseConfigUtils.KEY_ZOOKEEPER_SASL_CLIENT, hbaseSideTableInfo.getZookeeperSaslClient()); + } + + if (!StringUtils.isEmpty(hbaseSideTableInfo.getSecurityKrb5Conf())) { + String krb5ConfPath = System.getProperty("user.dir") + File.separator + hbaseSideTableInfo.getSecurityKrb5Conf(); + LOG.info("krb5ConfPath:{}", krb5ConfPath); + System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_KRB5_CONF, krb5ConfPath); + } + } + class CheckResult{ diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java index 3cedb2c68..d8b0a62bd 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java @@ -148,6 +148,54 @@ public void setPreRowKey(boolean preRowKey) { this.preRowKey = preRowKey; } + public boolean isKerberosAuthEnable() { + return kerberosAuthEnable; + } + + public void setKerberosAuthEnable(boolean kerberosAuthEnable) { + this.kerberosAuthEnable = kerberosAuthEnable; + } + + public String getRegionserverKeytabFile() { + return regionserverKeytabFile; + } + + public void setRegionserverKeytabFile(String regionserverKeytabFile) { + this.regionserverKeytabFile = regionserverKeytabFile; + } + + public String getRegionserverPrincipal() { + return regionserverPrincipal; + } + + public void setRegionserverPrincipal(String regionserverPrincipal) { + this.regionserverPrincipal = regionserverPrincipal; + } + + public String getJaasPrincipal() { + return jaasPrincipal; + } + + public void setJaasPrincipal(String jaasPrincipal) { + this.jaasPrincipal = jaasPrincipal; + } + + public String getSecurityKrb5Conf() { + return securityKrb5Conf; + } + + public void setSecurityKrb5Conf(String securityKrb5Conf) { + this.securityKrb5Conf = securityKrb5Conf; + } + + public String getZookeeperSaslClient() { + return zookeeperSaslClient; + } + + public void setZookeeperSaslClient(String zookeeperSaslClient) { + this.zookeeperSaslClient = zookeeperSaslClient; + } + @Override public void finish(){ super.finish(); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java new file mode 100644 index 000000000..09f2ea53d --- /dev/null +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.hbase.utils; + +import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; +import com.dtstack.flink.sql.util.AuthUtil; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * + * The utility class of HBase connection + * + * Date: 2019/12/24 + * Company: www.dtstack.com + * @author maqi + */ +public class HbaseConfigUtils { + + private static final Logger LOG = LoggerFactory.getLogger(HbaseConfigUtils.class); + // sync side kerberos + public final static String KEY_HBASE_SECURITY_AUTHENTICATION = "hbase.security.authentication"; + public final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization"; + public final static String KEY_HBASE_MASTER_KEYTAB_FILE = "hbase.master.keytab.file"; + public final static String KEY_HBASE_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal"; + public final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; + public final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; + + // async side kerberos + public final static String KEY_HBASE_SECURITY_AUTH_ENABLE = "hbase.security.auth.enable"; + public final static String KEY_HBASE_SASL_CLIENTCONFIG = "hbase.sasl.clientconfig"; + public final static String KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL = "hbase.kerberos.regionserver.principal"; + + public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; + public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_SYNC = "zookeeper.znode.parent"; + public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_ASYNC = "hbase.zookeeper.znode.parent"; + + + public static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; + public static final String KEY_ZOOKEEPER_SASL_CLIENT = "zookeeper.sasl.client"; + + public static final String KEY_JAVA_SECURITY_AUTH_LOGIN_CONF = "java.security.auth.login.config"; + + + public static AuthUtil.JAASConfig buildJaasConfig(HbaseSideTableInfo hbaseSideTableInfo) { + String keytabPath = System.getProperty("user.dir") + File.separator + hbaseSideTableInfo.getRegionserverKeytabFile(); + Map loginModuleOptions = new HashMap<>(); + loginModuleOptions.put("useKeyTab", "true"); + loginModuleOptions.put("useTicketCache", "false"); + loginModuleOptions.put("keyTab", "\"" + keytabPath + "\""); + loginModuleOptions.put("principal", "\"" + hbaseSideTableInfo.getJaasPrincipal() + "\""); + return AuthUtil.JAASConfig.builder().setEntryName("Client") + .setLoginModule("com.sun.security.auth.module.Krb5LoginModule") + .setLoginModuleFlag("required").setLoginModuleOptions(loginModuleOptions).build(); + } + + + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { + if (conf == null) { + throw new IllegalArgumentException("kerberos conf can not be null"); + } + + if (org.apache.commons.lang.StringUtils.isEmpty(principal)) { + throw new IllegalArgumentException("principal can not be null"); + } + + if (org.apache.commons.lang.StringUtils.isEmpty(keytab)) { + throw new IllegalArgumentException("keytab can not be null"); + } + + conf.set("hadoop.security.authentication", "Kerberos"); + UserGroupInformation.setConfiguration(conf); + + return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab); + } + +} From 1e3237f503754c38d4b89b6ee28d2473d8bf904b Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 25 May 2020 14:54:27 +0800 Subject: [PATCH 338/523] fix missing class --- .../com/dtstack/flink/sql/util/AuthUtil.java | 112 ++++++++++++++++++ .../flink/sql/side/hbase/HbaseAllReqRow.java | 4 + .../sql/side/hbase/HbaseAsyncReqRow.java | 1 + 3 files changed, 117 insertions(+) create mode 100644 core/src/main/java/com/dtstack/flink/sql/util/AuthUtil.java diff --git a/core/src/main/java/com/dtstack/flink/sql/util/AuthUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/AuthUtil.java new file mode 100644 index 000000000..646a11b66 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/util/AuthUtil.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.util; + +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Utility methods for helping with security tasks. + * Date: 2019/12/28 + * Company: www.dtstack.com + * @author maqi + */ +public class AuthUtil { + + public static String creatJaasFile(String prefix, String suffix, JAASConfig jaasConfig) throws IOException { + File krbConf = new File(System.getProperty("user.dir")); + File temp = File.createTempFile(prefix, suffix, krbConf); + temp.deleteOnExit(); + FileUtils.writeStringToFile(temp, jaasConfig.toString()); + return temp.getAbsolutePath(); + } + + + public static class JAASConfig { + private String entryName; + private String loginModule; + private String loginModuleFlag; + private Map loginModuleOptions; + + public JAASConfig(String entryName, String loginModule, String loginModuleFlag, Map loginModuleOptions) { + this.entryName = entryName; + this.loginModule = loginModule; + this.loginModuleFlag = loginModuleFlag; + this.loginModuleOptions = loginModuleOptions; + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public String toString() { + StringBuilder stringBuilder = new StringBuilder(entryName).append(" {\n\t") + .append(loginModule).append(" ").append(loginModuleFlag).append("\n\t"); + String[] keys = loginModuleOptions.keySet().toArray(new String[loginModuleOptions.size()]); + for (int i = 0; i < keys.length; i++) { + stringBuilder.append(keys[i]).append("=").append(loginModuleOptions.get(keys[i])); + if (i != keys.length - 1) { + stringBuilder.append("\n\t"); + } else { + stringBuilder.append(";\n"); + } + + } + stringBuilder.append("\n").append("};"); + return stringBuilder.toString(); + } + + public static class Builder { + private String entryName; + private String loginModule; + private String loginModuleFlag; + private Map loginModuleOptions; + + public Builder setEntryName(String entryName) { + this.entryName = entryName; + return this; + } + + public Builder setLoginModule(String loginModule) { + this.loginModule = loginModule; + return this; + } + + public Builder setLoginModuleFlag(String loginModuleFlag) { + this.loginModuleFlag = loginModuleFlag; + return this; + } + + public Builder setLoginModuleOptions(Map loginModuleOptions) { + this.loginModuleOptions = loginModuleOptions; + return this; + } + + public JAASConfig build() { + return new JAASConfig( + entryName, loginModule, loginModuleFlag, loginModuleOptions); + } + } + } +} diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 401ef00c0..c50e1a806 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -71,6 +71,10 @@ public class HbaseAllReqRow extends BaseAllReqRow { private Map aliasNameInversion; private AtomicReference>> cacheRef = new AtomicReference<>(); + private Connection conn = null; + private Table table = null; + private ResultScanner resultScanner = null; + private Configuration conf = null; public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new HbaseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index d43d1408a..f3a1d6691 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -32,6 +32,7 @@ import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; +import com.dtstack.flink.sql.util.AuthUtil; import com.google.common.collect.Maps; import com.stumbleupon.async.Deferred; import org.apache.commons.lang3.StringUtils; From 6bb89495cc5679bd4eea8a8ce5166472a6bf6e64 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 25 May 2020 14:56:59 +0800 Subject: [PATCH 339/523] fill missing field --- .../sql/side/hbase/table/HbaseSideTableInfo.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java index d8b0a62bd..2a5411682 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java @@ -52,6 +52,18 @@ public class HbaseSideTableInfo extends AbstractSideTableInfo { private String tableName; + private boolean kerberosAuthEnable; + + private String regionserverKeytabFile; + + private String regionserverPrincipal; + + private String jaasPrincipal; + + private String securityKrb5Conf; + + private String zookeeperSaslClient; + private String[] columnRealNames; private List columnRealNameList = Lists.newArrayList(); From 907aacb8961d6521c1b91f0fe1b1651318fc2a1b Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 25 May 2020 15:15:36 +0800 Subject: [PATCH 340/523] hbase kerberos sink --- .../sql/sink/hbase/HbaseOutputFormat.java | 33 +++++++++++++++- .../flink/sql/sink/hbase/HbaseSink.java | 39 +++++++++++++++---- 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index a3189b0a6..faea0845c 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -62,6 +62,12 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private String[] columnTypes; private Map columnNameFamily; + private boolean kerberosAuthEnable; + private String regionserverKeytabFile; + private String regionserverPrincipal; + private String securityKrb5Conf; + private String zookeeperSaslClient; + private String[] families; private String[] qualifiers; @@ -252,6 +258,31 @@ public HbaseOutputFormatBuilder setColumnNameFamily(Map columnNa format.columnNameFamily = columnNameFamily; return this; } + public HbaseOutputFormatBuilder setKerberosAuthEnable(boolean kerberosAuthEnable) { + format.kerberosAuthEnable = kerberosAuthEnable; + return this; + } + + public HbaseOutputFormatBuilder setRegionserverKeytabFile(String regionserverKeytabFile) { + format.regionserverKeytabFile = regionserverKeytabFile; + return this; + } + + public HbaseOutputFormatBuilder setRegionserverPrincipal(String regionserverPrincipal) { + format.regionserverPrincipal = regionserverPrincipal; + return this; + } + + public HbaseOutputFormatBuilder setSecurityKrb5Conf(String securityKrb5Conf) { + format.securityKrb5Conf = securityKrb5Conf; + return this; + } + + public HbaseOutputFormatBuilder setZookeeperSaslClient(String zookeeperSaslClient) { + format.zookeeperSaslClient = zookeeperSaslClient; + return this; + } + public HbaseOutputFormat finish() { Preconditions.checkNotNull(format.host, "zookeeperQuorum should be specified"); @@ -267,7 +298,7 @@ public HbaseOutputFormat finish() { String[] columns = keySet.toArray(new String[keySet.size()]); for (int i = 0; i < columns.length; ++i) { String col = columns[i]; - String[] part = StringUtils.split(col, ":"); + String[] part = col.split(":"); families[i] = part[0]; qualifiers[i] = part[1]; } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index 13bd98b70..8364b6c10 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -52,6 +52,15 @@ public class HbaseSink implements RetractStreamTableSink, IStreamSinkGener< protected String tableName; protected String updateMode; protected String rowkey; + protected String registerTabName; + + protected boolean kerberosAuthEnable; + protected String regionserverKeytabFile; + protected String regionserverPrincipal; + protected String securityKrb5Conf; + protected String zookeeperSaslClient; + private int parallelism = -1; + public HbaseSink() { // TO DO NOTHING @@ -66,20 +75,34 @@ public HbaseSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.tableName = hbaseTableInfo.getTableName(); this.rowkey = hbaseTableInfo.getRowkey(); this.columnNameFamily = hbaseTableInfo.getColumnNameFamily(); - this.updateMode = hbaseTableInfo.getUpdateMode(); + this.registerTabName = hbaseTableInfo.getName(); + + this.kerberosAuthEnable = hbaseTableInfo.isKerberosAuthEnable(); + this.regionserverKeytabFile = hbaseTableInfo.getRegionserverKeytabFile(); + this.regionserverPrincipal = hbaseTableInfo.getRegionserverPrincipal(); + this.securityKrb5Conf = hbaseTableInfo.getSecurityKrb5Conf(); + this.zookeeperSaslClient = hbaseTableInfo.getZookeeperSaslClient(); + + Integer tmpSinkParallelism = hbaseTableInfo.getParallelism(); + if (tmpSinkParallelism != null) { + this.parallelism = tmpSinkParallelism; + } return this; } @Override public void emitDataStream(DataStream> dataStream) { HbaseOutputFormat.HbaseOutputFormatBuilder builder = HbaseOutputFormat.buildHbaseOutputFormat(); - builder.setHost(this.zookeeperQuorum) - .setZkParent(this.parent) - .setTable(this.tableName) - .setRowkey(rowkey) - .setUpdateMode(updateMode) - .setColumnNames(fieldNames) - .setColumnNameFamily(columnNameFamily); + builder.setHost(this.zookeeperQuorum).setZkParent(this.parent).setTable(this.tableName); + + builder.setRowkey(rowkey); + builder.setColumnNames(fieldNames); + builder.setColumnNameFamily(columnNameFamily); + builder.setKerberosAuthEnable(kerberosAuthEnable); + builder.setRegionserverKeytabFile(regionserverKeytabFile); + builder.setRegionserverPrincipal(regionserverPrincipal); + builder.setSecurityKrb5Conf(securityKrb5Conf); + builder.setZookeeperSaslClient(zookeeperSaslClient); HbaseOutputFormat outputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(outputFormat); From 3a9cb360d3d6c7e54e75170c1ae638c3e154d428 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 25 May 2020 15:18:21 +0800 Subject: [PATCH 341/523] fix missing field --- .../sql/sink/hbase/table/HbaseTableInfo.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java index 36f8e89e5..49df2f757 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java @@ -23,6 +23,8 @@ import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; + import java.util.Map; /** @@ -54,6 +56,18 @@ public class HbaseTableInfo extends AbstractTargetTableInfo { private String updateMode; + private boolean kerberosAuthEnable; + + private String regionserverKeytabFile; + + private String regionserverPrincipal; + + private String securityKrb5Conf; + + private String zookeeperSaslClient; + + private Map hbaseConfig = Maps.newHashMap(); + public HbaseTableInfo(){ setType(CURR_TYPE); } @@ -149,4 +163,53 @@ public String getType() { return super.getType().toLowerCase(); } + + public Map getHbaseConfig() { + return hbaseConfig; + } + + public void setHbaseConfig(Map hbaseConfig) { + this.hbaseConfig = hbaseConfig; + } + + public boolean isKerberosAuthEnable() { + return kerberosAuthEnable; + } + + public void setKerberosAuthEnable(boolean kerberosAuthEnable) { + this.kerberosAuthEnable = kerberosAuthEnable; + } + + public String getRegionserverKeytabFile() { + return regionserverKeytabFile; + } + + public void setRegionserverKeytabFile(String regionserverKeytabFile) { + this.regionserverKeytabFile = regionserverKeytabFile; + } + + public String getRegionserverPrincipal() { + return regionserverPrincipal; + } + + public void setRegionserverPrincipal(String regionserverPrincipal) { + this.regionserverPrincipal = regionserverPrincipal; + } + + public String getSecurityKrb5Conf() { + return securityKrb5Conf; + } + + public void setSecurityKrb5Conf(String securityKrb5Conf) { + this.securityKrb5Conf = securityKrb5Conf; + } + + public String getZookeeperSaslClient() { + return zookeeperSaslClient; + } + + public void setZookeeperSaslClient(String zookeeperSaslClient) { + this.zookeeperSaslClient = zookeeperSaslClient; + } + } From 2f0fdfeec8ddec660618e3574c77226d2b9fc7a4 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 25 May 2020 17:46:25 +0800 Subject: [PATCH 342/523] =?UTF-8?q?hbase=20kerberos=20=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/hbase/table/HbaseSideParser.java | 14 ++++++++++++++ .../sql/sink/hbase/table/HbaseSinkParser.java | 12 ++++++++++++ 2 files changed, 26 insertions(+) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 80753c40b..832961834 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -53,6 +53,13 @@ public class HbaseSideParser extends AbstractSideTableParser { public static final String CACHE = "cache"; + public static final String KERBEROS_AUTH_ENABLE_KEY = "kerberosAuthEnable"; + public static final String REGIONSERVER_KEYTAB_FILE_KEY = "regionserverKeytabFile"; + public static final String REGIONSERVER_PRINCIPAL_KEY = "regionserverPrincipal"; + public static final String JAAS_PRINCIPAL_KEY = "jaasPrincipal"; + public static final String SECURITY_KRB5_CONF_KEY = "securityKrb5Conf"; + public static final String ZOOKEEPER_SASL_CLINT_KEY = "zookeeperSaslClient"; + public HbaseSideParser() { addParserHandler(FIELD_KEY, FIELD_PATTERN, this::dealField); } @@ -69,6 +76,13 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Mon, 25 May 2020 17:56:06 +0800 Subject: [PATCH 343/523] =?UTF-8?q?=E4=BF=9D=E6=8C=81=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index faea0845c..3def5cb1d 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -298,7 +298,7 @@ public HbaseOutputFormat finish() { String[] columns = keySet.toArray(new String[keySet.size()]); for (int i = 0; i < columns.length; ++i) { String col = columns[i]; - String[] part = col.split(":"); + String[] part = StringUtils.split(col, ":");; families[i] = part[0]; qualifiers[i] = part[1]; } From a9f9299d9b1d9624e921c0f8cce7f296d2b9ea60 Mon Sep 17 00:00:00 2001 From: whiletrue Date: Tue, 26 May 2020 20:04:34 +0800 Subject: [PATCH 344/523] Create SECURITY.md --- SECURITY.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..034e84803 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,21 @@ +# Security Policy + +## Supported Versions + +Use this section to tell people about which versions of your project are +currently being supported with security updates. + +| Version | Supported | +| ------- | ------------------ | +| 5.1.x | :white_check_mark: | +| 5.0.x | :x: | +| 4.0.x | :white_check_mark: | +| < 4.0 | :x: | + +## Reporting a Vulnerability + +Use this section to tell people how to report a vulnerability. + +Tell them where to go, how often they can expect to get an update on a +reported vulnerability, what to expect if the vulnerability is accepted or +declined, etc. From 9d80379c90be792d35bdd9ecef20713e72dcb49f Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 27 May 2020 10:56:38 +0800 Subject: [PATCH 345/523] =?UTF-8?q?fix=20=E8=A1=A5=E5=85=85=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/hbase/HbaseConfigUtils.java | 79 +++++++++++++++++++ .../sql/sink/hbase/HbaseOutputFormat.java | 56 +++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java new file mode 100644 index 000000000..2be03484b --- /dev/null +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.hbase; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * + * The utility class of HBase connection + * + * Date: 2019/12/24 + * Company: www.dtstack.com + * @author maqi + */ +public class HbaseConfigUtils { + + private static final Logger LOG = LoggerFactory.getLogger(HbaseConfigUtils.class); + // sync side kerberos + public final static String KEY_HBASE_SECURITY_AUTHENTICATION = "hbase.security.authentication"; + public final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization"; + public final static String KEY_HBASE_MASTER_KEYTAB_FILE = "hbase.master.keytab.file"; + public final static String KEY_HBASE_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal"; + public final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; + public final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; + + public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; + public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "hbase.zookeeper.znode.parent"; + + + public static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; + public static final String KEY_ZOOKEEPER_SASL_CLIENT = "zookeeper.sasl.client"; + + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { + if (conf == null) { + throw new IllegalArgumentException("kerberos conf can not be null"); + } + + if (org.apache.commons.lang.StringUtils.isEmpty(principal)) { + throw new IllegalArgumentException("principal can not be null"); + } + + if (org.apache.commons.lang.StringUtils.isEmpty(keytab)) { + throw new IllegalArgumentException("keytab can not be null"); + } + + conf.set("hadoop.security.authentication", "Kerberos"); + UserGroupInformation.setConfiguration(conf); + + return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab); + } +} diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 3def5cb1d..36b251d14 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -37,10 +37,13 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; +import java.security.PrivilegedAction; import java.util.List; import java.util.Map; import java.util.Set; @@ -91,6 +94,29 @@ public void open(int taskNumber, int numTasks) throws IOException { LOG.warn("---open---"); conn = ConnectionFactory.createConnection(conf); table = conn.getTable(TableName.valueOf(tableName)); + if (kerberosAuthEnable) { + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); + fillSyncKerberosConfig(conf, regionserverKeytabFile, regionserverPrincipal, zookeeperSaslClient, securityKrb5Conf); + + UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, regionserverPrincipal, regionserverKeytabFile); + org.apache.hadoop.conf.Configuration finalConf = conf; + conn = userGroupInformation.doAs(new PrivilegedAction() { + @Override + public Connection run() { + try { + return ConnectionFactory.createConnection(finalConf); + } catch (IOException e) { + LOG.error("Get connection fail with config:{}", finalConf); + throw new RuntimeException(e); + } + } + }); + } else { + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); + conn = ConnectionFactory.createConnection(conf); + } LOG.warn("---open end(get table from hbase) ---"); initMetric(); } @@ -311,5 +337,35 @@ public HbaseOutputFormat finish() { } + private void fillSyncKerberosConfig( org.apache.hadoop.conf.Configuration config, String regionserverKeytabFile, String regionserverPrincipal, + String zookeeperSaslClient, String securityKrb5Conf) throws IOException { + if (StringUtils.isEmpty(regionserverKeytabFile)) { + throw new IllegalArgumentException("Must provide regionserverKeytabFile when authentication is Kerberos"); + } + String regionserverKeytabFilePath = System.getProperty("user.dir") + File.separator + regionserverKeytabFile; + LOG.info("regionserverKeytabFilePath:{}",regionserverKeytabFilePath); + config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KEYTAB_FILE, regionserverKeytabFilePath); + config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KEYTAB_FILE, regionserverKeytabFilePath); + + if (StringUtils.isEmpty(regionserverPrincipal)) { + throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); + } + config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KERBEROS_PRINCIPAL, regionserverPrincipal); + config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL, regionserverPrincipal); + config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHORIZATION, "true"); + config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHENTICATION, "kerberos"); + + + if (!StringUtils.isEmpty(zookeeperSaslClient)) { + System.setProperty(HbaseConfigUtils.KEY_ZOOKEEPER_SASL_CLIENT, zookeeperSaslClient); + } + + if (!StringUtils.isEmpty(securityKrb5Conf)) { + String krb5ConfPath = System.getProperty("user.dir") + File.separator + securityKrb5Conf; + LOG.info("krb5ConfPath:{}", krb5ConfPath); + System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_KRB5_CONF, krb5ConfPath); + } + } + } From a44c2be5167ad98b86d0036ec66e52db6c7cab7d Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 27 May 2020 11:49:03 +0800 Subject: [PATCH 346/523] =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=A1=BA=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 36b251d14..97c3ad873 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -92,8 +92,6 @@ public void configure(Configuration parameters) { @Override public void open(int taskNumber, int numTasks) throws IOException { LOG.warn("---open---"); - conn = ConnectionFactory.createConnection(conf); - table = conn.getTable(TableName.valueOf(tableName)); if (kerberosAuthEnable) { conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); @@ -117,6 +115,7 @@ public Connection run() { conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); conn = ConnectionFactory.createConnection(conf); } + table = conn.getTable(TableName.valueOf(tableName)); LOG.warn("---open end(get table from hbase) ---"); initMetric(); } From 0de96e840c855eee6e52e71ca724a710dd497f85 Mon Sep 17 00:00:00 2001 From: xuchao Date: Wed, 27 May 2020 14:03:38 +0800 Subject: [PATCH 347/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=B9join?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E4=B8=AD=E9=9D=9Eequal=20=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E5=AF=B9=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../environment/MyLocalStreamEnvironment.java | 38 ++++-- .../flink/sql/side/JoinNodeDealer.java | 108 +++++++++++++++--- .../dtstack/flink/sql/util/TableUtils.java | 51 ++++++++- 3 files changed, 169 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java index 029a07f99..6aec8475e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestOptions; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.minicluster.MiniCluster; @@ -93,33 +94,48 @@ public JobExecutionResult execute(String jobName) throws Exception { // transform the streaming program into a JobGraph StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); + return execute(streamGraph); + } + + public JobExecutionResult execute(StreamGraph streamGraph) throws Exception { JobGraph jobGraph = streamGraph.getJobGraph(); jobGraph.setClasspaths(classpaths); + jobGraph.setAllowQueuedScheduling(true); Configuration configuration = new Configuration(); configuration.addAll(jobGraph.getJobConfiguration()); - - configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "512M"); - configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); + configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0"); // add (and override) the settings with what the user defined configuration.addAll(this.conf); - MiniClusterConfiguration.Builder configBuilder = new MiniClusterConfiguration.Builder(); - configBuilder.setConfiguration(configuration); + if (!configuration.contains(RestOptions.BIND_PORT)) { + configuration.setString(RestOptions.BIND_PORT, "0"); + } + + int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); + + MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder() + .setConfiguration(configuration) + .setNumSlotsPerTaskManager(numSlotsPerTaskManager) + .build(); if (LOG.isInfoEnabled()) { LOG.info("Running job on local embedded Flink mini cluster"); } - try (MiniCluster exec = new MiniCluster(configBuilder.build());) { - exec.start(); - JobExecutionResult jobExecutionResult = exec.executeJobBlocking(jobGraph); + MiniCluster miniCluster = new MiniCluster(cfg); + + try { + miniCluster.start(); + configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort()); + + return miniCluster.executeJobBlocking(jobGraph); + } + finally { transformations.clear(); - return jobExecutionResult; - } catch (Exception e) { - throw new RuntimeException(e); + miniCluster.close(); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index d2bb2cb0d..fd44dd47a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -512,10 +512,53 @@ private void extractJoinField(SqlNode condition, Set> joi } SqlKind joinKind = condition.getKind(); - if( joinKind == AND || joinKind == EQUALS ){ - extractJoinField(((SqlBasicCall)condition).operands[0], joinFieldSet); - extractJoinField(((SqlBasicCall)condition).operands[1], joinFieldSet); - }else{ + if ( AGGREGATE.contains(condition.getKind()) + || AVG_AGG_FUNCTIONS.contains(joinKind) + || COMPARISON.contains(joinKind) + || joinKind == OTHER_FUNCTION + || joinKind == DIVIDE + || joinKind == CAST + || joinKind == TRIM + || joinKind == TIMES + || joinKind == PLUS + || joinKind == NOT_IN + || joinKind == OR + || joinKind == AND + || joinKind == MINUS + || joinKind == TUMBLE + || joinKind == TUMBLE_START + || joinKind == TUMBLE_END + || joinKind == SESSION + || joinKind == SESSION_START + || joinKind == SESSION_END + || joinKind == HOP + || joinKind == HOP_START + || joinKind == HOP_END + || joinKind == BETWEEN + || joinKind == IS_NULL + || joinKind == IS_NOT_NULL + || joinKind == CONTAINS + || joinKind == TIMESTAMP_ADD + || joinKind == TIMESTAMP_DIFF + || joinKind == LIKE + || joinKind == COALESCE + || joinKind == EQUALS ){ + + SqlBasicCall sqlBasicCall = (SqlBasicCall) condition; + for(int i=0; i tuple2 = Tuple2.of(((SqlIdentifier)condition).names.get(0), ((SqlIdentifier)condition).names.get(1)); joinFieldSet.add(tuple2); @@ -822,20 +865,57 @@ public SqlBasicCall buildEmptyCondition(){ private SqlIdentifier checkAndReplaceJoinCondition(SqlNode node, Map tableMap){ SqlKind joinKind = node.getKind(); - if( joinKind == AND || joinKind == EQUALS ){ - SqlIdentifier leftNode = checkAndReplaceJoinCondition(((SqlBasicCall)node).operands[0], tableMap); - SqlIdentifier rightNode = checkAndReplaceJoinCondition(((SqlBasicCall)node).operands[1], tableMap); + if( AGGREGATE.contains(joinKind) + || AVG_AGG_FUNCTIONS.contains(joinKind) + || COMPARISON.contains(joinKind) + || joinKind == OTHER_FUNCTION + || joinKind == DIVIDE + || joinKind == CAST + || joinKind == TRIM + || joinKind == TIMES + || joinKind == PLUS + || joinKind == NOT_IN + || joinKind == OR + || joinKind == AND + || joinKind == MINUS + || joinKind == TUMBLE + || joinKind == TUMBLE_START + || joinKind == TUMBLE_END + || joinKind == SESSION + || joinKind == SESSION_START + || joinKind == SESSION_END + || joinKind == HOP + || joinKind == HOP_START + || joinKind == HOP_END + || joinKind == BETWEEN + || joinKind == IS_NULL + || joinKind == IS_NOT_NULL + || joinKind == CONTAINS + || joinKind == TIMESTAMP_ADD + || joinKind == TIMESTAMP_DIFF + || joinKind == LIKE + || joinKind == COALESCE + || joinKind == EQUALS ){ + SqlBasicCall sqlBasicCall = (SqlBasicCall) node; + for(int i=0; i Date: Wed, 27 May 2020 14:07:20 +0800 Subject: [PATCH 348/523] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=BC=BA=E5=A4=B1?= =?UTF-8?q?=E7=9A=84=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/hbase/HbaseConfigUtils.java | 14 +- .../sql/sink/hbase/HbaseOutputFormat.java | 132 ++++++++++++------ .../flink/sql/sink/hbase/HbaseSink.java | 9 ++ .../sql/sink/hbase/table/HbaseSinkParser.java | 7 + .../sql/sink/hbase/table/HbaseTableInfo.java | 20 +++ 5 files changed, 128 insertions(+), 54 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java index 2be03484b..57c63d243 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java @@ -18,19 +18,12 @@ package com.dtstack.flink.sql.sink.hbase; -import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; /** * @@ -52,13 +45,18 @@ public class HbaseConfigUtils { public final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; - public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "hbase.zookeeper.znode.parent"; + public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "zookeeper.znode.parent"; + + public final static String KEY_HBASE_CLIENT_KEYTAB_FILE = "hbase.client.keytab.file"; + public final static String KEY_HBASE_CLIENT_KERBEROS_PRINCIPAL = "hbase.client.kerberos.principal"; public static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; public static final String KEY_ZOOKEEPER_SASL_CLIENT = "zookeeper.sasl.client"; public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { + LOG.info("loginAndReturnUGI principal {}",principal); + LOG.info("loginAndReturnUGI keytab {}",keytab); if (conf == null) { throw new IllegalArgumentException("kerberos conf can not be null"); } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 97c3ad873..3b17c3551 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -29,8 +29,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Delete; @@ -70,6 +69,8 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private String regionserverPrincipal; private String securityKrb5Conf; private String zookeeperSaslClient; + private String clientPrincipal; + private String clientKeytabFile; private String[] families; private String[] qualifiers; @@ -78,48 +79,78 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private transient Connection conn; private transient Table table; + private transient ChoreService choreService; + @Override public void configure(Configuration parameters) { LOG.warn("---configure---"); conf = HBaseConfiguration.create(); - conf.set("hbase.zookeeper.quorum", host); - if (zkParent != null && !"".equals(zkParent)) { - conf.set("zookeeper.znode.parent", zkParent); - } - LOG.warn("---configure end ---"); } @Override public void open(int taskNumber, int numTasks) throws IOException { LOG.warn("---open---"); - if (kerberosAuthEnable) { - conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); - conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); - fillSyncKerberosConfig(conf, regionserverKeytabFile, regionserverPrincipal, zookeeperSaslClient, securityKrb5Conf); - - UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, regionserverPrincipal, regionserverKeytabFile); - org.apache.hadoop.conf.Configuration finalConf = conf; - conn = userGroupInformation.doAs(new PrivilegedAction() { - @Override - public Connection run() { - try { - return ConnectionFactory.createConnection(finalConf); - } catch (IOException e) { - LOG.error("Get connection fail with config:{}", finalConf); - throw new RuntimeException(e); - } - } - }); - } else { - conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); - conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); - conn = ConnectionFactory.createConnection(conf); - } + openConn(); table = conn.getTable(TableName.valueOf(tableName)); LOG.warn("---open end(get table from hbase) ---"); initMetric(); } + private void openConn(){ + try{ + if (kerberosAuthEnable) { + LOG.info("open kerberos conn"); + openKerberosConn(); + } else { + LOG.info("open conn"); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); + conn = ConnectionFactory.createConnection(conf); + } + }catch (Exception e){ + throw new RuntimeException(e); + } + + } + private void openKerberosConn() throws IOException { + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); + + LOG.info("kerberos config:{}", this.toString()); + Preconditions.checkArgument(!StringUtils.isEmpty(clientPrincipal), " clientPrincipal not null!"); + Preconditions.checkArgument(!StringUtils.isEmpty(clientKeytabFile), " clientKeytabFile not null!"); + + fillSyncKerberosConfig(conf, regionserverPrincipal, zookeeperSaslClient, securityKrb5Conf); + + clientKeytabFile = System.getProperty("user.dir") + File.separator + clientKeytabFile; + clientPrincipal = !StringUtils.isEmpty(clientPrincipal) ? clientPrincipal : regionserverPrincipal; + + conf.set(HbaseConfigUtils.KEY_HBASE_CLIENT_KEYTAB_FILE, clientKeytabFile); + conf.set(HbaseConfigUtils.KEY_HBASE_CLIENT_KERBEROS_PRINCIPAL, clientPrincipal); + + UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, clientPrincipal, clientKeytabFile); + org.apache.hadoop.conf.Configuration finalConf = conf; + conn = userGroupInformation.doAs(new PrivilegedAction() { + @Override + public Connection run() { + try { + ScheduledChore authChore = AuthUtil.getAuthChore(finalConf); + if (authChore != null) { + choreService = new ChoreService("hbaseKerberosSink"); + choreService.scheduleChore(authChore); + } + + return ConnectionFactory.createConnection(finalConf); + } catch (IOException e) { + LOG.error("Get connection fail with config:{}", finalConf); + throw new RuntimeException(e); + } + } + }); + } + + + @Override public void writeRecord(Tuple2 tuple2) { Tuple2 tupleTrans = tuple2; @@ -227,7 +258,6 @@ public void close() throws IOException { conn = null; } } - private HbaseOutputFormat() { } @@ -264,11 +294,6 @@ public HbaseOutputFormatBuilder setRowkey(String rowkey) { return this; } - public HbaseOutputFormatBuilder setUpdateMode(String updateMode) { - format.updateMode = updateMode; - return this; - } - public HbaseOutputFormatBuilder setColumnNames(String[] columnNames) { format.columnNames = columnNames; return this; @@ -283,6 +308,7 @@ public HbaseOutputFormatBuilder setColumnNameFamily(Map columnNa format.columnNameFamily = columnNameFamily; return this; } + public HbaseOutputFormatBuilder setKerberosAuthEnable(boolean kerberosAuthEnable) { format.kerberosAuthEnable = kerberosAuthEnable; return this; @@ -308,6 +334,16 @@ public HbaseOutputFormatBuilder setZookeeperSaslClient(String zookeeperSaslClien return this; } + public HbaseOutputFormatBuilder setClientPrincipal(String clientPrincipal) { + format.clientPrincipal = clientPrincipal; + return this; + } + + public HbaseOutputFormatBuilder setClientKeytabFile(String clientKeytabFile) { + format.clientKeytabFile = clientKeytabFile; + return this; + } + public HbaseOutputFormat finish() { Preconditions.checkNotNull(format.host, "zookeeperQuorum should be specified"); @@ -323,7 +359,7 @@ public HbaseOutputFormat finish() { String[] columns = keySet.toArray(new String[keySet.size()]); for (int i = 0; i < columns.length; ++i) { String col = columns[i]; - String[] part = StringUtils.split(col, ":");; + String[] part = col.split(":"); families[i] = part[0]; qualifiers[i] = part[1]; } @@ -336,16 +372,8 @@ public HbaseOutputFormat finish() { } - private void fillSyncKerberosConfig( org.apache.hadoop.conf.Configuration config, String regionserverKeytabFile, String regionserverPrincipal, + private void fillSyncKerberosConfig(org.apache.hadoop.conf.Configuration config, String regionserverPrincipal, String zookeeperSaslClient, String securityKrb5Conf) throws IOException { - if (StringUtils.isEmpty(regionserverKeytabFile)) { - throw new IllegalArgumentException("Must provide regionserverKeytabFile when authentication is Kerberos"); - } - String regionserverKeytabFilePath = System.getProperty("user.dir") + File.separator + regionserverKeytabFile; - LOG.info("regionserverKeytabFilePath:{}",regionserverKeytabFilePath); - config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KEYTAB_FILE, regionserverKeytabFilePath); - config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KEYTAB_FILE, regionserverKeytabFilePath); - if (StringUtils.isEmpty(regionserverPrincipal)) { throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); } @@ -366,5 +394,17 @@ private void fillSyncKerberosConfig( org.apache.hadoop.conf.Configuration config } } + @Override + public String toString() { + return "HbaseOutputFormat kerberos{" + + "kerberosAuthEnable=" + kerberosAuthEnable + + ", regionserverKeytabFile='" + regionserverKeytabFile + '\'' + + ", regionserverPrincipal='" + regionserverPrincipal + '\'' + + ", securityKrb5Conf='" + securityKrb5Conf + '\'' + + ", zookeeperSaslClient='" + zookeeperSaslClient + '\'' + + ", clientPrincipal='" + clientPrincipal + '\'' + + ", clientKeytabFile='" + clientKeytabFile + '\'' + + '}'; + } } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index 8364b6c10..09f5944b4 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -59,6 +59,9 @@ public class HbaseSink implements RetractStreamTableSink, IStreamSinkGener< protected String regionserverPrincipal; protected String securityKrb5Conf; protected String zookeeperSaslClient; + + private String clientPrincipal; + private String clientKeytabFile; private int parallelism = -1; @@ -83,6 +86,9 @@ public HbaseSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.securityKrb5Conf = hbaseTableInfo.getSecurityKrb5Conf(); this.zookeeperSaslClient = hbaseTableInfo.getZookeeperSaslClient(); + this.clientKeytabFile = hbaseTableInfo.getClientKeytabFile(); + this.clientPrincipal = hbaseTableInfo.getClientPrincipal(); + Integer tmpSinkParallelism = hbaseTableInfo.getParallelism(); if (tmpSinkParallelism != null) { this.parallelism = tmpSinkParallelism; @@ -104,6 +110,9 @@ public void emitDataStream(DataStream> dataStream) { builder.setSecurityKrb5Conf(securityKrb5Conf); builder.setZookeeperSaslClient(zookeeperSaslClient); + builder.setClientPrincipal(clientPrincipal); + builder.setClientKeytabFile(clientKeytabFile); + HbaseOutputFormat outputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(outputFormat); dataStream.addSink(richSinkFunction); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index 6374ddfaf..5105e0fc0 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -61,6 +61,10 @@ public class HbaseSinkParser extends AbstractTableParser { public static final String SECURITY_KRB5_CONF_KEY = "securityKrb5Conf"; public static final String ZOOKEEPER_SASL_CLINT_KEY = "zookeeperSaslClient"; + public static final String CLIENT_PRINCIPAL_KEY = "clientPrincipal"; + public static final String CLIENT_KEYTABFILE_KEY = "clientKeytabFile"; + + @Override protected boolean fieldNameNeedsUpperCase() { return false; @@ -85,6 +89,9 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map hbaseConfig = Maps.newHashMap(); public HbaseTableInfo(){ @@ -212,4 +216,20 @@ public void setZookeeperSaslClient(String zookeeperSaslClient) { this.zookeeperSaslClient = zookeeperSaslClient; } + public String getClientPrincipal() { + return clientPrincipal; + } + + public void setClientPrincipal(String clientPrincipal) { + this.clientPrincipal = clientPrincipal; + } + + public String getClientKeytabFile() { + return clientKeytabFile; + } + + public void setClientKeytabFile(String clientKeytabFile) { + this.clientKeytabFile = clientKeytabFile; + } + } From 1f48d456f5fc236c9be7fcee85e20ea6f64d970b Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 27 May 2020 14:16:25 +0800 Subject: [PATCH 349/523] =?UTF-8?q?=E5=BC=82=E6=AD=A5=E7=BB=B4=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/hbase/HbaseAsyncReqRow.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index f3a1d6691..26bf8ce28 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -98,6 +98,7 @@ public void open(Configuration parameters) throws Exception { super.open(parameters); AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; + Map hbaseConfig = hbaseSideTableInfo.getHbaseConfig(); ExecutorService executorService =new ThreadPoolExecutor(DEFAULT_POOL_SIZE, DEFAULT_POOL_SIZE, 0L, TimeUnit.MILLISECONDS, @@ -105,11 +106,18 @@ public void open(Configuration parameters) throws Exception { Config config = new Config(); config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); - config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_ASYNC, hbaseSideTableInfo.getParent()); - - if (hbaseSideTableInfo.isKerberosAuthEnable()) { - fillAsyncKerberosConfig(config, hbaseSideTableInfo); + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, hbaseSideTableInfo.getParent()); + HbaseConfigUtils.loadKrb5Conf(hbaseConfig); + hbaseConfig.entrySet().forEach(entity -> { + config.overrideConfig(entity.getKey(), (String) entity.getValue()); + }); + + if (HbaseConfigUtils.asyncOpenKerberos(hbaseConfig)) { + String jaasStr = HbaseConfigUtils.buildJaasStr(hbaseConfig); + String jaasFilePath = HbaseConfigUtils.creatJassFile(jaasStr); + config.overrideConfig(HbaseConfigUtils.KEY_JAVA_SECURITY_AUTH_LOGIN_CONF, jaasFilePath); } + hBaseClient = new HBaseClient(config, executorService); try { From 46763a77dadbb2be3e783238730d529e3524e3fd Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 27 May 2020 14:39:28 +0800 Subject: [PATCH 350/523] =?UTF-8?q?hbase=20kerberos=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/hbase/HbaseAllReqRow.java | 50 +----- .../sql/side/hbase/HbaseAsyncReqRow.java | 27 --- .../sql/side/hbase/table/HbaseSideParser.java | 18 +- .../side/hbase/table/HbaseSideTableInfo.java | 11 ++ .../side/hbase/utils/HbaseConfigUtils.java | 167 +++++++++++++++--- 5 files changed, 161 insertions(+), 112 deletions(-) diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index c50e1a806..376bccd81 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -179,22 +179,14 @@ private void loadData(Map> tmpCache) throws SQLExcep boolean openKerberos = hbaseSideTableInfo.isKerberosAuthEnable(); int loadDataCount = 0; try { - conf = HBaseConfiguration.create(); if (openKerberos) { + conf = HbaseConfigUtils.getHadoopConfiguration(hbaseSideTableInfo.getHbaseConfig()); conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); - conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_SYNC, hbaseSideTableInfo.getParent()); - - fillSyncKerberosConfig(conf,hbaseSideTableInfo); - LOG.info("hbase.security.authentication:{}", conf.get("hbase.security.authentication")); - LOG.info("hbase.security.authorization:{}", conf.get("hbase.security.authorization")); - LOG.info("hbase.master.keytab.file:{}", conf.get("hbase.master.keytab.file")); - LOG.info("hbase.master.kerberos.principal:{}", conf.get("hbase.master.kerberos.principal")); - LOG.info("hbase.regionserver.keytab.file:{}", conf.get("hbase.regionserver.keytab.file")); - LOG.info("hbase.regionserver.kerberos.principal:{}", conf.get("hbase.regionserver.kerberos.principal")); - - UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, hbaseSideTableInfo.getRegionserverPrincipal(), - hbaseSideTableInfo.getRegionserverKeytabFile()); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, hbaseSideTableInfo.getParent()); + String principal = HbaseConfigUtils.getPrincipal(hbaseSideTableInfo.getHbaseConfig()); + String keytab = HbaseConfigUtils.getKeytab(hbaseSideTableInfo.getHbaseConfig()); + UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, principal, keytab); Configuration finalConf = conf; conn = userGroupInformation.doAs(new PrivilegedAction() { @Override @@ -209,8 +201,9 @@ public Connection run() { }); } else { + conf = HbaseConfigUtils.getConfig(hbaseSideTableInfo.getHbaseConfig()); conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); - conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_SYNC, hbaseSideTableInfo.getParent()); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, hbaseSideTableInfo.getParent()); conn = ConnectionFactory.createConnection(conf); } @@ -253,33 +246,4 @@ public Connection run() { } } - private void fillSyncKerberosConfig(Configuration config, HbaseSideTableInfo hbaseSideTableInfo) throws IOException { - String regionserverKeytabFile = hbaseSideTableInfo.getRegionserverKeytabFile(); - if (StringUtils.isEmpty(regionserverKeytabFile)) { - throw new IllegalArgumentException("Must provide regionserverKeytabFile when authentication is Kerberos"); - } - String regionserverKeytabFilePath = System.getProperty("user.dir") + File.separator + regionserverKeytabFile; - LOG.info("regionserverKeytabFilePath:{}", regionserverKeytabFilePath); - config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KEYTAB_FILE, regionserverKeytabFilePath); - config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KEYTAB_FILE, regionserverKeytabFilePath); - - String regionserverPrincipal = hbaseSideTableInfo.getRegionserverPrincipal(); - if (StringUtils.isEmpty(regionserverPrincipal)) { - throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); - } - config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KERBEROS_PRINCIPAL, regionserverPrincipal); - config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL, regionserverPrincipal); - config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHORIZATION, "true"); - config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHENTICATION, "kerberos"); - - if (!StringUtils.isEmpty(hbaseSideTableInfo.getZookeeperSaslClient())) { - System.setProperty(HbaseConfigUtils.KEY_ZOOKEEPER_SASL_CLIENT, hbaseSideTableInfo.getZookeeperSaslClient()); - } - - if (!StringUtils.isEmpty(hbaseSideTableInfo.getSecurityKrb5Conf())) { - String krb5ConfPath = System.getProperty("user.dir") + File.separator + hbaseSideTableInfo.getSecurityKrb5Conf(); - LOG.info("krb5ConfPath:{}", krb5ConfPath); - System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_KRB5_CONF, krb5ConfPath); - } - } } \ No newline at end of file diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 26bf8ce28..a6bfaca7a 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -187,33 +187,6 @@ public void close() throws Exception { hBaseClient.shutdown(); } - private void fillAsyncKerberosConfig(Config config, HbaseSideTableInfo hbaseSideTableInfo) throws IOException { - AuthUtil.JAASConfig jaasConfig = HbaseConfigUtils.buildJaasConfig(hbaseSideTableInfo); - LOG.info("jaasConfig file:\n {}", jaasConfig.toString()); - String jaasFilePath = AuthUtil.creatJaasFile("JAAS", ".conf", jaasConfig); - config.overrideConfig(HbaseConfigUtils.KEY_JAVA_SECURITY_AUTH_LOGIN_CONF, jaasFilePath); - config.overrideConfig(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTH_ENABLE, "true"); - config.overrideConfig(HbaseConfigUtils.KEY_HBASE_SASL_CLIENTCONFIG, "Client"); - config.overrideConfig(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHENTICATION, "kerberos"); - - String regionserverPrincipal = hbaseSideTableInfo.getRegionserverPrincipal(); - if (StringUtils.isEmpty(regionserverPrincipal)) { - throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); - } - config.overrideConfig(HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL, regionserverPrincipal); - - if (!StringUtils.isEmpty(hbaseSideTableInfo.getZookeeperSaslClient())) { - System.setProperty(HbaseConfigUtils.KEY_ZOOKEEPER_SASL_CLIENT, hbaseSideTableInfo.getZookeeperSaslClient()); - } - - if (!StringUtils.isEmpty(hbaseSideTableInfo.getSecurityKrb5Conf())) { - String krb5ConfPath = System.getProperty("user.dir") + File.separator + hbaseSideTableInfo.getSecurityKrb5Conf(); - LOG.info("krb5ConfPath:{}", krb5ConfPath); - System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_KRB5_CONF, krb5ConfPath); - } - } - - class CheckResult{ private boolean connect; diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 832961834..03868d618 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -53,13 +53,6 @@ public class HbaseSideParser extends AbstractSideTableParser { public static final String CACHE = "cache"; - public static final String KERBEROS_AUTH_ENABLE_KEY = "kerberosAuthEnable"; - public static final String REGIONSERVER_KEYTAB_FILE_KEY = "regionserverKeytabFile"; - public static final String REGIONSERVER_PRINCIPAL_KEY = "regionserverPrincipal"; - public static final String JAAS_PRINCIPAL_KEY = "jaasPrincipal"; - public static final String SECURITY_KRB5_CONF_KEY = "securityKrb5Conf"; - public static final String ZOOKEEPER_SASL_CLINT_KEY = "zookeeperSaslClient"; - public HbaseSideParser() { addParserHandler(FIELD_KEY, FIELD_PATTERN, this::dealField); } @@ -76,13 +69,10 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map entity.getKey().contains(".")) + .map(entity -> hbaseTableInfo.getHbaseConfig().put(entity.getKey(), String.valueOf(entity.getValue()))) + .count(); return hbaseTableInfo; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java index 2a5411682..51597d583 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java @@ -50,6 +50,8 @@ public class HbaseSideTableInfo extends AbstractSideTableInfo { private Map columnNameFamily; + private Map hbaseConfig = Maps.newHashMap(); + private String tableName; private boolean kerberosAuthEnable; @@ -208,6 +210,15 @@ public void setZookeeperSaslClient(String zookeeperSaslClient) { this.zookeeperSaslClient = zookeeperSaslClient; } + public Map getHbaseConfig() { + return hbaseConfig; + } + + public void setHbaseConfig(Map hbaseConfig) { + this.hbaseConfig = hbaseConfig; + } + + @Override public void finish(){ super.finish(); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index 09f2ea53d..a7708aaae 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -18,8 +18,6 @@ package com.dtstack.flink.sql.side.hbase.utils; -import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; -import com.dtstack.flink.sql.util.AuthUtil; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -33,7 +31,6 @@ import java.io.FileWriter; import java.io.IOException; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -50,42 +47,157 @@ public class HbaseConfigUtils { private static final Logger LOG = LoggerFactory.getLogger(HbaseConfigUtils.class); // sync side kerberos - public final static String KEY_HBASE_SECURITY_AUTHENTICATION = "hbase.security.authentication"; - public final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization"; - public final static String KEY_HBASE_MASTER_KEYTAB_FILE = "hbase.master.keytab.file"; - public final static String KEY_HBASE_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal"; - public final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; - public final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; + private final static String AUTHENTICATION_TYPE = "Kerberos"; + private final static String KEY_HBASE_SECURITY_AUTHENTICATION = "hbase.security.authentication"; + private final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization"; + private final static String KEY_HBASE_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal"; + private final static String KEY_HBASE_MASTER_KEYTAB_FILE = "hbase.master.keytab.file"; + private final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; + private final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; // async side kerberos - public final static String KEY_HBASE_SECURITY_AUTH_ENABLE = "hbase.security.auth.enable"; - public final static String KEY_HBASE_SASL_CLIENTCONFIG = "hbase.sasl.clientconfig"; - public final static String KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL = "hbase.kerberos.regionserver.principal"; + private final static String KEY_HBASE_SECURITY_AUTH_ENABLE = "hbase.security.auth.enable"; + private final static String KEY_HBASE_SASL_CLIENTCONFIG = "hbase.sasl.clientconfig"; + private final static String KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL = "hbase.kerberos.regionserver.principal"; + private static final String KEY_KEY_TAB = "hbase.keytab"; + private static final String KEY_PRINCIPAL = "hbase.principal"; public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; - public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_SYNC = "zookeeper.znode.parent"; - public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_ASYNC = "hbase.zookeeper.znode.parent"; + public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "hbase.zookeeper.znode.parent"; - public static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; - public static final String KEY_ZOOKEEPER_SASL_CLIENT = "zookeeper.sasl.client"; - + private static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; public static final String KEY_JAVA_SECURITY_AUTH_LOGIN_CONF = "java.security.auth.login.config"; - public static AuthUtil.JAASConfig buildJaasConfig(HbaseSideTableInfo hbaseSideTableInfo) { - String keytabPath = System.getProperty("user.dir") + File.separator + hbaseSideTableInfo.getRegionserverKeytabFile(); - Map loginModuleOptions = new HashMap<>(); - loginModuleOptions.put("useKeyTab", "true"); - loginModuleOptions.put("useTicketCache", "false"); - loginModuleOptions.put("keyTab", "\"" + keytabPath + "\""); - loginModuleOptions.put("principal", "\"" + hbaseSideTableInfo.getJaasPrincipal() + "\""); - return AuthUtil.JAASConfig.builder().setEntryName("Client") - .setLoginModule("com.sun.security.auth.module.Krb5LoginModule") - .setLoginModuleFlag("required").setLoginModuleOptions(loginModuleOptions).build(); + private static final String SP = File.separator; + private static final String KEY_KRB5_CONF = "krb5.conf"; + + + private static List KEYS_KERBEROS_REQUIRED = Arrays.asList( + KEY_HBASE_SECURITY_AUTHENTICATION, + KEY_HBASE_MASTER_KERBEROS_PRINCIPAL, + KEY_HBASE_MASTER_KEYTAB_FILE, + KEY_HBASE_REGIONSERVER_KEYTAB_FILE, + KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL + ); + + private static List ASYNC_KEYS_KERBEROS_REQUIRED = Arrays.asList( + KEY_HBASE_SECURITY_AUTH_ENABLE, + KEY_HBASE_SASL_CLIENTCONFIG, + KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL, + KEY_HBASE_SECURITY_AUTHENTICATION, + KEY_KEY_TAB); + + + public static Configuration getConfig(Map hbaseConfigMap) { + Configuration hConfiguration = HBaseConfiguration.create(); + + for (Map.Entry entry : hbaseConfigMap.entrySet()) { + if (entry.getValue() != null && !(entry.getValue() instanceof Map)) { + hConfiguration.set(entry.getKey(), entry.getValue().toString()); + } + } + return hConfiguration; + } + + public static boolean openKerberos(Map hbaseConfigMap) { + if (!MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHORIZATION)) { + return false; + } + return AUTHENTICATION_TYPE.equalsIgnoreCase(MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION)); + } + + public static boolean asyncOpenKerberos(Map hbaseConfigMap) { + if (!MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTH_ENABLE)) { + return false; + } + return AUTHENTICATION_TYPE.equalsIgnoreCase(MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION)); } + + + public static Configuration getHadoopConfiguration(Map hbaseConfigMap) { + for (String key : KEYS_KERBEROS_REQUIRED) { + if (StringUtils.isEmpty(MapUtils.getString(hbaseConfigMap, key))) { + throw new IllegalArgumentException(String.format("Must provide [%s] when authentication is Kerberos", key)); + } + } + loadKrb5Conf(hbaseConfigMap); + + Configuration conf = new Configuration(); + if (hbaseConfigMap == null) { + return conf; + } + + hbaseConfigMap.forEach((key, val) -> { + if (val != null) { + conf.set(key, val.toString()); + } + }); + + return conf; + } + + public static String getPrincipal(Map hbaseConfigMap) { + String principal = MapUtils.getString(hbaseConfigMap, KEY_HBASE_MASTER_KERBEROS_PRINCIPAL); + if (StringUtils.isNotEmpty(principal)) { + return principal; + } + + throw new IllegalArgumentException(""); + } + + public static String getKeytab(Map hbaseConfigMap) { + String keytab = MapUtils.getString(hbaseConfigMap, KEY_HBASE_MASTER_KEYTAB_FILE); + if (StringUtils.isNotEmpty(keytab)) { + return keytab; + } + + throw new IllegalArgumentException(""); + } + + public static void loadKrb5Conf(Map kerberosConfig) { + String krb5FilePath = MapUtils.getString(kerberosConfig, KEY_JAVA_SECURITY_KRB5_CONF); + if (!org.apache.commons.lang.StringUtils.isEmpty(krb5FilePath)) { + System.setProperty(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath);; + } + } + + public static String creatJassFile(String configStr) throws IOException { + String fileName = System.getProperty("user.dir"); + File krbConf = new File(fileName); + File temp = File.createTempFile("JAAS", ".conf", krbConf); + temp.deleteOnExit(); + BufferedWriter out = new BufferedWriter(new FileWriter(temp, false)); + out.write(configStr + "\n"); + out.close(); + return temp.getAbsolutePath(); + } + + public static String buildJaasStr(Map kerberosConfig) { + for (String key : ASYNC_KEYS_KERBEROS_REQUIRED) { + if (StringUtils.isEmpty(MapUtils.getString(kerberosConfig, key))) { + throw new IllegalArgumentException(String.format("Must provide [%s] when authentication is Kerberos", key)); + } + } + + String keyTab = MapUtils.getString(kerberosConfig, KEY_KEY_TAB); + String principal = MapUtils.getString(kerberosConfig, KEY_PRINCIPAL); + + StringBuilder jaasSB = new StringBuilder("Client {\n" + + " com.sun.security.auth.module.Krb5LoginModule required\n" + + " useKeyTab=true\n" + + " useTicketCache=false\n"); + jaasSB.append(" keyTab=\"").append(keyTab).append("\"").append("\n"); + jaasSB.append(" principal=\"").append(principal).append("\"").append(";\n"); + jaasSB.append("};"); + return jaasSB.toString(); + } + + + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { if (conf == null) { throw new IllegalArgumentException("kerberos conf can not be null"); @@ -104,5 +216,4 @@ public static UserGroupInformation loginAndReturnUGI(Configuration conf, String return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab); } - } From fa0f6f2535e3bebb31c523115f31d6b29fe25a21 Mon Sep 17 00:00:00 2001 From: xuchao Date: Wed, 27 May 2020 15:12:06 +0800 Subject: [PATCH 351/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0launch=20main=20?= =?UTF-8?q?=E6=9C=AC=E5=9C=B0=E7=9A=84=E6=97=A5=E5=BF=97=E6=89=93=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- launcher/pom.xml | 6 ++++++ .../java/com/dtstack/flink/sql/launcher/LauncherMain.java | 8 ++++++++ launcher/src/main/resource/log4j.properties | 7 +++++++ 3 files changed, 21 insertions(+) create mode 100644 launcher/src/main/resource/log4j.properties diff --git a/launcher/pom.xml b/launcher/pom.xml index 1771df1ec..ea921d87a 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -29,6 +29,12 @@ 1.2.17 + + org.slf4j + slf4j-log4j12 + 1.6.1 + + com.alibaba fastjson diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 551a00533..b0b96332a 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -41,6 +41,8 @@ import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -57,6 +59,8 @@ */ public class LauncherMain { + + private static final Logger LOG = LoggerFactory.getLogger(LauncherMain.class); private static final String CORE_JAR = "core"; private static String SP = File.separator; @@ -68,9 +72,13 @@ private static String getLocalCoreJarPath(String localSqlRootJar) throws Excepti } public static void main(String[] args) throws Exception { + + LOG.info("----start----"); + if (args.length == 1 && args[0].endsWith(".json")){ args = parseJson(args); } + OptionParser optionParser = new OptionParser(args); Options launcherOptions = optionParser.getOptions(); String mode = launcherOptions.getMode(); diff --git a/launcher/src/main/resource/log4j.properties b/launcher/src/main/resource/log4j.properties new file mode 100644 index 000000000..44e3a5ef2 --- /dev/null +++ b/launcher/src/main/resource/log4j.properties @@ -0,0 +1,7 @@ +#控制台输出： +log4j.rootLogger = INFO,consoleAppender +log4j.appender.consoleAppender = org.apache.log4j.ConsoleAppender +log4j.appender.console.Target = System.out +log4j.appender.consoleAppender.layout = org.apache.log4j.PatternLayout +log4j.appender.consoleAppender.layout.ConversionPattern =%d %-5p %m %n +log4j.appender.consoleAppender.ImmediateFlush = true \ No newline at end of file From ea3ba9d46f27ca669dd7f6a1fd5fe7dcb0f7457a Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 21 May 2020 20:00:40 +0800 Subject: [PATCH 352/523] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E4=B8=AD=E6=96=87?= =?UTF-8?q?=E4=B9=B1=E5=90=97=E9=97=AE=E9=A2=98...?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/impala/ImpalaDialect.java | 16 ++++++++++++++-- .../flink/sql/sink/impala/ImpalaSink.java | 4 ++-- .../sql/sink/impala/table/ImpalaSinkParser.java | 1 + 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java index 3eaffc0ad..89fb769c3 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; import java.util.Arrays; import java.util.List; @@ -38,6 +39,12 @@ public class ImpalaDialect implements JDBCDialect { private static final String IMPALA_PARTITION_KEYWORD = "partition"; + private TypeInformation[] fieldTypes; + + public ImpalaDialect(TypeInformation[] fieldTypes){ + this.fieldTypes = fieldTypes; + } + @Override public boolean canHandle(String url) { return url.startsWith("jdbc:impala:"); @@ -70,8 +77,13 @@ public String getInsertIntoStatement(String schema, String tableName, String[] f .map(this::quoteIdentifier) .collect(Collectors.joining(", ")); - String placeholders = Arrays.stream(fieldNames) - .map(f -> "?") + String placeholders = Arrays.stream(fieldTypes) + .map(f -> { + if(String.class.getName().equals(f.getTypeClass().getName())){ + return "cast( ? as string)"; + } + return "?"; + }) .collect(Collectors.joining(", ")); String partitionFieldStr = partitionFieldsList.stream() diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index cd1e1e945..910f6f4c5 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -41,14 +41,14 @@ public class ImpalaSink extends AbstractRdbSink implements IStreamSinkGener Date: Wed, 27 May 2020 14:03:38 +0800 Subject: [PATCH 353/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=B9join?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E4=B8=AD=E9=9D=9Eequal=20=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E5=AF=B9=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../environment/MyLocalStreamEnvironment.java | 38 ++++-- .../flink/sql/side/JoinNodeDealer.java | 108 +++++++++++++++--- .../dtstack/flink/sql/util/TableUtils.java | 51 ++++++++- 3 files changed, 169 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java index 029a07f99..6aec8475e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestOptions; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.minicluster.MiniCluster; @@ -93,33 +94,48 @@ public JobExecutionResult execute(String jobName) throws Exception { // transform the streaming program into a JobGraph StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); + return execute(streamGraph); + } + + public JobExecutionResult execute(StreamGraph streamGraph) throws Exception { JobGraph jobGraph = streamGraph.getJobGraph(); jobGraph.setClasspaths(classpaths); + jobGraph.setAllowQueuedScheduling(true); Configuration configuration = new Configuration(); configuration.addAll(jobGraph.getJobConfiguration()); - - configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "512M"); - configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); + configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0"); // add (and override) the settings with what the user defined configuration.addAll(this.conf); - MiniClusterConfiguration.Builder configBuilder = new MiniClusterConfiguration.Builder(); - configBuilder.setConfiguration(configuration); + if (!configuration.contains(RestOptions.BIND_PORT)) { + configuration.setString(RestOptions.BIND_PORT, "0"); + } + + int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); + + MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder() + .setConfiguration(configuration) + .setNumSlotsPerTaskManager(numSlotsPerTaskManager) + .build(); if (LOG.isInfoEnabled()) { LOG.info("Running job on local embedded Flink mini cluster"); } - try (MiniCluster exec = new MiniCluster(configBuilder.build());) { - exec.start(); - JobExecutionResult jobExecutionResult = exec.executeJobBlocking(jobGraph); + MiniCluster miniCluster = new MiniCluster(cfg); + + try { + miniCluster.start(); + configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort()); + + return miniCluster.executeJobBlocking(jobGraph); + } + finally { transformations.clear(); - return jobExecutionResult; - } catch (Exception e) { - throw new RuntimeException(e); + miniCluster.close(); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index d2bb2cb0d..fd44dd47a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -512,10 +512,53 @@ private void extractJoinField(SqlNode condition, Set> joi } SqlKind joinKind = condition.getKind(); - if( joinKind == AND || joinKind == EQUALS ){ - extractJoinField(((SqlBasicCall)condition).operands[0], joinFieldSet); - extractJoinField(((SqlBasicCall)condition).operands[1], joinFieldSet); - }else{ + if ( AGGREGATE.contains(condition.getKind()) + || AVG_AGG_FUNCTIONS.contains(joinKind) + || COMPARISON.contains(joinKind) + || joinKind == OTHER_FUNCTION + || joinKind == DIVIDE + || joinKind == CAST + || joinKind == TRIM + || joinKind == TIMES + || joinKind == PLUS + || joinKind == NOT_IN + || joinKind == OR + || joinKind == AND + || joinKind == MINUS + || joinKind == TUMBLE + || joinKind == TUMBLE_START + || joinKind == TUMBLE_END + || joinKind == SESSION + || joinKind == SESSION_START + || joinKind == SESSION_END + || joinKind == HOP + || joinKind == HOP_START + || joinKind == HOP_END + || joinKind == BETWEEN + || joinKind == IS_NULL + || joinKind == IS_NOT_NULL + || joinKind == CONTAINS + || joinKind == TIMESTAMP_ADD + || joinKind == TIMESTAMP_DIFF + || joinKind == LIKE + || joinKind == COALESCE + || joinKind == EQUALS ){ + + SqlBasicCall sqlBasicCall = (SqlBasicCall) condition; + for(int i=0; i tuple2 = Tuple2.of(((SqlIdentifier)condition).names.get(0), ((SqlIdentifier)condition).names.get(1)); joinFieldSet.add(tuple2); @@ -822,20 +865,57 @@ public SqlBasicCall buildEmptyCondition(){ private SqlIdentifier checkAndReplaceJoinCondition(SqlNode node, Map tableMap){ SqlKind joinKind = node.getKind(); - if( joinKind == AND || joinKind == EQUALS ){ - SqlIdentifier leftNode = checkAndReplaceJoinCondition(((SqlBasicCall)node).operands[0], tableMap); - SqlIdentifier rightNode = checkAndReplaceJoinCondition(((SqlBasicCall)node).operands[1], tableMap); + if( AGGREGATE.contains(joinKind) + || AVG_AGG_FUNCTIONS.contains(joinKind) + || COMPARISON.contains(joinKind) + || joinKind == OTHER_FUNCTION + || joinKind == DIVIDE + || joinKind == CAST + || joinKind == TRIM + || joinKind == TIMES + || joinKind == PLUS + || joinKind == NOT_IN + || joinKind == OR + || joinKind == AND + || joinKind == MINUS + || joinKind == TUMBLE + || joinKind == TUMBLE_START + || joinKind == TUMBLE_END + || joinKind == SESSION + || joinKind == SESSION_START + || joinKind == SESSION_END + || joinKind == HOP + || joinKind == HOP_START + || joinKind == HOP_END + || joinKind == BETWEEN + || joinKind == IS_NULL + || joinKind == IS_NOT_NULL + || joinKind == CONTAINS + || joinKind == TIMESTAMP_ADD + || joinKind == TIMESTAMP_DIFF + || joinKind == LIKE + || joinKind == COALESCE + || joinKind == EQUALS ){ + SqlBasicCall sqlBasicCall = (SqlBasicCall) node; + for(int i=0; i Date: Wed, 27 May 2020 15:12:06 +0800 Subject: [PATCH 354/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0launch=20main=20?= =?UTF-8?q?=E6=9C=AC=E5=9C=B0=E7=9A=84=E6=97=A5=E5=BF=97=E6=89=93=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- launcher/pom.xml | 6 ++++++ .../java/com/dtstack/flink/sql/launcher/LauncherMain.java | 8 ++++++++ launcher/src/main/resource/log4j.properties | 7 +++++++ 3 files changed, 21 insertions(+) create mode 100644 launcher/src/main/resource/log4j.properties diff --git a/launcher/pom.xml b/launcher/pom.xml index 1771df1ec..ea921d87a 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -29,6 +29,12 @@ 1.2.17 + + org.slf4j + slf4j-log4j12 + 1.6.1 + + com.alibaba fastjson diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 551a00533..b0b96332a 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -41,6 +41,8 @@ import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -57,6 +59,8 @@ */ public class LauncherMain { + + private static final Logger LOG = LoggerFactory.getLogger(LauncherMain.class); private static final String CORE_JAR = "core"; private static String SP = File.separator; @@ -68,9 +72,13 @@ private static String getLocalCoreJarPath(String localSqlRootJar) throws Excepti } public static void main(String[] args) throws Exception { + + LOG.info("----start----"); + if (args.length == 1 && args[0].endsWith(".json")){ args = parseJson(args); } + OptionParser optionParser = new OptionParser(args); Options launcherOptions = optionParser.getOptions(); String mode = launcherOptions.getMode(); diff --git a/launcher/src/main/resource/log4j.properties b/launcher/src/main/resource/log4j.properties new file mode 100644 index 000000000..44e3a5ef2 --- /dev/null +++ b/launcher/src/main/resource/log4j.properties @@ -0,0 +1,7 @@ +#控制台输出： +log4j.rootLogger = INFO,consoleAppender +log4j.appender.consoleAppender = org.apache.log4j.ConsoleAppender +log4j.appender.console.Target = System.out +log4j.appender.consoleAppender.layout = org.apache.log4j.PatternLayout +log4j.appender.consoleAppender.layout.ConversionPattern =%d %-5p %m %n +log4j.appender.consoleAppender.ImmediateFlush = true \ No newline at end of file From eb0ae659caf4fef618262dcf96c6999e3123281c Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 28 May 2020 15:00:19 +0800 Subject: [PATCH 355/523] =?UTF-8?q?fix=20=E7=BA=BF=E7=A8=8B=E6=95=B0?= =?UTF-8?q?=E5=92=8C=E9=98=9F=E5=88=97=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java | 4 ++-- .../java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java | 5 +++-- .../com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java | 4 ++-- .../com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java | 4 ++-- .../com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java | 4 ++-- .../dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java | 4 ++-- .../flink/sql/side/postgresql/PostgresqlAsyncReqRow.java | 4 ++-- .../flink/sql/side/sqlserver/SqlserverAsyncReqRow.java | 4 ++-- 8 files changed, 17 insertions(+), 16 deletions(-) diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index 742f58d68..d251bf98b 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -67,8 +67,8 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("clickhouseAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("clickhouseAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index dc55bf604..e7ae0b650 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -76,8 +76,9 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, db2lientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("dbAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("dbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); + } } diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 3f2172f6d..51fe6abc5 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -83,8 +83,8 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("impalaAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("impalaAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index 4ac7e8ba5..ffdd09b51 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -76,8 +76,8 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("mysqlAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("mysqlAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index 53d6ff896..b60a794b1 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -68,7 +68,7 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("oracleAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("oracleAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 63b80ef0d..333b9e7e2 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -75,7 +75,7 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("polardbAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("polardbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index cedcb9eca..3bc745e4c 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -76,8 +76,8 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, pgClientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("postgresqlAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("postgresqlAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index e8a22dfe8..aadf2b811 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -78,7 +78,7 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("sqlServerAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("sqlServerAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } From 814584977637ab8c2a8d2e9863e440de06c8f030 Mon Sep 17 00:00:00 2001 From: xuchao Date: Thu, 28 May 2020 17:04:50 +0800 Subject: [PATCH 356/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B9=8B=E5=89=8D?= =?UTF-8?q?=E5=90=88=E5=B9=B6=E4=BB=A3=E7=A0=81=E5=86=B2=E7=AA=81=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=9A=84=E4=BB=A3=E7=A0=81=E4=B8=A2=E5=A4=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sink/kafka/AbstractKafkaProducerFactory.java | 3 --- .../DTJsonRowSerializationSchema.java | 3 ++- .../JsonCRowSerializationSchema.java | 16 ++++++++-------- 3 files changed, 10 insertions(+), 12 deletions(-) rename kafka-base/kafka-base-sink/src/main/java/{org/apache/flink/formats/json => com/dtstack/flink/sql/sink/kafka/serialization}/DTJsonRowSerializationSchema.java (98%) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java index 429b260b1..9958a2544 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java @@ -26,9 +26,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.formats.avro.AvroRowSerializationSchema; -import org.apache.flink.formats.csv.CsvRowSerializationSchema; -import org.apache.flink.formats.json.DTJsonRowSerializationSchema; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.runtime.types.CRow; diff --git a/kafka-base/kafka-base-sink/src/main/java/org/apache/flink/formats/json/DTJsonRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/DTJsonRowSerializationSchema.java similarity index 98% rename from kafka-base/kafka-base-sink/src/main/java/org/apache/flink/formats/json/DTJsonRowSerializationSchema.java rename to kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/DTJsonRowSerializationSchema.java index f67d2353d..da22dcc55 100644 --- a/kafka-base/kafka-base-sink/src/main/java/org/apache/flink/formats/json/DTJsonRowSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/DTJsonRowSerializationSchema.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.flink.formats.json; +package com.dtstack.flink.sql.sink.kafka.serialization; import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.api.common.serialization.SerializationSchema; @@ -25,6 +25,7 @@ import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.formats.json.JsonRowSchemaConverter; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java index bfe801d52..34491d367 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java @@ -159,34 +159,34 @@ private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) { } private JsonNode convert(ContainerNode container, JsonNode reuse, TypeInformation info, Object object) { - if (info == Types.VOID || object == null) { + if (info.equals(Types.VOID) || object == null) { return container.nullNode(); - } else if (info == Types.BOOLEAN) { + } else if (info.equals(Types.BOOLEAN)) { return container.booleanNode((Boolean) object); - } else if (info == Types.STRING) { + } else if (info.equals(Types.STRING)) { return container.textNode((String) object); - } else if (info == Types.BIG_DEC) { + } else if (info.equals(Types.BIG_DEC)) { // convert decimal if necessary if (object instanceof BigDecimal) { return container.numberNode((BigDecimal) object); } return container.numberNode(BigDecimal.valueOf(((Number) object).doubleValue())); - } else if (info == Types.BIG_INT) { + } else if (info.equals(Types.BIG_INT)) { // convert integer if necessary if (object instanceof BigInteger) { return container.numberNode((BigInteger) object); } return container.numberNode(BigInteger.valueOf(((Number) object).longValue())); - } else if (info == Types.SQL_DATE) { + } else if (info.equals(Types.SQL_DATE)) { return container.textNode(object.toString()); - } else if (info == Types.SQL_TIME) { + } else if (info.equals(Types.SQL_TIME)) { final Time time = (Time) object; // strip milliseconds if possible if (time.getTime() % 1000 > 0) { return container.textNode(timeFormatWithMillis.format(time)); } return container.textNode(timeFormat.format(time)); - } else if (info == Types.SQL_TIMESTAMP) { + } else if (info.equals(Types.SQL_TIMESTAMP)) { return container.textNode(timestampFormat.format((Timestamp) object)); } else if (info instanceof RowTypeInfo) { if (reuse != null && reuse instanceof ObjectNode) { From 44be6d0bcc44e160c422dbacf6ac2cb1f5e9a587 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 3 Jun 2020 12:43:58 +0800 Subject: [PATCH 357/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=A9=BA=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 3b17c3551..3f18a3a20 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -165,7 +165,7 @@ public void writeRecord(Tuple2 tuple2) { protected void dealInsert(Row record) { Put put = getPutByRow(record); - if (put == null) { + if (put.isEmpty()) { return; } From 28ae8889c12cda67769fe59be5e657d8cc364af3 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 3 Jun 2020 14:05:59 +0800 Subject: [PATCH 358/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=A9=BA=E7=BD=AE?= =?UTF-8?q?=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 3f18a3a20..acb6b5448 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -165,7 +165,7 @@ public void writeRecord(Tuple2 tuple2) { protected void dealInsert(Row record) { Put put = getPutByRow(record); - if (put.isEmpty()) { + if (put == null || put.isEmpty()) { return; } From 9371b9d6244c74c00932df023375225ba2a568df Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 3 Jun 2020 14:32:30 +0800 Subject: [PATCH 359/523] =?UTF-8?q?=E8=AE=B0=E5=BD=95=E8=84=8F=E6=95=B0?= =?UTF-8?q?=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index acb6b5448..a6a889cc4 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -166,12 +166,13 @@ public void writeRecord(Tuple2 tuple2) { protected void dealInsert(Row record) { Put put = getPutByRow(record); if (put == null || put.isEmpty()) { + outDirtyRecords.inc(); return; } try { table.put(put); - } catch (IOException e) { + } catch (Exception e) { if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed ..{}", record.toString()); LOG.error("", e); From baafc80800913455d35cb80aa5c6b32d4f005af7 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 3 Jun 2020 20:13:39 +0800 Subject: [PATCH 360/523] =?UTF-8?q?kerberos=20=E8=AF=AD=E5=8F=A5=E7=A4=BA?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/config.md | 5 +++ docs/plugin/hbaseSide.md | 80 +++++++++++++++++++++++++++++++++++++++- docs/plugin/hbaseSink.md | 64 ++++++++++++++++++++++++++++++-- 3 files changed, 145 insertions(+), 4 deletions(-) diff --git a/docs/config.md b/docs/config.md index 9aa8df994..27c4244e7 100644 --- a/docs/config.md +++ b/docs/config.md @@ -46,6 +46,11 @@ sh submit.sh -key1 val1 -key2 val2 * 描述：扩展jar路径,当前主要是UDF定义的jar； * 必选：否 * 默认值：无 + +* **addShipfile** + * 描述：扩展上传的文件，比如开启；Kerberos认证需要的keytab文件和krb5.conf文件 + * 必选：否 + * 默认值：无 * **confProp** * 描述：一些参数设置 diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index e590b02e7..29dc60bf9 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -43,7 +43,14 @@ | tableName | hbase 的表名称|是|| | cache | 维表缓存策略(NONE/LRU)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - +|kerberosAuthEnable | 是否开启kerberos认证|否|false| +|regionserverPrincipal | regionserver的principal，这个值从hbase-site.xml的hbase.regionserver.kerberos.principal属性中获取|否|| +|clientKeytabFile|client的keytab 文件|否| +|clientPrincipal|client的principal|否|| +|zookeeperSaslClient | zookeeper.sasl.client值|否|true| +|securityKrb5Conf | java.security.krb5.conf值|否|| + 另外开启Kerberos认证还需要在VM参数中配置krb5, -Djava.security.krb5.conf=/Users/xuchao/Documents/flinkSql/kerberos/krb5.conf + 同时在addShipfile参数中添加keytab文件的路径，参数具体细节请看[命令参数说明](../config.md) -------------- ## 5.样例 @@ -168,4 +175,75 @@ into sideTable b on a.id=b.rowkey1 and a.name = b.rowkey2; ``` +### kerberos维表示例 +``` +CREATE TABLE MyTable( + name varchar, + channel varchar, + pv INT, + xctime bigint +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='es_test', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' +); + +CREATE TABLE MyResult( + name varchar, + channel varchar +)WITH( + type ='mysql', + url ='jdbc:mysql://172.16.10.45:3306/test', + userName ='dtstack', + password ='abc123', + tableName ='myresult', + updateMode ='append', + parallelism ='1', + batchSize ='100', + batchWaitInterval ='1000' +); + +CREATE TABLE sideTable( + cf:name varchar as name, + cf:info varchar as info, + PRIMARY KEY(md5(name) +'test') , + PERIOD FOR SYSTEM_TIME +)WITH( + type ='hbase', + zookeeperQuorum ='172.16.10.104:2181,172.16.10.224:2181,172.16.10.252:2181', + zookeeperParent ='/hbase', + tableName ='workerinfo', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + asyncTimeoutNum ='0', + parallelism ='1', + kerberosAuthEnable='true', + regionserverPrincipal='hbase/_HOST@DTSTACK.COM', + clientKeytabFile='test.keytab', + clientPrincipal='test@DTSTACK.COM', + securityKrb5Conf='krb5.conf', +); +insert into + MyResult +select + b.name as name, + a.channel + +from + MyTable a + +join + sideTable b + +on a.channel=b.name +``` diff --git a/docs/plugin/hbaseSink.md b/docs/plugin/hbaseSink.md index ef1be339b..5006f11a2 100644 --- a/docs/plugin/hbaseSink.md +++ b/docs/plugin/hbaseSink.md @@ -37,9 +37,17 @@ hbase2.0 |rowkey | hbase的rowkey关联的列信息,多个值以逗号隔开|是|| |updateMode|APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|APPEND｜ |parallelism | 并行度设置|否|1| - - +|kerberosAuthEnable | 是否开启kerberos认证|否|false| +|regionserverPrincipal | regionserver的principal，这个值从hbase-site.xml的hbase.regionserver.kerberos.principal属性中获取|否|| +|clientKeytabFile|client的keytab 文件|否| +|clientPrincipal|client的principal|否|| +|zookeeperSaslClient | zookeeper.sasl.client值|否|true| +|securityKrb5Conf | java.security.krb5.conf值|否|| + 另外开启Kerberos认证还需要在VM参数中配置krb5, -Djava.security.krb5.conf=/Users/xuchao/Documents/flinkSql/kerberos/krb5.conf + 同时在addShipfile参数中添加keytab文件的路径，参数具体细节请看[命令参数说明](../config.md) ## 5.样例： + +### 普通结果表语句示例 ``` CREATE TABLE MyTable( name varchar, @@ -78,9 +86,59 @@ into channel, name from - MyTable a + MyTable a + + ``` +### kerberos认证结果表语句示例 +``` +CREATE TABLE MyTable( + name varchar, + channel varchar, + age int + )WITH( + type ='kafka10', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest01', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + +CREATE TABLE MyResult( + cf:name varchar , + cf:channel varchar + )WITH( + type ='hbase', + zookeeperQuorum ='cdh2.cdhsite:2181,cdh4.cdhsite:2181', + zookeeperParent ='/hbase', + tableName ='myresult', + partitionedJoin ='false', + parallelism ='1', + rowKey='name', + kerberosAuthEnable='true', + regionserverPrincipal='hbase/_HOST@DTSTACK.COM', + clientKeytabFile='test.keytab', + clientPrincipal='test@DTSTACK.COM', + securityKrb5Conf='krb5.conf', + ); + +insert +into + MyResult + select + channel, + name + from + MyTable a + +``` + ## 6.hbase数据 ### 数据内容说明 hbase的rowkey 构建规则：以描述的rowkey字段值作为key，多个字段以'-'连接 From 75f03491eea0f68a92e62ae2bbf1698e20c69f24 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 4 Jun 2020 14:14:07 +0800 Subject: [PATCH 361/523] =?UTF-8?q?fix=20=E5=8F=82=E6=95=B0=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/parser/CreateTableParser.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java index ae6e1f708..9e3a96cec 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java @@ -41,6 +41,8 @@ public class CreateTableParser implements IParser { private static final Pattern PATTERN = Pattern.compile(PATTERN_STR); + private static final Pattern PROP_PATTERN = Pattern.compile("^'\\s*(.+)\\s*'$"); + public static CreateTableParser newInstance(){ return new CreateTableParser(); } @@ -69,18 +71,27 @@ public void parseSql(String sql, SqlTree sqlTree) { } private Map parseProp(String propsStr){ - String[] strs = propsStr.trim().split("'\\s*,"); + propsStr = propsStr.replaceAll("'\\s*,", "'|"); + String[] strs = propsStr.trim().split("\\|"); Map propMap = Maps.newHashMap(); for(int i=0; i ss = DtStringUtil.splitIgnoreQuota(strs[i], '='); String key = ss.get(0).trim(); - String value = ss.get(1).trim().replaceAll("'", "").trim(); + String value = extractValue(ss.get(1)); propMap.put(key, value); } return propMap; } + private String extractValue(String value) { + Matcher matcher = PROP_PATTERN.matcher(value); + if (matcher.find()) { + return matcher.group(1); + } + throw new RuntimeException("[" + value + "] format is invalid"); + } + public static class SqlParserResult{ private String tableName; From 715bce35607eb0354a7875f90fbcbdd42453eadf Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 4 Jun 2020 15:42:44 +0800 Subject: [PATCH 362/523] fix compile fail --- core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 2320fddf3..bb8faf8ee 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -114,7 +114,7 @@ public void exec(String sql, SideSQLParser sideSQLParser = new SideSQLParser(); sideSQLParser.setLocalTableCache(localTableCache); - Queue exeQueue = sideSQLParser.getExeQueue(sql, sideTableMap.keySet()); + Queue exeQueue = sideSQLParser.getExeQueue(sql, sideTableMap.keySet(), scope); Object pollObj; //need clean From 46c7f3c8ec1b583a6629b40749602a031420f428 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 4 Jun 2020 15:45:49 +0800 Subject: [PATCH 363/523] fix compile fail --- .../main/java/com/dtstack/flink/sql/launcher/LauncherMain.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index f8d063cce..27f1c64c9 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -63,8 +63,6 @@ public class LauncherMain { private static final Logger LOG = LoggerFactory.getLogger(LauncherMain.class); private static final String CORE_JAR = "core"; - private static final Logger LOG = LoggerFactory.getLogger(LauncherMain.class); - private static String SP = File.separator; private static String getLocalCoreJarPath(String localSqlRootJar) throws Exception { From a8f16d6cb784b9d760508ab3880959951a849e1b Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 4 Jun 2020 17:40:38 +0800 Subject: [PATCH 364/523] remove blank --- .../java/com/dtstack/flink/sql/parser/CreateTableParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java index 9e3a96cec..91b1fd2ac 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java @@ -77,7 +77,7 @@ private Map parseProp(String propsStr){ for(int i=0; i ss = DtStringUtil.splitIgnoreQuota(strs[i], '='); String key = ss.get(0).trim(); - String value = extractValue(ss.get(1)); + String value = extractValue(ss.get(1).trim()); propMap.put(key, value); } From a923fff36b088b48cb79424962f4e3c153d3ba3b Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 4 Jun 2020 21:08:11 +0800 Subject: [PATCH 365/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0addshipfile=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/option/Options.java | 13 +++++++++++++ .../launcher/perjob/PerJobClusterClientBuilder.java | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/option/Options.java b/core/src/main/java/com/dtstack/flink/sql/option/Options.java index ba6296d1e..b7d425a53 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/Options.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/Options.java @@ -72,6 +72,10 @@ public class Options { @OptionRequired(description = "log level") private String logLevel = "info"; + @OptionRequired(description = "file add to ship file") + private String addShipfile; + + public String getMode() { return mode; } @@ -183,4 +187,13 @@ public String getLogLevel() { public void setLogLevel(String logLevel) { this.logLevel = logLevel; } + + public String getAddShipfile() { + return addShipfile; + } + + public void setAddShipfile(String addShipfile) { + this.addShipfile = addShipfile; + } + } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 5dc74900e..9d99f2928 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -106,6 +106,13 @@ public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJ } else { throw new RuntimeException("The Flink jar path is null"); } + // add user customized file to shipfile + if (!StringUtils.isBlank(launcherOptions.getAddShipfile())) { + List paths = ConfigParseUtil.parsePathFromStr(launcherOptions.getAddShipfile()); + paths.forEach(path -> { + shipFiles.add(new File(path)); + }); + } // classpath , all node need contain plugin jar String pluginLoadMode = launcherOptions.getPluginLoadMode(); if (StringUtils.equalsIgnoreCase(pluginLoadMode, EPluginLoadMode.CLASSPATH.name())) { From 4b462d842256fcb49e99c31f1e183715a2ba2537 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 4 Jun 2020 21:10:26 +0800 Subject: [PATCH 366/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=BC=BA=E5=A4=B1?= =?UTF-8?q?=E7=9A=84=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/launcher/perjob/ConfigParseUtil.java | 32 ++++++++----------- .../perjob/PerJobClusterClientBuilder.java | 19 ++++++----- 2 files changed, 25 insertions(+), 26 deletions(-) rename clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java => launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/ConfigParseUtil.java (55%) diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/ConfigParseUtil.java similarity index 55% rename from clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java rename to launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/ConfigParseUtil.java index 29bf54798..5acdb59a1 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/ConfigParseUtil.java @@ -16,31 +16,27 @@ * limitations under the License. */ -package com.dtstack.flink.sql.sink.clickhouse; +package com.dtstack.flink.sql.launcher.perjob; -import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import org.apache.commons.io.Charsets; -import java.util.Optional; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.util.Arrays; +import java.util.List; /** - * Date: 2020/1/15 + * Date: 2019/12/28 * Company: www.dtstack.com * @author maqi */ -public class ClickhouseDialect implements JDBCDialect { +public class ConfigParseUtil { - @Override - public boolean canHandle(String url) { - return url.startsWith("jdbc:clickhouse:"); - } - - @Override - public Optional defaultDriverName() { - return Optional.of("ru.yandex.clickhouse.ClickHouseDriver"); - } - - @Override - public String getUpdateStatement(String tableName, String[] fieldNames, String[] conditionFields) { - throw new RuntimeException("Clickhouse does not support update sql, please remove primary key or use append mode"); + public static List parsePathFromStr(String pathStr) throws UnsupportedEncodingException { + String addjarPath = URLDecoder.decode(pathStr, Charsets.UTF_8.toString()); + if (addjarPath.length() > 2) { + addjarPath = addjarPath.substring(1,addjarPath.length()-1).replace("\"",""); + } + return Arrays.asList(addjarPath.split(",")); } } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 9d99f2928..84a4e23a3 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; @@ -82,7 +83,7 @@ public void init(String yarnConfDir, Configuration flinkConfig, Properties userC } public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJarPath, Options launcherOptions, JobGraph jobGraph) - throws MalformedURLException { + throws MalformedURLException, UnsupportedEncodingException { String flinkConf = StringUtils.isEmpty(launcherOptions.getFlinkconf()) ? DEFAULT_CONF_DIR : launcherOptions.getFlinkconf(); AbstractYarnClusterDescriptor clusterDescriptor = getClusterDescriptor(flinkConfig, yarnConf, flinkConf); @@ -106,13 +107,7 @@ public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJ } else { throw new RuntimeException("The Flink jar path is null"); } - // add user customized file to shipfile - if (!StringUtils.isBlank(launcherOptions.getAddShipfile())) { - List paths = ConfigParseUtil.parsePathFromStr(launcherOptions.getAddShipfile()); - paths.forEach(path -> { - shipFiles.add(new File(path)); - }); - } + // classpath , all node need contain plugin jar String pluginLoadMode = launcherOptions.getPluginLoadMode(); if (StringUtils.equalsIgnoreCase(pluginLoadMode, EPluginLoadMode.CLASSPATH.name())) { @@ -125,6 +120,14 @@ public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJ + " Currently only classpath and shipfile are supported."); } + // add user customized file to shipfile + if (!StringUtils.isBlank(launcherOptions.getAddShipfile())) { + List paths = ConfigParseUtil.parsePathFromStr(launcherOptions.getAddShipfile()); + paths.forEach(path -> { + shipFiles.add(new File(path)); + }); + } + clusterDescriptor.addShipFiles(shipFiles); clusterDescriptor.setName(launcherOptions.getName()); String queue = launcherOptions.getQueue(); From 0b85867a644e3d1ff6990f1506466c0bcb827e5c Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 8 Jun 2020 18:04:50 +0800 Subject: [PATCH 367/523] =?UTF-8?q?redis=20=20sink=20=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/redis/RedisOutputFormat.java | 79 +++++++------------ .../flink/sql/sink/redis/enums/RedisType.java | 33 ++++++++ 2 files changed, 62 insertions(+), 50 deletions(-) create mode 100644 redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/enums/RedisType.java diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index ae4fe5a4b..ab97cf60e 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -19,6 +19,8 @@ package com.dtstack.flink.sql.sink.redis; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.sink.redis.enums.RedisType; +import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; @@ -35,11 +37,7 @@ import java.io.Closeable; import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; +import java.util.*; /** * @author yanxi @@ -49,7 +47,7 @@ public class RedisOutputFormat extends AbstractDtRichOutputFormat { private String url; - private String database; + private String database = "0"; private String tableName; @@ -71,7 +69,7 @@ public class RedisOutputFormat extends AbstractDtRichOutputFormat { protected List primaryKeys; - protected int timeout; + protected int timeout = 10000; private JedisPool pool; @@ -121,29 +119,21 @@ private void establishConnection() { String[] ipPortPair = StringUtils.split(ipPort, ":"); addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.valueOf(ipPortPair[1].trim()))); } - if (timeout == 0){ - timeout = 10000; - } - if (database == null) - { - database = "0"; - } - switch (redisType){ - //单机 - case 1: + switch (RedisType.parse(redisType)){ + case STANDALONE: pool = new JedisPool(poolConfig, firstIp, Integer.parseInt(firstPort), timeout, password, Integer.parseInt(database)); jedis = pool.getResource(); break; - //哨兵 - case 2: + case SENTINEL: jedisSentinelPool = new JedisSentinelPool(masterName, ipPorts, poolConfig, timeout, password, Integer.parseInt(database)); jedis = jedisSentinelPool.getResource(); break; - //集群 - case 3: + case CLUSTER: jedis = new JedisCluster(addresses, timeout, timeout, 10, password, poolConfig); + break; default: + throw new RuntimeException("unsupport redis type[ " + redisType + "]"); } } @@ -158,36 +148,14 @@ public void writeRecord(Tuple2 record) throws IOException { if (row.getArity() != fieldNames.length) { return; } - - HashMap map = new HashMap<>(8); - for (String primaryKey : primaryKeys) { - for (int i = 0; i < fieldNames.length; i++) { - if (fieldNames[i].equals(primaryKey)) { - map.put(primaryKey, i); - } - } - } - - List kvList = new LinkedList<>(); - for (String primaryKey : primaryKeys){ - StringBuilder primaryKv = new StringBuilder(); - int index = map.get(primaryKey).intValue(); - primaryKv.append(primaryKey).append(":").append(row.getField(index)); - kvList.add(primaryKv.toString()); - } - - String perKey = String.join(":", kvList); - for (int i = 0; i < fieldNames.length; i++) { - StringBuilder key = new StringBuilder(); - key.append(tableName).append(":").append(perKey).append(":").append(fieldNames[i]); - - String value = "null"; - Object field = row.getField(i); - if (field != null) { - value = field.toString(); - } - jedis.set(key.toString(), value); + Map refData = Maps.newHashMap(); + for(int i = 0; i < fieldNames.length; i++){ + refData.put(fieldNames[i], row.getField(i)); } + String redisKey = buildCacheKey(refData); + refData.entrySet().forEach(e ->{ + jedis.hset(redisKey, e.getKey(), String.valueOf(e.getValue())); + }); if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0){ LOG.info(record.toString()); @@ -211,6 +179,17 @@ public void close() throws IOException { } + public String buildCacheKey(Map refData) { + StringBuilder keyBuilder = new StringBuilder(tableName); + for(String primaryKey : primaryKeys){ + if(!refData.containsKey(primaryKey)){ + return null; + } + keyBuilder.append("_").append(refData.get(primaryKey)); + } + return keyBuilder.toString(); + } + public static RedisOutputFormatBuilder buildRedisOutputFormat(){ return new RedisOutputFormatBuilder(); } diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/enums/RedisType.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/enums/RedisType.java new file mode 100644 index 000000000..7a4054dfc --- /dev/null +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/enums/RedisType.java @@ -0,0 +1,33 @@ +package com.dtstack.flink.sql.sink.redis.enums; + +public enum RedisType { + /** + * 单机 + */ + STANDALONE(1), + /** + * 哨兵 + */ + SENTINEL(2), + /** + * 集群 + */ + CLUSTER(3); + int type; + RedisType(int type){ + this.type = type; + } + + public int getType(){ + return type; + } + + public static RedisType parse(int redisType){ + for(RedisType type : RedisType.values()){ + if(type.getType() == redisType){ + return type; + } + } + throw new RuntimeException("unsupport redis type["+ redisType + "]"); + } +} From 7fafc922a26c5eb5aff114bfd44c6041c07307bc Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 8 Jun 2020 19:27:55 +0800 Subject: [PATCH 368/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=BC=82=E6=AD=A5?= =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E6=83=85=E5=86=B5=E4=B8=8B=E3=80=82=E5=A6=82?= =?UTF-8?q?=E6=9E=9C=E5=AD=97=E6=AE=B5=E9=80=89=E6=8B=A9=E4=BA=86ROWTIME,?= =?UTF-8?q?=20=E6=A1=86=E6=9E=B6=E6=9C=AC=E8=BA=AB=E5=9C=A8=E8=BE=93?= =?UTF-8?q?=E5=87=BA=E7=9A=84=E6=97=B6=E5=80=99=E4=BC=9A=E5=AF=B9rowtime?= =?UTF-8?q?=E8=BF=9B=E8=A1=8C=E6=97=B6=E5=8C=BA=E5=A4=84=E7=90=86=EF=BC=8C?= =?UTF-8?q?=E5=AF=BC=E8=87=B4window=E6=97=A0=E6=B3=95=E8=A7=A6=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/cassandra/CassandraAsyncReqRow.java | 6 +++++- .../side/elasticsearch6/Elasticsearch6AsyncReqRow.java | 5 ++++- .../dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java | 6 +++++- .../dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java | 10 ++++++---- .../dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java | 10 ++++++---- .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 6 +++++- .../flink/sql/side/redis/table/RedisSideReqRow.java | 6 +++++- 7 files changed, 36 insertions(+), 13 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index ef4380c7a..5197284e1 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -60,6 +60,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.TimeZone; /** * Reason: @@ -73,6 +74,8 @@ public class CassandraAsyncReqRow extends BaseAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(CassandraAsyncReqRow.class); + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + private final static int DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE = 10; private final static int DEFAULT_VERTX_WORKER_POOL_SIZE = 20; @@ -251,7 +254,8 @@ public Row fillData(Row input, Object line) { boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index 502f23f0a..fa6d507bb 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -57,6 +57,7 @@ import java.sql.Timestamp; import java.util.List; import java.util.Map; +import java.util.TimeZone; /** * @author yinxi @@ -65,6 +66,7 @@ public class Elasticsearch6AsyncReqRow extends BaseAsyncReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AsyncReqRow.class); + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); private transient RestHighLevelClient rhlClient; private SearchRequest searchRequest; private List sqlJoinCompareOperate = Lists.newArrayList(); @@ -195,7 +197,8 @@ public Row fillData(Row input, Object line) { Object obj = input.getField(entry.getValue()); boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 56f50e27c..ed7f53e4d 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -48,6 +48,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.TimeZone; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -63,6 +64,8 @@ public class HbaseAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 2098635104857937717L; + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + private static final Logger LOG = LoggerFactory.getLogger(HbaseAsyncReqRow.class); //match to the rule of netty3 @@ -143,7 +146,8 @@ public Row fillData(Row input, Object sideInput){ boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); if(obj instanceof Timestamp && isTimeIndicatorTypeInfo){ - obj = ((Timestamp)obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index e3200c293..edb49814f 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -36,14 +36,14 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; public class KuduAsyncReqRow extends BaseAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(KuduAsyncReqRow.class); + + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + /** * 获取连接的尝试次数 */ @@ -184,6 +184,8 @@ public Row fillData(Row input, Object sideInput) { //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); } diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 0cf8158c7..f19179602 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -50,10 +50,7 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicInteger; /** @@ -63,8 +60,11 @@ * @author xuqianjin */ public class MongoAsyncReqRow extends BaseAsyncReqRow { + private static final long serialVersionUID = -1183158242862673706L; + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + private static final Logger LOG = LoggerFactory.getLogger(MongoAsyncReqRow.class); private transient MongoClient mongoClient; @@ -165,6 +165,8 @@ public Row fillData(Row input, Object line) { if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 2374e4d2a..3fb63410a 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -47,6 +47,7 @@ import java.time.Instant; import java.util.List; import java.util.Map; +import java.util.TimeZone; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -62,6 +63,8 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 2098635244857937720L; + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + private static final Logger LOG = LoggerFactory.getLogger(RdbAsyncReqRow.class); public final static int DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE = 1; @@ -211,7 +214,8 @@ public Row fillData(Row input, Object line) { Object obj = input.getField(entry.getValue()); boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java index 17557f916..634cc66fa 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java @@ -28,6 +28,7 @@ import java.sql.Date; import java.sql.Timestamp; import java.util.Map; +import java.util.TimeZone; /** * redis fill row data @@ -40,6 +41,8 @@ public class RedisSideReqRow implements ISideReqRow, Serializable { private static final long serialVersionUID = 3751171828444748982L; + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + private BaseSideInfo sideInfo; public RedisSideReqRow(BaseSideInfo sideInfo){ @@ -55,7 +58,8 @@ public Row fillData(Row input, Object sideInput) { boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); if(obj instanceof Timestamp && isTimeIndicatorTypeInfo){ - obj = ((Timestamp)obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); } From eee1867ec18dd68f8527b2180cb0592b548dae12 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 8 Jun 2020 21:27:39 +0800 Subject: [PATCH 369/523] =?UTF-8?q?=E6=96=87=E6=A1=A3=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/redisSink.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/plugin/redisSink.md b/docs/plugin/redisSink.md index 103cb4997..eb0109f06 100644 --- a/docs/plugin/redisSink.md +++ b/docs/plugin/redisSink.md @@ -94,7 +94,8 @@ redis5.0 ## 6.redis完整样例 ### redis数据说明 -redis使用k-v格式存储，key的构建格式为tableName:privateKey:privateKeyValue:columnName, value=columnValue +redis使用散列类型 hash 数据结构，key=tableName_primaryKey1_primaryKey2,value={column1=value1, column2=value2} +如果以班级class表为例，id和name作为联合主键，那么redis的结构为 ### 源表数据内容 ``` @@ -103,10 +104,10 @@ redis使用k-v格式存储，key的构建格式为tableName:privateKey:privateKe ### redis实际数据内容 ``` 127.0.0.1:6379> keys * -1) "resultTable:name:roc:name" -2) "resultTable:name:roc:channel" -127.0.0.1:6379> get "resultTable:name:roc:name" -"roc" -127.0.0.1:6379> get "resultTable:name:roc:channel" -"daishu" +1) "resultTable_roc" +127.0.0.1:6379> hgetall resultTable_roc +1) "channel" +2) "daishu" +3) "name" +4) "roc" ``` \ No newline at end of file From df063184819ee48b10068b38ddaee8b1426af066 Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 8 Jun 2020 21:37:09 +0800 Subject: [PATCH 370/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=BB=B4=E8=A1=A8All?= =?UTF-8?q?Cache=20=E6=83=85=E5=86=B5=E4=B8=8B=E3=80=82=E5=A6=82=E6=9E=9C?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E9=80=89=E6=8B=A9=E4=BA=86ROWTIME,=20?= =?UTF-8?q?=E6=A1=86=E6=9E=B6=E6=9C=AC=E8=BA=AB=E5=9C=A8=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E7=9A=84=E6=97=B6=E5=80=99=E4=BC=9A=E5=AF=B9rowtime=E8=BF=9B?= =?UTF-8?q?=E8=A1=8C=E6=97=B6=E5=8C=BA=E5=A4=84=E7=90=86=EF=BC=8C=E5=AF=BC?= =?UTF-8?q?=E8=87=B4window=E6=97=A0=E6=B3=95=E8=A7=A6=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/cassandra/CassandraAllReqRow.java | 9 ++++----- .../java/com/dtstack/flink/sql/side/BaseAllReqRow.java | 3 +++ .../sql/side/elasticsearch6/Elasticsearch6AllReqRow.java | 4 +++- .../com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java | 4 +++- .../com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java | 4 +++- .../com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java | 4 +++- .../flink/sql/side/rdb/all/AbstractRdbAllReqRow.java | 8 +++----- 7 files changed, 22 insertions(+), 14 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index 5f642d176..2fd33b4f9 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -50,10 +50,7 @@ import java.net.InetAddress; import java.sql.SQLException; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicReference; /** @@ -91,8 +88,10 @@ public Row fillData(Row input, Object sideInput) { //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } + row.setField(entry.getKey(), obj); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java index ad9dcf1bd..28f678f10 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java @@ -32,6 +32,7 @@ import org.apache.calcite.sql.JoinType; import java.sql.SQLException; +import java.util.TimeZone; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -49,6 +50,8 @@ public abstract class BaseAllReqRow extends RichFlatMapFunction impl public static final long LOAD_DATA_ERROR_SLEEP_TIME = 5_000L; + public static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + protected BaseSideInfo sideInfo; private ScheduledExecutorService es; diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index e50ed5edc..bc60eb30a 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -106,9 +106,11 @@ public Row fillData(Row input, Object sideInput) { //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } + row.setField(entry.getKey(), obj); } diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 63d26d5dd..63dc5cb29 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -88,8 +88,10 @@ public Row fillData(Row input, Object sideInput) { //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. if(obj instanceof Timestamp && isTimeIndicatorTypeInfo){ - obj = ((Timestamp)obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } + row.setField(entry.getKey(), obj); } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 31f9aa13d..cf7f0dea7 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -76,8 +76,10 @@ public Row fillData(Row input, Object sideInput) { //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } + row.setField(entry.getKey(), obj); } diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index 1857dbede..b3e1b3e9d 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -88,8 +88,10 @@ public Row fillData(Row input, Object sideInput) { //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } + row.setField(entry.getKey(), obj); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index bb88c7787..afdcac63a 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -43,10 +43,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -161,7 +158,8 @@ public Row fillData(Row input, Object sideInput) { protected Object dealTimeAttributeType(Class entry, Object obj) { boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(entry); if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } return obj; } From 3339014f1dc2f16da2fc4f28a9f79f3dce6fe58b Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 10 Jun 2020 11:15:51 +0800 Subject: [PATCH 371/523] add tidb --- tidb/pom.xml | 47 ++++++++++ tidb/tidb-side/pom.xml | 36 +++++++ tidb/tidb-side/tidb-all-side/pom.xml | 91 ++++++++++++++++++ .../flink/sql/side/tidb/TidbAllReqRow.java | 71 ++++++++++++++ .../flink/sql/side/tidb/TidbAllSideInfo.java | 43 +++++++++ tidb/tidb-side/tidb-async-side/pom.xml | 92 ++++++++++++++++++ .../flink/sql/side/tidb/TidbAsyncReqRow.java | 72 ++++++++++++++ .../sql/side/tidb/TidbAsyncSideInfo.java | 44 +++++++++ tidb/tidb-side/tidb-side-core/pom.xml | 16 ++++ .../sql/side/tidb/table/TidbSideParser.java | 41 ++++++++ tidb/tidb-sink/pom.xml | 93 +++++++++++++++++++ .../flink/sql/sink/tidb/TidbDialect.java | 75 +++++++++++++++ .../dtstack/flink/sql/sink/tidb/TidbSink.java | 58 ++++++++++++ .../sql/sink/tidb/table/TidbSinkParser.java | 41 ++++++++ 14 files changed, 820 insertions(+) create mode 100644 tidb/pom.xml create mode 100644 tidb/tidb-side/pom.xml create mode 100644 tidb/tidb-side/tidb-all-side/pom.xml create mode 100644 tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllReqRow.java create mode 100644 tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllSideInfo.java create mode 100644 tidb/tidb-side/tidb-async-side/pom.xml create mode 100644 tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java create mode 100644 tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncSideInfo.java create mode 100644 tidb/tidb-side/tidb-side-core/pom.xml create mode 100644 tidb/tidb-side/tidb-side-core/src/main/java/com/dtstack/flink/sql/side/tidb/table/TidbSideParser.java create mode 100644 tidb/tidb-sink/pom.xml create mode 100644 tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbDialect.java create mode 100644 tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbSink.java create mode 100644 tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/table/TidbSinkParser.java diff --git a/tidb/pom.xml b/tidb/pom.xml new file mode 100644 index 000000000..851acba6b --- /dev/null +++ b/tidb/pom.xml @@ -0,0 +1,47 @@ + + + + flink.sql + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.tidb + pom + + tidb-side + tidb-sink + + + + 5.1.46 + 1.0-SNAPSHOT + + + + + junit + junit + 3.8.1 + test + + + + com.dtstack.flink + sql.core + ${sql.core.version} + provided + + + + mysql + mysql-connector-java + ${tidb.connector.version} + + + + diff --git a/tidb/tidb-side/pom.xml b/tidb/tidb-side/pom.xml new file mode 100644 index 000000000..18c2b8402 --- /dev/null +++ b/tidb/tidb-side/pom.xml @@ -0,0 +1,36 @@ + + + + sql.tidb + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.side.tidb + 1.0-SNAPSHOT + + tidb-all-side + tidb-side-core + tidb-async-side + + tidb-side + pom + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.side.rdb + ${rdb.side.version} + + + + + diff --git a/tidb/tidb-side/tidb-all-side/pom.xml b/tidb/tidb-side/tidb-all-side/pom.xml new file mode 100644 index 000000000..e158fe9f5 --- /dev/null +++ b/tidb/tidb-side/tidb-all-side/pom.xml @@ -0,0 +1,91 @@ + + + + sql.side.tidb + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.side.all.tidb + tidb-all-side + jar + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.side.tidb.core + ${sql.side.tidb.core.version} + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + false + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllReqRow.java b/tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllReqRow.java new file mode 100644 index 000000000..c1529cd63 --- /dev/null +++ b/tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllReqRow.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.tidb; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.google.common.collect.Maps; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbAllReqRow extends AbstractRdbAllReqRow { + + private static final Logger LOG = LoggerFactory.getLogger(TidbAllReqRow.class); + + private static final String TIDB_DRIVER = "com.mysql.jdbc.Driver"; + + public TidbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + super(new TidbAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + } + + @Override + public Connection getConn(String dbUrl, String userName, String password) { + try { + Class.forName(TIDB_DRIVER); + Map addParams = Maps.newHashMap(); + addParams.put("userCursorFetch", "true"); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); + return DriverManager.getConnection(targetDbUrl, userName, password); + } catch (ClassNotFoundException | SQLException e) { + LOG.error("TiDB get connect error!", e); + throw new RuntimeException("TiDB get connect error! ", e); + + } + } + + @Override + public int getFetchSize() { + return Integer.MIN_VALUE; + } +} diff --git a/tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllSideInfo.java b/tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllSideInfo.java new file mode 100644 index 000000000..2a7a39076 --- /dev/null +++ b/tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllSideInfo.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.tidb; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; + +import java.util.List; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbAllSideInfo extends RdbAllSideInfo { + public TidbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } +} diff --git a/tidb/tidb-side/tidb-async-side/pom.xml b/tidb/tidb-side/tidb-async-side/pom.xml new file mode 100644 index 000000000..bdb5ec521 --- /dev/null +++ b/tidb/tidb-side/tidb-async-side/pom.xml @@ -0,0 +1,92 @@ + + + + sql.side.tidb + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.side.async.tidb + 1.0-SNAPSHOT + tidb-async-side + jar + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.side.tidb.core + ${sql.side.tidb.core.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + false + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java new file mode 100644 index 000000000..c4c712be5 --- /dev/null +++ b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.tidb; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import io.vertx.core.Vertx; +import io.vertx.core.VertxOptions; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.jdbc.JDBCClient; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; + +import java.util.List; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbAsyncReqRow extends RdbAsyncReqRow { + + private final static String TIDB_DRIVER = "com.mysql.jdbc.Driver"; + + public TidbAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + super(new TidbAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + } + + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + JsonObject tidbClientConfig = new JsonObject(); + RdbSideTableInfo sideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + tidbClientConfig.put("url", sideTableInfo.getUrl()) + .put("driver_class", TIDB_DRIVER) + .put("max_pool_size", sideTableInfo.getAsyncPoolSize()) + .put("user", sideTableInfo.getUserName()) + .put("password", sideTableInfo.getPassword()) + .put("provider_class", DT_PROVIDER_CLASS) + .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) + .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN) + .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID); + + System.setProperty("vertx.disableFileCPResolving", "true"); + + VertxOptions vo = new VertxOptions(); + vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); + vo.setWorkerPoolSize(sideTableInfo.getAsyncPoolSize()); + vo.setFileResolverCachingEnabled(false); + Vertx vertx = Vertx.vertx(vo); + setRdbSqlClient(JDBCClient.createNonShared(vertx, tidbClientConfig)); + } +} diff --git a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncSideInfo.java b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncSideInfo.java new file mode 100644 index 000000000..f672f0b83 --- /dev/null +++ b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncSideInfo.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.tidb; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; + +import java.util.List; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbAsyncSideInfo extends RdbAsyncSideInfo { + + public TidbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } +} diff --git a/tidb/tidb-side/tidb-side-core/pom.xml b/tidb/tidb-side/tidb-side-core/pom.xml new file mode 100644 index 000000000..201dc9198 --- /dev/null +++ b/tidb/tidb-side/tidb-side-core/pom.xml @@ -0,0 +1,16 @@ + + + + sql.side.tidb + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.side.tidb.core + jar + tidb-side-core + 1.0-SNAPSHOT + \ No newline at end of file diff --git a/tidb/tidb-side/tidb-side-core/src/main/java/com/dtstack/flink/sql/side/tidb/table/TidbSideParser.java b/tidb/tidb-side/tidb-side-core/src/main/java/com/dtstack/flink/sql/side/tidb/table/TidbSideParser.java new file mode 100644 index 000000000..ae8f59686 --- /dev/null +++ b/tidb/tidb-side/tidb-side-core/src/main/java/com/dtstack/flink/sql/side/tidb/table/TidbSideParser.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.tidb.table; + +import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; + +import java.util.Map; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbSideParser extends RdbSideParser { + + private static final String CURRENT_TYPE = "tidb"; + + @Override + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + tableInfo.setType(CURRENT_TYPE); + return tableInfo; + } +} diff --git a/tidb/tidb-sink/pom.xml b/tidb/tidb-sink/pom.xml new file mode 100644 index 000000000..2eaad0e99 --- /dev/null +++ b/tidb/tidb-sink/pom.xml @@ -0,0 +1,93 @@ + + + + sql.tidb + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.sink.tidb + tidb-sink + jar + 1.0-SNAPSHOT + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.sink.rdb + ${sql.sink.rdb.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + false + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbDialect.java b/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbDialect.java new file mode 100644 index 000000000..3a7770ede --- /dev/null +++ b/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbDialect.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.tidb; + +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; + +import java.util.Arrays; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbDialect implements JDBCDialect { + private static final long serialVersionUID = 1L; + + @Override + public boolean canHandle(String url) { + return url.startsWith("jdbc:mysql:"); + } + + @Override + public Optional defaultDriverName() { + return Optional.of("com.mysql.jdbc.Driver"); + } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } + + /** + * 根据ALLReplace参数，选择使用replace语句还是ON DUPLICATE KEY UPDATE 语句 + */ + @Override + public Optional getUpsertStatement(String schema, String tableName, String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + return allReplace ? buildReplaceIntoStatement(tableName, fieldNames) : buildDuplicateUpsertStatement(tableName, fieldNames); + } + + public Optional buildDuplicateUpsertStatement(String tableName, String[] fieldNames) { + String updateClause = Arrays.stream(fieldNames).map(f -> quoteIdentifier(f) + "=IFNULL(VALUES(" + quoteIdentifier(f) + ")," + quoteIdentifier(f) + ")") + .collect(Collectors.joining(", ")); + return Optional.of(getInsertIntoStatement("", tableName, fieldNames, null) + + " ON DUPLICATE KEY UPDATE " + updateClause); + } + + public Optional buildReplaceIntoStatement(String tableName, String[] fieldNames) { + String columns = Arrays.stream(fieldNames) + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")); + String placeholders = Arrays.stream(fieldNames) + .map(f -> "?") + .collect(Collectors.joining(", ")); + return Optional.of("REPLACE INTO " + quoteIdentifier(tableName) + + "(" + columns + ")" + " VALUES (" + placeholders + ")"); + } +} diff --git a/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbSink.java b/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbSink.java new file mode 100644 index 000000000..a8b031a17 --- /dev/null +++ b/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbSink.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.tidb; + +import com.dtstack.flink.sql.sink.IStreamSinkGener; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.JDBCOptions; +import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbSink extends AbstractRdbSink implements IStreamSinkGener { + + public TidbSink() { + super(new TidbDialect()); + } + + @Override + public JDBCUpsertOutputFormat getOutputFormat() { + JDBCOptions jdbcOptions = JDBCOptions.builder() + .setDbUrl(dbUrl) + .setDialect(jdbcDialect) + .setUsername(userName) + .setPassword(password) + .setTableName(tableName) + .build(); + + return JDBCUpsertOutputFormat.builder() + .setOptions(jdbcOptions) + .setFieldNames(fieldNames) + .setFlushMaxSize(batchNum) + .setFlushIntervalMills(batchWaitInterval) + .setFieldTypes(sqlTypes) + .setKeyFields(primaryKeys) + .setAllReplace(allReplace) + .setUpdateMode(updateMode) + .build(); + } +} diff --git a/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/table/TidbSinkParser.java b/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/table/TidbSinkParser.java new file mode 100644 index 000000000..d00206cc2 --- /dev/null +++ b/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/table/TidbSinkParser.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.tidb.table; + +import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; + +import java.util.Map; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbSinkParser extends RdbSinkParser { + + private static final String CURRENT_TYPE = "tidb"; + + @Override + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + tableInfo.setType(CURRENT_TYPE); + return tableInfo; + } +} From 64afbc6651202fbdb4d93f7f548cff591c07b242 Mon Sep 17 00:00:00 2001 From: gkd Date: Wed, 10 Jun 2020 16:30:43 +0800 Subject: [PATCH 372/523] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=B7=BB=E5=8A=A0che?= =?UTF-8?q?ck()=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java | 2 +- .../com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java index 4fe1ebee1..99bec0b74 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java @@ -59,7 +59,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Wed, 10 Jun 2020 20:06:26 +0800 Subject: [PATCH 373/523] =?UTF-8?q?=E7=94=A8isValid=E4=BB=A3=E6=9B=BFisClo?= =?UTF-8?q?sed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/rdb/format/JDBCUpsertOutputFormat.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index e8b6dc8a4..4e047492f 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -33,7 +33,9 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.util.List; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -44,6 +46,7 @@ /** * An upsert OutputFormat for JDBC. + * * @author maqi */ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat> { @@ -103,7 +106,7 @@ public JDBCUpsertOutputFormat( * * @param taskNumber The number of the parallel instance. * @throws IOException Thrown, if the output could not be opened due to an - * I/O problem. + * I/O problem. */ @Override public void open(int taskNumber, int numTasks) throws IOException { @@ -167,7 +170,7 @@ public synchronized void writeRecord(Tuple2 tuple2) throws IOExcep private void checkConnectionOpen() { try { - if (connection.isClosed()) { + if (!connection.isValid(10)) { LOG.info("db connection reconnect.."); establishConnection(); jdbcWriter.prepareStatement(connection); @@ -270,7 +273,8 @@ public Builder setFieldTypes(int[] fieldTypes) { } /** - * optional, partition Fields + * optional, partition Fields + * * @param partitionFields * @return */ From 4e747f5537fef5ac53e79b1d12c11302504ab299 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 11 Jun 2020 17:06:47 +0800 Subject: [PATCH 374/523] fix --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 87c8e97e6..374ec3c31 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,7 @@ db2 polardb oceanbase + tidb From c0a56ef2c41f30708f1c9565977b50aabe54f33e Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 11 Jun 2020 18:02:12 +0800 Subject: [PATCH 375/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=BC=82=E6=AD=A5?= =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java index c4c712be5..643b6368d 100644 --- a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java +++ b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.tidb; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; @@ -31,6 +32,9 @@ import org.apache.flink.configuration.Configuration; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * @author tiezhu @@ -68,5 +72,7 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, tidbClientConfig)); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("TidbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } From 512320f7604267e908999f6b2ade01e1f4685f1e Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 12 Jun 2020 15:09:52 +0800 Subject: [PATCH 376/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9fastjson=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E4=B8=BA1.2.70?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- launcher/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launcher/pom.xml b/launcher/pom.xml index ea921d87a..df9e8ae29 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -38,7 +38,7 @@ com.alibaba fastjson - 1.2.7 + 1.2.70 From 20635f032ac8d1e77df893079682df0594c6dd55 Mon Sep 17 00:00:00 2001 From: gkd Date: Mon, 15 Jun 2020 12:51:18 +0800 Subject: [PATCH 377/523] =?UTF-8?q?=E8=A7=A3=E5=86=B3kafka=20topic?= =?UTF-8?q?=E4=B8=8D=E8=AE=BE=E7=BD=AE=E6=8A=A5NPE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/pom.xml | 12 ++++++++++++ .../sql/source/kafka/table/KafkaSourceParser.java | 2 -- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index be1020b04..a5381da8d 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -122,6 +122,18 @@ junit 4.12 + + log4j + log4j + 1.2.17 + + + + org.slf4j + slf4j-simple + 1.7.30 + + diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index bef86f10d..00424127d 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -48,10 +48,8 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Mon, 15 Jun 2020 13:05:16 +0800 Subject: [PATCH 378/523] =?UTF-8?q?=E8=A7=A3=E5=86=B3kafka=20topic?= =?UTF-8?q?=E4=B8=8D=E8=AE=BE=E7=BD=AE=E6=8A=A5NPE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java | 1 - 1 file changed, 1 deletion(-) diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 00424127d..10cdf27e7 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -51,7 +51,6 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Mon, 15 Jun 2020 13:08:22 +0800 Subject: [PATCH 379/523] =?UTF-8?q?=E8=A7=A3=E5=86=B3kafka=20topic?= =?UTF-8?q?=E4=B8=8D=E8=AE=BE=E7=BD=AE=E6=8A=A5NPE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java | 1 + 1 file changed, 1 insertion(+) diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 10cdf27e7..4a84223cc 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -50,6 +50,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Tue, 16 Jun 2020 10:28:31 +0800 Subject: [PATCH 380/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=B9hadoop(?= =?UTF-8?q?=E9=9D=9EKerberos)=20=E5=86=99=E5=85=A5=20hbase=EF=BC=88?= =?UTF-8?q?=E5=BC=80=E5=90=AFkerberos=EF=BC=89=E5=85=BC=E5=AE=B9=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/pom.xml | 12 ++++++++++++ .../flink/sql/sink/hbase/HbaseConfigUtils.java | 18 ++++++++++++++++-- .../sql/sink/hbase/HbaseOutputFormat.java | 2 +- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index be1020b04..2db344e64 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -108,6 +108,18 @@ org.apache.flink flink-yarn_2.11 ${flink.version} + + + org.apache.flink + flink-shaded-hadoop2 + + + + + + org.apache.flink + flink-shaded-hadoop2 + 2.7.5-1.8.1 diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java index 57c63d243..7117644db 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java @@ -19,9 +19,13 @@ package com.dtstack.flink.sql.sink.hbase; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.HadoopKerberosName; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import sun.security.krb5.Config; +import sun.security.krb5.KrbException; import java.io.IOException; @@ -54,7 +58,11 @@ public class HbaseConfigUtils { public static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; public static final String KEY_ZOOKEEPER_SASL_CLIENT = "zookeeper.sasl.client"; - public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { + public static final String KEY_HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; + public static final String KEY_HADOOP_SECURITY_AUTH_TO_LOCAL = "hadoop.security.auth_to_local"; + public static final String KEY_HADOOP_SECURITY_AUTHORIZATION = "hadoop.security.authorization"; + + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException, KrbException { LOG.info("loginAndReturnUGI principal {}",principal); LOG.info("loginAndReturnUGI keytab {}",keytab); if (conf == null) { @@ -69,7 +77,13 @@ public static UserGroupInformation loginAndReturnUGI(Configuration conf, String throw new IllegalArgumentException("keytab can not be null"); } - conf.set("hadoop.security.authentication", "Kerberos"); + conf.set(KEY_HADOOP_SECURITY_AUTHENTICATION, "Kerberos"); + //conf.set("hadoop.security.auth_to_local", "DEFAULT"); + conf.set(KEY_HADOOP_SECURITY_AUTH_TO_LOCAL, "RULE:[1:$1] RULE:[2:$1]"); + conf.set(KEY_HADOOP_SECURITY_AUTHORIZATION, "true"); + + Config.refresh(); + KerberosName.resetDefaultRealm(); UserGroupInformation.setConfiguration(conf); return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index a6a889cc4..222d2b052 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -112,7 +112,7 @@ private void openConn(){ } } - private void openKerberosConn() throws IOException { + private void openKerberosConn() throws Exception { conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); From b50c63c00b49b5b715516e2050ad98cbb21ecc2b Mon Sep 17 00:00:00 2001 From: gkd Date: Tue, 16 Jun 2020 17:25:05 +0800 Subject: [PATCH 381/523] =?UTF-8?q?=E5=8E=BB=E6=8E=89log4j?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/pom.xml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index a5381da8d..e98800402 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -122,17 +122,6 @@ junit 4.12 - - log4j - log4j - 1.2.17 - - - - org.slf4j - slf4j-simple - 1.7.30 - From 3bb095cdfc9cd7f98474e35af322d049b3b699e1 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 17 Jun 2020 19:22:13 +0800 Subject: [PATCH 382/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20--=20=E9=97=AE?= =?UTF-8?q?=E9=A2=98=20=E5=B9=B6=E6=B7=BB=E5=8A=A0=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/parser/SqlParser.java | 2 +- .../dtstack/flink/sql/util/DtStringUtil.java | 51 +++++++++++++++++++ .../flink/sql/util/TestDtStringUtil.java | 38 ++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/com/dtstack/flink/sql/util/TestDtStringUtil.java diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index 2afc76c48..23f8e4942 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -68,7 +68,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("need to set local sql plugin root"); } - sql = sql.replaceAll("--.*", "") + sql = DtStringUtil.dealSqlComment(sql) .replaceAll("\r\n", " ") .replaceAll("\n", " ") .replace("\t", " ").trim(); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index 3106a1bd5..b4200d733 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -106,6 +106,57 @@ public static String replaceIgnoreQuota(String str, String oriStr, String replac return str.replaceAll(splitPatternStr, replaceStr); } + /** + * 处理 sql 中 "--" 注释，而不删除引号内的内容 + * + * @param sql 解析出来的 sql + * @return 返回无注释内容的 sql + */ + public static String dealSqlComment(String sql) { + boolean inQuotes = false; + boolean inSingleQuotes = false; + int bracketLeftNum = 0; + StringBuilder b = new StringBuilder(sql.length()); + char[] chars = sql.toCharArray(); + for (int index = 0; index < chars.length; index ++) { + if (index == chars.length) { + return b.toString(); + } + StringBuilder tempSb = new StringBuilder(2); + if (index > 1) { + tempSb.append(chars[index - 1]); + tempSb.append(chars[index]); + } + + if (tempSb.toString().equals("--")) { + if (inQuotes) { + b.append(chars[index]); + } else if (inSingleQuotes) { + b.append(chars[index]); + } else if (bracketLeftNum > 0) { + b.append(chars[index]); + } else { + b.deleteCharAt(b.length() - 1); + while (chars[index] != '\n') { + // 判断注释内容是不是行尾或者 sql 的最后一行 + if (index == chars.length - 1) { + break; + } + index++; + } + } + } else if (chars[index] == '\"' && '\\' != chars[index] && !inSingleQuotes) { + inQuotes = !inQuotes; + b.append(chars[index]); + } else if (chars[index] == '\'' && '\\' != chars[index] && !inQuotes) { + inSingleQuotes = !inSingleQuotes; + b.append(chars[index]); + } else { + b.append(chars[index]); + } + } + return b.toString(); + } public static String col2string(Object column, String type) { String rowData = column.toString(); diff --git a/core/src/test/java/com/dtstack/flink/sql/util/TestDtStringUtil.java b/core/src/test/java/com/dtstack/flink/sql/util/TestDtStringUtil.java new file mode 100644 index 000000000..cf3fdb90b --- /dev/null +++ b/core/src/test/java/com/dtstack/flink/sql/util/TestDtStringUtil.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.util; + +import org.junit.Assert; +import org.junit.Test; + +/** + * @author tiezhu + * Date 2020/6/17 星期三 + */ +public class TestDtStringUtil { + @Test + public void dealSqlCommentTest() { + String testSQLWithComment = "CREATE TABLE MyTable --this is a comment\n"; + Assert.assertEquals("CREATE TABLE MyTable ", DtStringUtil.dealSqlComment(testSQLWithComment)); + testSQLWithComment = "CREATE TABLE 'MyTable--' --this is a comment"; + Assert.assertEquals("CREATE TABLE 'MyTable--' ", DtStringUtil.dealSqlComment(testSQLWithComment)); + testSQLWithComment = "CREATE TABLE MyTable -- this is a '--comment'\n"; + Assert.assertEquals("CREATE TABLE MyTable ", DtStringUtil.dealSqlComment(testSQLWithComment)); + } +} From 86b625526a69023b1b938d8e9fbb85f6d5a8ed24 Mon Sep 17 00:00:00 2001 From: xuchao Date: Fri, 19 Jun 2020 13:59:55 +0800 Subject: [PATCH 383/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9sonar=20=E7=9A=84?= =?UTF-8?q?=E9=92=89=E9=92=89=E5=9C=B0=E5=9D=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ci/sonar_notify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/sonar_notify.sh b/ci/sonar_notify.sh index 41f8a3c0e..da36b8c5e 100644 --- a/ci/sonar_notify.sh +++ b/ci/sonar_notify.sh @@ -1,7 +1,7 @@ #!/bin/bash #参考钉钉文档 https://open-doc.dingtalk.com/microapp/serverapi2/qf2nxq sonarreport=$(curl -s http://172.16.100.198:8082/?projectname=dt-insight-engine/flinkStreamSQL) - curl -s "https://oapi.dingtalk.com/robot/send?access_token=71555061297a53d3ac922a6f4d94285d8e23bccdca0c00b4dc6df0a2d49da724" \ + curl -s "https://oapi.dingtalk.com/robot/send?access_token=58fd731d8bed3b17708d3aa27e49a7e2c41c7e6545f6c4be3170963a7bba7e2a" \ -H "Content-Type: application/json" \ -d "{ \"msgtype\": \"markdown\", From c931b0b39871ff391b0249170dbfd53dadfc94b3 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 19 Jun 2020 14:34:27 +0800 Subject: [PATCH 384/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9sql=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E7=9A=84=E9=BB=98=E8=AE=A4=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java | 2 +- .../com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index b60a794b1..ab3e0517b 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -56,7 +56,7 @@ public void open(Configuration parameters) throws Exception { .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) - .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) + .put("preferred_test_query", "select 1 from dual") .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 2374e4d2a..434d1b3c0 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -78,7 +78,7 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { public final static String DT_PROVIDER_CLASS = "com.dtstack.flink.sql.side.rdb.provider.DTC3P0DataSourceProvider"; - public final static String PREFERRED_TEST_QUERY_SQL = "select 1 from dual"; + public final static String PREFERRED_TEST_QUERY_SQL = "select 1"; private transient SQLClient rdbSqlClient; From a831177893c16154f7d04db625cc4a708be6f907 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 19 Jun 2020 16:47:57 +0800 Subject: [PATCH 385/523] =?UTF-8?q?=E7=BB=B4=E8=A1=A8all=E7=9A=84=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/hbase/HbaseAllReqRow.java | 8 ++++---- .../flink/sql/side/hbase/HbaseAllSideInfo.java | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 63d26d5dd..5f60ce690 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -25,6 +25,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; +import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.map.HashedMap; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -165,6 +166,7 @@ public void flatMap(CRow input, Collector out) throws Exception { private void loadData(Map> tmpCache) throws SQLException { AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + Map colRefType = ((HbaseAllSideInfo)sideInfo).getColRefType(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; Configuration conf = new Configuration(); conf.set("hbase.zookeeper.quorum", hbaseSideTableInfo.getHost()); @@ -177,14 +179,12 @@ private void loadData(Map> tmpCache) throws SQLExcep resultScanner = table.getScanner(new Scan()); for (Result r : resultScanner) { Map kv = new HashedMap(); - for (Cell cell : r.listCells()) - { + for (Cell cell : r.listCells()) { String family = Bytes.toString(CellUtil.cloneFamily(cell)); String qualifier = Bytes.toString(CellUtil.cloneQualifier(cell)); - String value = Bytes.toString(CellUtil.cloneValue(cell)); StringBuilder key = new StringBuilder(); key.append(family).append(":").append(qualifier); - + Object value = HbaseUtils.convertByte(CellUtil.cloneValue(cell), colRefType.get(key.toString())); kv.put(aliasNameInversion.get(key.toString()), value); } tmpCache.put(new String(r.getRow()), kv); diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java index 663c2927b..1b1da02ab 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java @@ -24,17 +24,22 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; +import com.google.common.collect.Maps; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import java.util.List; +import java.util.Map; public class HbaseAllSideInfo extends BaseSideInfo { private RowKeyBuilder rowKeyBuilder; + private Map colRefType; + public HbaseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -48,6 +53,14 @@ public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInf rowKeyBuilder.init(sideTableInfo.getPrimaryKeys().get(0)); + HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; + colRefType = Maps.newHashMap(); + for(int i=0; i getColRefType() { + return colRefType; + } + } \ No newline at end of file From 5693ad448a6ae5b57136b3278f605a1fdd38124a Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 19 Jun 2020 17:07:32 +0800 Subject: [PATCH 386/523] =?UTF-8?q?=E7=BB=93=E6=9E=9C=E8=A1=A8=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/hbase/HbaseOutputFormat.java | 27 +------------------ .../flink/sql/sink/hbase/HbaseUtil.java | 5 ++++ 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index a3189b0a6..e88324936 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -92,13 +92,8 @@ public void open(int taskNumber, int numTasks) throws IOException { @Override public void writeRecord(Tuple2 tuple2) { Tuple2 tupleTrans = tuple2; - Boolean retract = tupleTrans.f0; Row row = tupleTrans.f1; - if (retract) { - dealInsert(row); - } else if (!retract && StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { - dealDelete(row); - } + dealInsert(row); } protected void dealInsert(Row record) { @@ -123,26 +118,6 @@ protected void dealInsert(Row record) { outRecords.inc(); } - protected void dealDelete(Row record) { - String rowKey = buildRowKey(record); - if (!StringUtils.isEmpty(rowKey)) { - Delete delete = new Delete(Bytes.toBytes(rowKey)); - try { - table.delete(delete); - } catch (IOException e) { - if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { - LOG.error("record insert failed ..{}", record.toString()); - LOG.error("", e); - } - outDirtyRecords.inc(); - } - if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0) { - LOG.info(record.toString()); - } - outRecords.inc(); - } - } - private Put getPutByRow(Row record) { String rowKey = buildRowKey(record); if (StringUtils.isEmpty(rowKey)) { diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java index 8f1166574..6191bd177 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java @@ -64,4 +64,9 @@ public static TypeInformation columnTypeToTypeInformation(String type) { } + public static byte[] toByte(Object value){ + if() + return new byte[]; + } + } From 83e919a1697c34103aef9409ddda7b56ffb87f89 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 19 Jun 2020 21:37:46 +0800 Subject: [PATCH 387/523] =?UTF-8?q?=E8=A7=A3=E5=86=B3join=E7=9B=B8?= =?UTF-8?q?=E5=90=8C=E5=AD=97=E6=AE=B5=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/parser/InsertSqlParser.java | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java index 8775c5244..d5e6b21bc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java @@ -21,16 +21,10 @@ package com.dtstack.flink.sql.parser; import org.apache.calcite.config.Lex; -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlInsert; -import org.apache.calcite.sql.SqlJoin; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlMatchRecognize; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlOrderBy; -import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.*; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.lang3.StringUtils; import com.google.common.collect.Lists; import org.apache.flink.table.calcite.FlinkPlannerImpl; @@ -81,10 +75,21 @@ private static void parseNode(SqlNode sqlNode, SqlParseResult sqlParseResult){ parseNode(sqlSource, sqlParseResult); break; case SELECT: - SqlNode sqlFrom = ((SqlSelect)sqlNode).getFrom(); - if(sqlFrom.getKind() == IDENTIFIER){ + SqlSelect sqlSelect = (SqlSelect) sqlNode; + SqlNodeList selectList = sqlSelect.getSelectList(); + SqlNodeList sqlNodes = new SqlNodeList(selectList.getParserPosition()); + for (int index = 0; index < selectList.size(); index++) { + if (selectList.get(index).getKind().equals(SqlKind.AS)) { + sqlNodes.add(selectList.get(index)); + continue; + } + sqlNodes.add(transformToSqlBasicCall(selectList.get(index))); + } + sqlSelect.setSelectList(sqlNodes); + SqlNode sqlFrom = ((SqlSelect) sqlNode).getFrom(); + if (sqlFrom.getKind() == IDENTIFIER) { sqlParseResult.addSourceTable(sqlFrom.toString()); - }else{ + } else { parseNode(sqlFrom, sqlParseResult); } break; @@ -141,6 +146,20 @@ private static void parseNode(SqlNode sqlNode, SqlParseResult sqlParseResult){ } } + // 将 sqlNode 转换为 SqlBasicCall + public static SqlBasicCall transformToSqlBasicCall(SqlNode sqlNode) { + String asName = ""; + SqlParserPos pos = new SqlParserPos(sqlNode.getParserPosition().getLineNum(), + sqlNode.getParserPosition().getEndColumnNum()); + if (sqlNode.getKind().equals(SqlKind.IDENTIFIER)) { + asName = ((SqlIdentifier) sqlNode).names.get(1); + } + SqlNode[] operands = new SqlNode[2]; + operands[0] = sqlNode; + operands[1] = new SqlIdentifier(asName, null, pos); + return new SqlBasicCall(new SqlAsOperator(), operands, pos); + } + public static class SqlParseResult { private List sourceTableList = Lists.newArrayList(); From 1485cebca7ad34119e7e67d4d7913000a8d46f3c Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 19 Jun 2020 21:55:21 +0800 Subject: [PATCH 388/523] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E6=97=A0=E5=85=B3imp?= =?UTF-8?q?ort?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/parser/InsertSqlParser.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java index d5e6b21bc..542a1f4bb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java @@ -20,10 +20,17 @@ package com.dtstack.flink.sql.parser; -import org.apache.calcite.config.Lex; -import org.apache.calcite.sql.*; -import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.SqlInsert; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlMatchRecognize; +import org.apache.calcite.sql.SqlOrderBy; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.lang3.StringUtils; import com.google.common.collect.Lists; From eb5a0c8d2ce8e378bd5fa49e0f494a9b20cde090 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 22 Jun 2020 10:20:15 +0800 Subject: [PATCH 389/523] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=BA=90=E9=98=9F?= =?UTF-8?q?=E5=88=97=E6=BA=A2=E5=87=BA=E4=BF=9D=E6=8A=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/rdb/async/RdbAsyncReqRow.java | 58 ++++++++++--------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 2374e4d2a..518f457a6 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -121,35 +121,41 @@ private void connectWithRetry(Map inputParams, CRow input, Resul AtomicLong failCounter = new AtomicLong(0); AtomicBoolean finishFlag = new AtomicBoolean(false); while(!finishFlag.get()){ - CountDownLatch latch = new CountDownLatch(1); - rdbSqlClient.getConnection(conn -> { - try { - if(conn.failed()){ - connectionStatus.set(false); - if(failCounter.getAndIncrement() % 1000 == 0){ - LOG.error("getConnection error", conn.cause()); - } - if(failCounter.get() >= sideInfo.getSideTableInfo().getConnectRetryMaxNum(100)){ - resultFuture.completeExceptionally(conn.cause()); - finishFlag.set(true); + try{ + CountDownLatch latch = new CountDownLatch(1); + rdbSqlClient.getConnection(conn -> { + try { + if(conn.failed()){ + connectionStatus.set(false); + if(failCounter.getAndIncrement() % 1000 == 0){ + LOG.error("getConnection error", conn.cause()); + } + if(failCounter.get() >= sideInfo.getSideTableInfo().getConnectRetryMaxNum(100)){ + resultFuture.completeExceptionally(conn.cause()); + finishFlag.set(true); + } + return; } - return; + connectionStatus.set(true); + ScheduledFuture timerFuture = registerTimer(input, resultFuture); + cancelTimerWhenComplete(resultFuture, timerFuture); + handleQuery(conn.result(), inputParams, input, resultFuture); + finishFlag.set(true); + } catch (Exception e) { + dealFillDataError(input, resultFuture, e); + } finally { + latch.countDown(); } - connectionStatus.set(true); - ScheduledFuture timerFuture = registerTimer(input, resultFuture); - cancelTimerWhenComplete(resultFuture, timerFuture); - handleQuery(conn.result(), inputParams, input, resultFuture); - finishFlag.set(true); - } catch (Exception e) { - dealFillDataError(input, resultFuture, e); - } finally { - latch.countDown(); + }); + try { + latch.await(); + } catch (InterruptedException e) { + LOG.error("", e); } - }); - try { - latch.await(); - } catch (InterruptedException e) { - LOG.error("", e); + + } catch (Exception e){ + //数据源队列溢出情况 + connectionStatus.set(false); } if(!finishFlag.get()){ try { From 6a35aab88637428455882b35bc35c0add423cb46 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 22 Jun 2020 14:59:59 +0800 Subject: [PATCH 390/523] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=A0=87=E8=AF=86?= =?UTF-8?q?=E5=8F=98=E9=87=8FparentIsInsert=EF=BC=8C=E5=B9=B6=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E4=BB=A3=E7=A0=81=EF=BC=8C=E4=BF=AE=E6=94=B9=E5=91=BD?= =?UTF-8?q?=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/parser/InsertSqlParser.java | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java index 542a1f4bb..e9d8cc179 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java @@ -50,6 +50,9 @@ public class InsertSqlParser implements IParser { + // 用来标识当前解析节点的上一层节点是否为 insert 节点 + private static Boolean parentIsInsert = false; + @Override public boolean verify(String sql) { return StringUtils.isNotBlank(sql) && sql.trim().toLowerCase().startsWith("insert"); @@ -79,24 +82,19 @@ private static void parseNode(SqlNode sqlNode, SqlParseResult sqlParseResult){ SqlNode sqlTarget = ((SqlInsert)sqlNode).getTargetTable(); SqlNode sqlSource = ((SqlInsert)sqlNode).getSource(); sqlParseResult.addTargetTable(sqlTarget.toString()); + parentIsInsert = true; parseNode(sqlSource, sqlParseResult); break; case SELECT: SqlSelect sqlSelect = (SqlSelect) sqlNode; - SqlNodeList selectList = sqlSelect.getSelectList(); - SqlNodeList sqlNodes = new SqlNodeList(selectList.getParserPosition()); - for (int index = 0; index < selectList.size(); index++) { - if (selectList.get(index).getKind().equals(SqlKind.AS)) { - sqlNodes.add(selectList.get(index)); - continue; - } - sqlNodes.add(transformToSqlBasicCall(selectList.get(index))); + if (parentIsInsert) { + rebuildSelectNode(sqlSelect.getSelectList(), sqlSelect); } - sqlSelect.setSelectList(sqlNodes); SqlNode sqlFrom = ((SqlSelect) sqlNode).getFrom(); if (sqlFrom.getKind() == IDENTIFIER) { sqlParseResult.addSourceTable(sqlFrom.toString()); } else { + parentIsInsert = false; parseNode(sqlFrom, sqlParseResult); } break; @@ -153,8 +151,30 @@ private static void parseNode(SqlNode sqlNode, SqlParseResult sqlParseResult){ } } - // 将 sqlNode 转换为 SqlBasicCall - public static SqlBasicCall transformToSqlBasicCall(SqlNode sqlNode) { + /** + * 将第一层 select 中的 sqlNode 转化为 AsNode，解决字段名冲突问题 + * @param selectList select Node 的 select 字段 + * @param sqlSelect 第一层解析出来的 selectNode + */ + private static void rebuildSelectNode(SqlNodeList selectList, SqlSelect sqlSelect) { + SqlNodeList sqlNodes = new SqlNodeList(selectList.getParserPosition()); + + for (int index = 0; index < selectList.size(); index++) { + if (selectList.get(index).getKind().equals(SqlKind.AS)) { + sqlNodes.add(selectList.get(index)); + continue; + } + sqlNodes.add(transformToAsNode(selectList.get(index))); + } + sqlSelect.setSelectList(sqlNodes); + } + + /** + * 将 sqlNode 转化为 AsNode + * @param sqlNode 需要转化的 sqlNode + * @return 重新构造的 AsNode + */ + public static SqlBasicCall transformToAsNode(SqlNode sqlNode) { String asName = ""; SqlParserPos pos = new SqlParserPos(sqlNode.getParserPosition().getLineNum(), sqlNode.getParserPosition().getEndColumnNum()); From b1b56ad37bf5700c3e24d5442fdb7f103f68b338 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 22 Jun 2020 16:29:07 +0800 Subject: [PATCH 391/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E7=BA=BF=E7=A8=8B=E6=B1=A0=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 094f5a6ea..ae8c75f7e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -22,6 +22,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.enums.ECacheType; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.metric.MetricConstant; import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; @@ -68,6 +69,7 @@ public abstract class BaseAsyncReqRow extends RichAsyncFunction impl private int timeOutNum = 0; protected BaseSideInfo sideInfo; protected transient Counter parseErrorRecords; + private transient ThreadPoolExecutor cancelExecutor; public BaseAsyncReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; @@ -82,6 +84,8 @@ public void open(Configuration parameters) throws Exception { super.open(parameters); initCache(); initMetric(); + cancelExecutor = new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(100000), + new DTThreadFactory("cancel-timer-executor")); LOG.info("async dim table config info: {} ", sideInfo.getSideTableInfo().toString()); } @@ -248,12 +252,11 @@ public void onProcessingTime(long timestamp) throws Exception { } protected void cancelTimerWhenComplete(ResultFuture resultFuture, ScheduledFuture timerFuture){ - ThreadPoolExecutor executors = new ThreadPoolExecutor(1, 1,0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue()); if(resultFuture instanceof StreamRecordQueueEntry){ StreamRecordQueueEntry streamRecordBufferEntry = (StreamRecordQueueEntry) resultFuture; streamRecordBufferEntry.onComplete((Object value) -> { timerFuture.cancel(true); - },executors); + }, cancelExecutor); } } From 16398a1b1ffeac77408b775ac05552e0f2c40114 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 22 Jun 2020 20:08:48 +0800 Subject: [PATCH 392/523] =?UTF-8?q?=E7=BB=9F=E4=B8=80=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E7=BA=BF=E7=A8=8B=E6=B1=A0=EF=BC=8C=E8=B0=83=E6=95=B4=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/clickhouse/ClickhouseAsyncReqRow.java | 2 -- .../dtstack/flink/sql/side/db2/Db2AsyncReqRow.java | 3 --- .../flink/sql/side/impala/ImpalaAsyncReqRow.java | 2 -- .../flink/sql/launcher/PluginLoadModeTest.java | 2 +- .../flink/sql/side/mysql/MysqlAsyncReqRow.java | 2 -- .../flink/sql/side/oracle/OracleAsyncReqRow.java | 2 -- .../flink/sql/side/polardb/PolardbAsyncReqRow.java | 2 -- .../sql/side/postgresql/PostgresqlAsyncReqRow.java | 2 -- .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 14 ++++++++++---- .../sql/side/sqlserver/SqlserverAsyncReqRow.java | 2 -- 10 files changed, 11 insertions(+), 22 deletions(-) diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index d251bf98b..7a10bf264 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -67,8 +67,6 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("clickhouseAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index e7ae0b650..3ebc3ac8f 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -76,9 +76,6 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, db2lientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("dbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); - } } diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 51fe6abc5..3f4817580 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -83,8 +83,6 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("impalaAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } diff --git a/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java b/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java index 9c4606e98..fcfc9afce 100644 --- a/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java +++ b/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java @@ -50,7 +50,7 @@ public static void testClasspathMode() throws Exception { public static void testRocSql() throws Exception{ - String[] sql = new String[]{"-mode", "local", "-sql", "/Users/roc/Documents/flink_sql/sql/zy_sql/hbase_side.sql", "-name", "roc", + String[] sql = new String[]{"-mode", "local", "-sql", "/Users/roc/Documents/flink_sql/sql/impala_sink.sql", "-name", "roc", "-localSqlPluginPath", "/Users/roc/workspace/git_code/flinkStreamSQL/plugins", "-remoteSqlPluginPath", "/Users/roc/workspace/git_code/flinkStreamSQL/plugins", "-flinkconf", "/Users/roc/Documents/flink_sql/flinkconf", diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index ffdd09b51..c3b9a6a16 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -76,8 +76,6 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("mysqlAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index b60a794b1..e1de7b80f 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -68,7 +68,5 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("oracleAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 333b9e7e2..d29ba71b9 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -75,7 +75,5 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("polardbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index 3bc745e4c..1ded64371 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -76,8 +76,6 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, pgClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("postgresqlAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 2374e4d2a..249b47c4f 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -35,6 +35,7 @@ import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -86,6 +87,15 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private transient ThreadPoolExecutor executor; + private final static int MAX_TASK_QUEUE_SIZE = 100000; + + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + executor = new ThreadPoolExecutor(MAX_DB_CONN_POOL_SIZE_LIMIT, MAX_DB_CONN_POOL_SIZE_LIMIT, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(MAX_TASK_QUEUE_SIZE), new DTThreadFactory("rdbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy()); + } + public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); init(sideInfo); @@ -248,10 +258,6 @@ public void setRdbSqlClient(SQLClient rdbSqlClient) { this.rdbSqlClient = rdbSqlClient; } - public void setExecutor(ThreadPoolExecutor executor) { - this.executor = executor; - } - private void handleQuery(SQLConnection connection, Map inputParams, CRow input, ResultFuture resultFuture){ String key = buildCacheKey(inputParams); JsonArray params = new JsonArray(Lists.newArrayList(inputParams.values())); diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index aadf2b811..c96a4af28 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -78,7 +78,5 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("sqlServerAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } From b2d9e013baa1f6bc14d7fa1b10c76645b531cc73 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 23 Jun 2020 10:03:26 +0800 Subject: [PATCH 393/523] fix tidb --- .../java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java index 643b6368d..ab9269d5a 100644 --- a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java +++ b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java @@ -72,7 +72,5 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, tidbClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("TidbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } From 8521aecf4b86fc5562393b30452545f86bfd1377 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 23 Jun 2020 10:09:07 +0800 Subject: [PATCH 394/523] fix compile error --- .../java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java index 643b6368d..ab9269d5a 100644 --- a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java +++ b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java @@ -72,7 +72,5 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, tidbClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("TidbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } From 31f1ea14a44dc5df28823b0897609031044a15b1 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 23 Jun 2020 11:52:01 +0800 Subject: [PATCH 395/523] fix compilie --- .../java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java index 643b6368d..ab9269d5a 100644 --- a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java +++ b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java @@ -72,7 +72,5 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, tidbClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("TidbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } From 8a3d6efc5fac41a52e5994b5d8a3a3fc45774ef8 Mon Sep 17 00:00:00 2001 From: gkd Date: Tue, 23 Jun 2020 15:03:56 +0800 Subject: [PATCH 396/523] =?UTF-8?q?rdb=20float=E6=95=B0=E6=8D=AE=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E8=BD=AC=E6=8D=A2=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java | 2 +- .../com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java index a41cad5ef..3813386bc 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java @@ -106,7 +106,7 @@ public static void setField(PreparedStatement upload, int type, Object field, in break; case java.sql.Types.FLOAT: case java.sql.Types.DOUBLE: - upload.setDouble(index + 1, (double) field); + upload.setDouble(index + 1, Double.parseDouble(field.toString())); break; case java.sql.Types.DECIMAL: case java.sql.Types.NUMERIC: diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 3559d4376..7c3ff4b09 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -118,11 +118,11 @@ public void executeUpdate(Connection connection) { } catch (SQLException e1) { throw new RuntimeException(e1); } - metricOutputFormat.outDirtyRecords.inc(); if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed ,this row is {}", row.toString()); LOG.error("", e); } + metricOutputFormat.outDirtyRecords.inc(); } }); rows.clear(); From 81b35d39d7162356bb7dd61abb734fe8a538f356 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 24 Jun 2020 10:35:05 +0800 Subject: [PATCH 397/523] fix redmine-27342 --- .../flink/sql/side/rdb/all/AbstractRdbAllReqRow.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 082c546e9..23b7b19d4 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -93,7 +93,6 @@ protected void initCache() throws SQLException { protected void reloadCache() { //reload cacheRef and replace to old cacheRef Map>> newCache = Maps.newConcurrentMap(); - cacheRef.set(newCache); try { loadData(newCache); } catch (SQLException e) { @@ -123,6 +122,11 @@ public void flatMap(CRow value, Collector out) throws Exception { List> cacheList = cacheRef.get().get(cacheKey); if (CollectionUtils.isEmpty(cacheList) && sideInfo.getJoinType() == JoinType.LEFT) { out.collect(new CRow(fillData(value.row(), null), value.change())); + return; + } + + if (CollectionUtils.isEmpty(cacheList)) { + return; } cacheList.forEach(one -> out.collect(new CRow(fillData(value.row(), one), value.change()))); From 3a3bf1c909e463e45a483c6b5241f9ab5aee38ea Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 24 Jun 2020 14:58:11 +0800 Subject: [PATCH 398/523] fix redmine 27343 --- .../java/com/dtstack/flink/sql/parser/InsertSqlParser.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java index e9d8cc179..2cacd01d4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java @@ -16,7 +16,6 @@ * limitations under the License. */ - package com.dtstack.flink.sql.parser; @@ -153,6 +152,7 @@ private static void parseNode(SqlNode sqlNode, SqlParseResult sqlParseResult){ /** * 将第一层 select 中的 sqlNode 转化为 AsNode，解决字段名冲突问题 + * 仅对 table.xx 这种类型的字段进行替换 * @param selectList select Node 的 select 字段 * @param sqlSelect 第一层解析出来的 selectNode */ @@ -160,7 +160,8 @@ private static void rebuildSelectNode(SqlNodeList selectList, SqlSelect sqlSelec SqlNodeList sqlNodes = new SqlNodeList(selectList.getParserPosition()); for (int index = 0; index < selectList.size(); index++) { - if (selectList.get(index).getKind().equals(SqlKind.AS)) { + if (selectList.get(index).getKind().equals(SqlKind.AS) + || ((SqlIdentifier) selectList.get(index)).names.size() == 1) { sqlNodes.add(selectList.get(index)); continue; } From 6aee24568e832c329ece0f0d60391ccabb29a733 Mon Sep 17 00:00:00 2001 From: dapeng Date: Sun, 28 Jun 2020 15:07:22 +0800 Subject: [PATCH 399/523] hbase sink by dateType --- .../sql/sink/hbase/HbaseOutputFormat.java | 2 +- .../flink/sql/sink/hbase/HbaseUtil.java | 33 +++++++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index e88324936..2bebb8c01 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -129,7 +129,7 @@ private Put getPutByRow(Row record) { if (fieldVal == null) { continue; } - byte[] val = fieldVal.toString().getBytes(); + byte[] val = HbaseUtil.toByte(fieldVal); byte[] cf = families[i].getBytes(); byte[] qualifier = qualifiers[i].getBytes(); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java index 6191bd177..b5a39aeff 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java @@ -22,9 +22,15 @@ import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.ByteWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; +import scala.tools.nsc.Global; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.Objects; /** * Created by softfly on 17/6/30. @@ -64,9 +70,30 @@ public static TypeInformation columnTypeToTypeInformation(String type) { } - public static byte[] toByte(Object value){ - if() - return new byte[]; + public static byte[] toByte(Object value) { + if (Objects.isNull(value)) { + return new byte[]{}; + } + if (value instanceof Integer) { + return Bytes.toBytes((Integer) value); + } else if (value instanceof Boolean) { + return Bytes.toBytes((Boolean) value); + } else if (value instanceof ByteBuffer) { + return Bytes.toBytes((ByteBuffer) value); + } else if (value instanceof Double) { + return Bytes.toBytes((Double) value); + } else if (value instanceof Float) { + return Bytes.toBytes((Float) value); + } else if (value instanceof Long) { + return Bytes.toBytes((Long) value); + } else if (value instanceof Short) { + return Bytes.toBytes((Short) value); + } else if (value instanceof String) { + return Bytes.toBytes(String.valueOf(value)); + } else if (value instanceof BigDecimal) { + return Bytes.toBytes((BigDecimal) value); + } + throw new RuntimeException("unkown dateType[" + value.toString() + "]"); } } From 37d25889846d0e53654e43ab0007ddbd83d862ca Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 29 Jun 2020 09:55:14 +0800 Subject: [PATCH 400/523] fix conflict --- .../flink/sql/side/hbase/HbaseAllReqRow.java | 14 +++----------- .../flink/sql/sink/hbase/HbaseOutputFormat.java | 4 ---- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 553ffbd4b..49801a876 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -25,13 +25,12 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; -import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; +import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; +import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.map.HashedMap; -import org.apache.commons.lang.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Maps; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; @@ -39,20 +38,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; import java.security.PrivilegedAction; import java.sql.SQLException; diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index c56e73f03..6e8bc0783 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -20,7 +20,6 @@ package com.dtstack.flink.sql.sink.hbase; -import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; @@ -31,10 +30,8 @@ import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +39,6 @@ import java.io.File; import java.io.IOException; import java.security.PrivilegedAction; -import java.util.List; import java.util.Map; import java.util.Set; From 1b39d51dcf132d87948a94f0494f4537ad5bb09a Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 29 Jun 2020 10:07:13 +0800 Subject: [PATCH 401/523] valid keytab exist --- .../com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java index 57c63d243..270eb0688 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java @@ -19,10 +19,12 @@ package com.dtstack.flink.sql.sink.hbase; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; /** @@ -69,6 +71,10 @@ public static UserGroupInformation loginAndReturnUGI(Configuration conf, String throw new IllegalArgumentException("keytab can not be null"); } + if (!new File(keytab).exists()){ + throw new IllegalArgumentIOException("keytab ["+ keytab + "] not exist"); + } + conf.set("hadoop.security.authentication", "Kerberos"); UserGroupInformation.setConfiguration(conf); From 9c137766e705107a13a6ca50c0aac052d3e5d3ca Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 2 Jul 2020 21:30:08 +0800 Subject: [PATCH 402/523] =?UTF-8?q?Kerberos=E8=AE=A4=E8=AF=81=E7=9A=84hbas?= =?UTF-8?q?e=20=E7=BB=B4=E8=A1=A8=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/hbase/HbaseAsyncReqRow.java | 19 +++++++++++-------- .../side/hbase/utils/HbaseConfigUtils.java | 6 +++--- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 62524e7ed..0250decae 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -20,20 +20,16 @@ package com.dtstack.flink.sql.side.hbase; -import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.hbase.rowkeydealer.AbstractRowKeyModeDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.PreRowKeyModeDealerDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.RowKeyEqualModeDealer; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; -import com.dtstack.flink.sql.util.AuthUtil; -import com.google.common.collect.Maps; import com.stumbleupon.async.Deferred; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -42,15 +38,14 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.hbase.async.Config; import org.hbase.async.HBaseClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import sun.security.krb5.KrbException; -import java.io.File; -import java.io.IOException; import java.sql.Timestamp; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.TimeZone; @@ -118,9 +113,10 @@ public void open(Configuration parameters) throws Exception { if (HbaseConfigUtils.asyncOpenKerberos(hbaseConfig)) { String jaasStr = HbaseConfigUtils.buildJaasStr(hbaseConfig); String jaasFilePath = HbaseConfigUtils.creatJassFile(jaasStr); + System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_AUTH_LOGIN_CONF, jaasFilePath); config.overrideConfig(HbaseConfigUtils.KEY_JAVA_SECURITY_AUTH_LOGIN_CONF, jaasFilePath); + refreshConfig(); } - hBaseClient = new HBaseClient(config, executorService); try { @@ -148,6 +144,13 @@ public void open(Configuration parameters) throws Exception { } } + private void refreshConfig() throws KrbException { + sun.security.krb5.Config.refresh(); + KerberosName.resetDefaultRealm(); + //reload java.security.auth.login.config + javax.security.auth.login.Configuration.setConfiguration(null); + } + @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { rowKeyMode.asyncGetData(tableName, buildCacheKey(inputParams), input, resultFuture, sideInfo.getSideCache()); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index a7708aaae..c6399e27a 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -59,7 +59,7 @@ public class HbaseConfigUtils { private final static String KEY_HBASE_SECURITY_AUTH_ENABLE = "hbase.security.auth.enable"; private final static String KEY_HBASE_SASL_CLIENTCONFIG = "hbase.sasl.clientconfig"; private final static String KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL = "hbase.kerberos.regionserver.principal"; - private static final String KEY_KEY_TAB = "hbase.keytab"; + public static final String KEY_KEY_TAB = "hbase.keytab"; private static final String KEY_PRINCIPAL = "hbase.principal"; public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; @@ -159,7 +159,7 @@ public static String getKeytab(Map hbaseConfigMap) { } public static void loadKrb5Conf(Map kerberosConfig) { - String krb5FilePath = MapUtils.getString(kerberosConfig, KEY_JAVA_SECURITY_KRB5_CONF); + String krb5FilePath = System.getProperty("user.dir") + File.separator + MapUtils.getString(kerberosConfig, KEY_JAVA_SECURITY_KRB5_CONF); if (!org.apache.commons.lang.StringUtils.isEmpty(krb5FilePath)) { System.setProperty(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath);; } @@ -183,7 +183,7 @@ public static String buildJaasStr(Map kerberosConfig) { } } - String keyTab = MapUtils.getString(kerberosConfig, KEY_KEY_TAB); + String keyTab = System.getProperty("user.dir") + File.separator + MapUtils.getString(kerberosConfig, KEY_KEY_TAB); String principal = MapUtils.getString(kerberosConfig, KEY_PRINCIPAL); StringBuilder jaasSB = new StringBuilder("Client {\n" + From 5656872f481d1420c675a5c75a3ec6159868a746 Mon Sep 17 00:00:00 2001 From: wuren Date: Fri, 3 Jul 2020 15:54:42 +0800 Subject: [PATCH 403/523] fix create view with cep --- .gitignore | 1 + .../dtstack/flink/sql/parser/CreateTmpTableParser.java | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index a32168f1a..d7842e1ed 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ target/ *.eclipse.* *.iml plugins/ +sqlplugins/ lib/ .vertx/ .DS_Store diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java index 9ec5dcc0c..c0b55f7ee 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java @@ -22,11 +22,7 @@ import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.calcite.config.Lex; -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlJoin; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.*; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import com.google.common.collect.Lists; @@ -164,6 +160,10 @@ private static void parseNode(SqlNode sqlNode, CreateTmpTableParser.SqlParserRes parseNode(unionRight, sqlParseResult); } break; + case MATCH_RECOGNIZE: + SqlMatchRecognize node = (SqlMatchRecognize) sqlNode; + sqlParseResult.addSourceTable(node.getTableRef().toString()); + break; default: //do nothing break; From e0a10435dcb243a911c0405daebc6aa667d5119d Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 6 Jul 2020 15:43:27 +0800 Subject: [PATCH 404/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E5=90=88=E5=B9=B6=E5=87=BA=E7=8E=B0=E7=9A=84=E7=B1=BB=E4=B8=A2?= =?UTF-8?q?=E5=A4=B1=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sink/clickhouse/ClickhouseDialect.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java new file mode 100644 index 000000000..29bf54798 --- /dev/null +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.clickhouse; + +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; + +import java.util.Optional; + +/** + * Date: 2020/1/15 + * Company: www.dtstack.com + * @author maqi + */ +public class ClickhouseDialect implements JDBCDialect { + + @Override + public boolean canHandle(String url) { + return url.startsWith("jdbc:clickhouse:"); + } + + @Override + public Optional defaultDriverName() { + return Optional.of("ru.yandex.clickhouse.ClickHouseDriver"); + } + + @Override + public String getUpdateStatement(String tableName, String[] fieldNames, String[] conditionFields) { + throw new RuntimeException("Clickhouse does not support update sql, please remove primary key or use append mode"); + } +} From 603d5bcda7e817386048c057540b57007b516e2e Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 8 Jul 2020 11:32:22 +0800 Subject: [PATCH 405/523] =?UTF-8?q?=E3=80=90feat=E3=80=91tableSource=20?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=B9=20charset=20=E7=9A=84=E6=94=AF?= =?UTF-8?q?=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/exec/ExecuteProcessHelper.java | 1 - .../DtNestRowDeserializationSchema.java | 25 +++++++++++-------- .../kafka/table/KafkaSourceTableInfo.java | 24 ++++++++++++------ 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 0af1331f3..6f2fbd9ea 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -38,7 +38,6 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.constrant.ConfigConstrant; import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.enums.EPluginLoadMode; diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index cf76eb903..010a5491d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -43,7 +43,7 @@ /** * source data parse to json format - * + *

* Date: 2019/12/12 * Company: www.dtstack.com * @@ -58,9 +58,12 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final String[] fieldNames; private final TypeInformation[] fieldTypes; - private List fieldExtraInfos; + private List fieldExtraInfos; + private String charsetName; - public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { + public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, + List fieldExtraInfos, + String charsetName) { this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); this.rowAndFieldMapping = rowAndFieldMapping; @@ -99,7 +102,7 @@ public Row deserialize(byte[] message) throws IOException { } } - private void parseTree(JsonNode jsonNode, String prefix){ + private void parseTree(JsonNode jsonNode, String prefix) { if (jsonNode.isArray()) { ArrayNode array = (ArrayNode) jsonNode; for (int i = 0; i < array.size(); i++) { @@ -118,15 +121,15 @@ private void parseTree(JsonNode jsonNode, String prefix){ return; } Iterator iterator = jsonNode.fieldNames(); - while (iterator.hasNext()){ + while (iterator.hasNext()) { String next = iterator.next(); JsonNode child = jsonNode.get(next); String nodeKey = getNodeKey(prefix, next); nodeAndJsonNodeMapping.put(nodeKey, child); - if(child.isArray()){ + if (child.isArray()) { parseTree(child, nodeKey); - }else { + } else { parseTree(child, nodeKey); } } @@ -137,8 +140,8 @@ private JsonNode getIgnoreCase(String key) { return nodeAndJsonNodeMapping.get(nodeMappingKey); } - private String getNodeKey(String prefix, String nodeName){ - if(Strings.isNullOrEmpty(prefix)){ + private String getNodeKey(String prefix, String nodeName) { + if (Strings.isNullOrEmpty(prefix)) { return nodeName; } return prefix + "." + nodeName; @@ -162,7 +165,7 @@ private Object convert(JsonNode node, TypeInformation info) { } else { return node.asText(); } - } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { + } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { return Date.valueOf(node.asText()); } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { // local zone @@ -170,7 +173,7 @@ private Object convert(JsonNode node, TypeInformation info) { } else if (info.getTypeClass().equals(Types.SQL_TIMESTAMP.getTypeClass())) { // local zone return Timestamp.valueOf(node.asText()); - } else { + } else { // for types that were specified without JSON schema // e.g. POJOs try { diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index 62453166f..88e3dcfc3 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.source.kafka.table; +import com.dtstack.flink.sql.format.FormatType; import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.google.common.base.Preconditions; @@ -71,6 +72,7 @@ public class KafkaSourceTableInfo extends AbstractSourceTableInfo { public Map kafkaParams = new HashMap<>(); + public String charsetName; public String getBootstrapServers() { return bootstrapServers; @@ -148,11 +150,19 @@ public void setFieldDelimiter(String fieldDelimiter) { this.fieldDelimiter = fieldDelimiter; } - @Override - public boolean check() { - Preconditions.checkNotNull(getType(), "kafka of type is required"); - Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); - Preconditions.checkNotNull(topic, "kafka of topic is required"); - return false; - } + public String getCharsetName() { + return charsetName; + } + + public void setCharsetName(String charsetName) { + this.charsetName = charsetName; + } + + @Override + public boolean check() { + Preconditions.checkNotNull(getType(), "kafka of type is required"); + Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); + Preconditions.checkNotNull(topic, "kafka of topic is required"); + return false; + } } From 3e1f7a180a743bf181b4b8c962f30c0c80f128ad Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 8 Jul 2020 11:47:12 +0800 Subject: [PATCH 406/523] =?UTF-8?q?=E3=80=90fix=E3=80=91add=20charsetName?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/format/dtnest/DtNestRowDeserializationSchema.java | 8 ++++---- .../flink/sql/source/kafka/table/KafkaSourceParser.java | 1 + .../sql/source/kafka/table/KafkaSourceTableInfo.java | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 010a5491d..8f2658fdf 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -53,13 +53,13 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final ObjectMapper objectMapper = new ObjectMapper(); - private Map rowAndFieldMapping; - private Map nodeAndJsonNodeMapping = Maps.newHashMap(); + private final Map rowAndFieldMapping; + private final Map nodeAndJsonNodeMapping = Maps.newHashMap(); private final String[] fieldNames; private final TypeInformation[] fieldTypes; - private List fieldExtraInfos; - private String charsetName; + private final List fieldExtraInfos; + private final String charsetName; public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos, diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index bef86f10d..cf9a75ea5 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -53,6 +53,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Thu, 9 Jul 2020 14:23:48 +0800 Subject: [PATCH 407/523] =?UTF-8?q?hbase=20=E5=85=81=E8=AE=B8=E5=A1=AB?= =?UTF-8?q?=E5=85=85null=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index cddef858e..b21f5b057 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -213,10 +213,10 @@ private Put getPutByRow(Row record) { Put put = new Put(rowKey.getBytes()); for (int i = 0; i < record.getArity(); ++i) { Object fieldVal = record.getField(i); - if (fieldVal == null) { - continue; + byte[] val = null; + if (fieldVal != null) { + val = fieldVal.toString().getBytes(); } - byte[] val = fieldVal.toString().getBytes(); byte[] cf = families[i].getBytes(); byte[] qualifier = qualifiers[i].getBytes(); From 93fb59f56786569096054d56e991ee797242ee5f Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 9 Jul 2020 15:03:37 +0800 Subject: [PATCH 408/523] =?UTF-8?q?hbase=20=E5=88=97=E6=97=8F=E9=A1=BA?= =?UTF-8?q?=E5=BA=8F=E9=94=99=E4=B9=B1=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index 5105e0fc0..35fe16941 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -129,7 +129,7 @@ public void parseFieldsInfo(String fieldsInfo, HbaseTableInfo tableInfo){ } private Map parseColumnFamily(Map physicalFieldMap){ - Map columnFamiles = Maps.newHashMap(); + Map columnFamiles = new LinkedHashMap<>(); physicalFieldMap.values().forEach(x -> { String[] columnFamily = StringUtils.split(x.trim(), ":"); columnFamiles.put(x, columnFamily[1]); From 4c3cd67908c5362cb38b854a7b1118fb1deddd7d Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 9 Jul 2020 19:23:44 +0800 Subject: [PATCH 409/523] =?UTF-8?q?=E3=80=90fix=E3=80=91fix=20charset=20de?= =?UTF-8?q?code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/format/dtnest/DtNestRowDeserializationSchema.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 8f2658fdf..d6e26538f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -34,6 +34,7 @@ import org.apache.flink.types.Row; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; @@ -73,7 +74,8 @@ public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map Date: Thu, 9 Jul 2020 20:31:52 +0800 Subject: [PATCH 410/523] =?UTF-8?q?=E3=80=90fix=E3=80=91decode=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/format/dtnest/DtNestRowDeserializationSchema.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index d6e26538f..68f7a2f87 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -74,8 +74,7 @@ public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map Date: Thu, 9 Jul 2020 21:29:34 +0800 Subject: [PATCH 411/523] =?UTF-8?q?=E6=96=B0=E5=A2=9Eupdate=E8=AF=AD?= =?UTF-8?q?=E5=8F=A5=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/impala/ImpalaDialect.java | 23 ++++++++++---- .../flink/sql/sink/impala/ImpalaSink.java | 2 +- .../sql/sink/rdb/JDBCTypeConvertUtils.java | 31 ++++++++++++++++++- .../sink/rdb/writer/AbstractUpsertWriter.java | 4 ++- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java index 89fb769c3..754be49d6 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java @@ -20,13 +20,11 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.google.common.collect.Lists; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.Optional; +import java.util.*; import java.util.stream.Collectors; /** @@ -41,8 +39,11 @@ public class ImpalaDialect implements JDBCDialect { private TypeInformation[] fieldTypes; - public ImpalaDialect(TypeInformation[] fieldTypes){ + private List primaryKeys; + + public ImpalaDialect(TypeInformation[] fieldTypes, List primaryKeys){ this.fieldTypes = fieldTypes; + this.primaryKeys = primaryKeys; } @Override @@ -62,7 +63,17 @@ public String quoteIdentifier(String identifier) { @Override public String getUpdateStatement(String tableName, String[] fieldNames, String[] conditionFields) { - throw new RuntimeException("impala does not support update sql, please remove primary key or use append mode"); + //跳过primary key字段 + String setClause = Arrays.stream(fieldNames) + .filter(f -> CollectionUtils.isNotEmpty(primaryKeys) ? !primaryKeys.contains(f) : true) + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(", ")); + String conditionClause = Arrays.stream(conditionFields) + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(" AND ")); + return "UPDATE " + quoteIdentifier(tableName) + + " SET " + setClause + + " WHERE " + conditionClause; } @Override diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index 910f6f4c5..34101d410 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -48,7 +48,7 @@ public ImpalaSink() { public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() .setDbUrl(getImpalaJdbcUrl()) - .setDialect(new ImpalaDialect(getFieldTypes())) + .setDialect(new ImpalaDialect(getFieldTypes(), primaryKeys)) .setUsername(userName) .setPassword(password) .setTableName(tableName) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java index 3813386bc..ded966c68 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.rdb; +import com.google.common.collect.Lists; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +52,10 @@ public class JDBCTypeConvertUtils { * @see PreparedStatement */ public static void setRecordToStatement(PreparedStatement upload, int[] typesArray, Row row) throws SQLException { + setRecordToStatement(upload, typesArray, row, null); + } + + public static void setRecordToStatement(PreparedStatement upload, int[] typesArray, Row row, int[] pkFieldIndices) throws SQLException { if (typesArray != null && typesArray.length > 0 && typesArray.length != row.getArity()) { LOG.warn("Column SQL types array doesn't match arity of passed Row! Check the passed array..."); } @@ -62,10 +67,34 @@ public static void setRecordToStatement(PreparedStatement upload, int[] typesArr } } else { // types provided + int placeIndex = 0; for (int i = 0; i < row.getArity(); i++) { - setField(upload, typesArray[i], row.getField(i), i); + if(isPrimaryKeyField(pkFieldIndices, i)){ + continue; + } + setField(upload, typesArray[i], row.getField(i), placeIndex ++); + } + + //填充whereClause中的主键占位符 + if (pkFieldIndices != null && pkFieldIndices.length > 0) { + for (int j = 0; j < pkFieldIndices.length; j++) { + int pkIndex = pkFieldIndices[j]; + setField(upload, typesArray[pkIndex], row.getField(pkIndex), placeIndex + j); + } + } + } + } + + private static boolean isPrimaryKeyField(int[] pkFieldIndices, int fieldIndex) { + if (pkFieldIndices == null || pkFieldIndices.length <= 0) { + return false; + } + for (int index : pkFieldIndices) { + if (index == fieldIndex) { + return true; } } + return false; } public static void setField(PreparedStatement upload, int type, Object field, int index) throws SQLException { diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java index 2c070b680..42ed545a6 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -268,6 +268,7 @@ private static final class UpsertWriterUsingInsertUpdateStatement extends Abstra private final String existSql; private final String insertSql; private final String updateSql; + private final int[] pkFields; private transient PreparedStatement existStatement; private transient PreparedStatement insertStatement; @@ -287,6 +288,7 @@ private UpsertWriterUsingInsertUpdateStatement( this.existSql = existSql; this.insertSql = insertSql; this.updateSql = updateSql; + this.pkFields = pkFields; } @Override @@ -310,7 +312,7 @@ void processOneRowInBatch(Row pk, Row row) throws SQLException { resultSet.close(); if (exist) { // do update - setRecordToStatement(updateStatement, fieldTypes, row); + setRecordToStatement(updateStatement, fieldTypes, row, pkFields); updateStatement.addBatch(); } else { // do insert From 27c0f8b4428c664b65492617003cca3551a4bee2 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 13 Jul 2020 11:36:19 +0800 Subject: [PATCH 412/523] =?UTF-8?q?=E5=85=BC=E5=AE=B9primaryKey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/sink/redis/table/RedisSinkParser.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index e965eeecb..8961f7da9 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -50,11 +50,11 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map primaryKeysList = Lists.newArrayList(); if (!StringUtils.isEmpty(primaryKeysStr)) { + List primaryKeysList = Lists.newArrayList(); primaryKeysList = Arrays.asList(StringUtils.split(primaryKeysStr, ",")); + redisTableInfo.setPrimaryKeys(primaryKeysList); } - redisTableInfo.setPrimaryKeys(primaryKeysList); return redisTableInfo; } From a8d3a3276770612436ecee1377efff8b874ed9bd Mon Sep 17 00:00:00 2001 From: wuren Date: Mon, 13 Jul 2020 14:31:29 +0800 Subject: [PATCH 413/523] add array type mode --- .gitignore | 1 + .../com/dtstack/flink/sql/util/ClassUtil.java | 10 +- .../dtstack/flink/sql/util/DataTypeUtils.java | 157 ++++++++++++++++++ .../flink/sql/util/DataTypeUtilsTest.java | 54 ++++++ .../sql/source/kafka/AbstractKafkaSource.java | 13 +- 5 files changed, 232 insertions(+), 3 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java create mode 100644 core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java diff --git a/.gitignore b/.gitignore index a32168f1a..d7842e1ed 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ target/ *.eclipse.* *.iml plugins/ +sqlplugins/ lib/ .vertx/ .DS_Store diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index 4bb4ff0d8..7d092141e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.util; +import java.lang.reflect.Array; import java.math.BigDecimal; import java.sql.Date; import java.sql.Time; @@ -34,7 +35,14 @@ public class ClassUtil { public static Class stringConvertClass(String str) { - switch (str.toLowerCase()) { + + // 这部分主要是告诉Class转TypeInfomation的方法，字段是Array类型 + String lowerStr = str.toLowerCase(); + if (lowerStr.startsWith("array")) { + return Array.newInstance(Integer.class, 0).getClass(); + } + + switch (lowerStr) { case "boolean": case "bit": return Boolean.class; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java new file mode 100644 index 000000000..ba1a76fd4 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -0,0 +1,157 @@ +package com.dtstack.flink.sql.util; + +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import org.apache.flink.api.common.typeinfo.TypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.shaded.guava18.com.google.common.base.Splitter; +import org.apache.flink.table.api.Types; +import scala.reflect.macros.TypecheckException; + +import java.util.ArrayList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @program: flink.sql + * @author: wuren + * @create: 2020/07/09 + **/ +public class DataTypeUtils { + + private final static Pattern COMPOUND_TYPE_PATTERN = Pattern.compile("(.+?)<(.+)>"); + private final static String ARRAY = "ARRAY"; + private final static String ROW = "ROW"; + private final static char FIELD_DELIMITER = ','; + private final static char TYPE_DELIMITER = ' '; + + private DataTypeUtils() {} + + public static TypeInformation convertToCollectionType(String string) { + Matcher matcher = COMPOUND_TYPE_PATTERN.matcher(string); + // TODO 现在只支持ARRAY类型后续可以加入 MAP等类型 + if (matcher.find() && ARRAY.equals(matcher.group(1))) { + return convertToArray(string); + } else { + throw new RuntimeException(""); + } + } + + /** + * 目前ARRAY里只支持ROW和其他基本类型 + * @param arrayTypeString + * @return + */ + public static TypeInformation convertToArray(String arrayTypeString) { + Matcher matcher = COMPOUND_TYPE_PATTERN.matcher(arrayTypeString); + if (matcher.find() && ARRAY.equals(matcher.group(1))) { + String elementTypeString = matcher.group(2); + TypeInformation elementType; + if (elementTypeString.toUpperCase().startsWith(ROW)) { + elementType = convertToRow(elementTypeString); + } else { + elementType = convertToAtomicType(elementTypeString); + } + return Types.OBJECT_ARRAY(elementType); + } else { + throw new RuntimeException("convert to array type error!"); + } + + } + + /** + * 目前ROW里只支持基本类型 + * @param string + */ + public static RowTypeInfo convertToRow(String string) { + Matcher matcher = COMPOUND_TYPE_PATTERN.matcher(string); + + if (matcher.find() && + ROW.equals( + matcher.group(1).toUpperCase() + ) + ) { + String elementTypeStr = matcher.group(2); + Iterable typeInfo = splitCompositeTypeFields(elementTypeStr); + Tuple2 tuple = genFieldInfo(typeInfo); + return new RowTypeInfo(tuple.f0, tuple.f1); + } else { + throw new RuntimeException(""); + } + } + + private static Iterable splitCompositeTypeFields(String string) { + return Splitter + .on(FIELD_DELIMITER) + .trimResults() + .split(string); + } + + private static Tuple2 genFieldInfo(Iterable typeInfo) { + int fieldsSize = Iterators.size(typeInfo.iterator()); + ArrayList types = Lists.newArrayList(); + ArrayList fieldNames = Lists.newArrayList(); + for (String type : typeInfo) { + Iterable fieldInfo = Splitter + .on(TYPE_DELIMITER) + .trimResults() + .omitEmptyStrings() + .split(type); + ArrayList array = Lists.newArrayList(fieldInfo.iterator()); + if (array.size() == 2) { + TypeInformation fieldType = convertToAtomicType(array.get(1)); + types.add(fieldType); + fieldNames.add(array.get(0)); + } else { + throw new RuntimeException(); + } + } + + TypeInformation[] typesArray = types.toArray(new TypeInformation[types.size()]); + String[] fieldNamesArray = fieldNames.toArray(new String[fieldNames.size()]); + return Tuple2.of(typesArray, fieldNamesArray); + } + + /** + * 转换基本类型，所有类型参考Flink官方文档，一共12个基本类型。 + * @param string + * @return + */ + public static TypeInformation convertToAtomicType(String string) { + switch (string.toUpperCase()) { + case "VARCHAR": + case "STRING": + return Types.STRING(); + case "BOOLEAN": + return Types.BOOLEAN(); + case "TINYINT": + return Types.BYTE(); + case "SMALLINT": + return Types.SHORT(); + case "INT": + case "INTEGER": + return Types.INT(); + case "BIGINT": + return Types.LONG(); + case "FLOAT": + case "REAL": + return Types.FLOAT(); + case "DOUBLE": + return Types.DOUBLE(); + case "DECIMAL": + case "DEC": + case "NUMERIC": + return Types.DECIMAL(); + case "DATE": + return Types.SQL_DATE(); + case "TIME": + return Types.SQL_TIME(); + case "TIMESTAMP": + return Types.SQL_TIMESTAMP(); + default: + throw new RuntimeException("type " + string + "not supported"); + } + } +} diff --git a/core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java b/core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java new file mode 100644 index 000000000..4d5627348 --- /dev/null +++ b/core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java @@ -0,0 +1,54 @@ +package com.dtstack.flink.sql.util; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.api.Types; +import org.apache.flink.types.Row; +import org.junit.Assert; +import org.junit.Test; + +import java.sql.Array; + +import static org.junit.Assert.*; + +public class DataTypeUtilsTest { + + @Test + public void convertToArray() { + String atomicStr = "ARRAY"; + String compositeTypeStr = "ARRAY>"; + + TypeInformation atomicArrayType = DataTypeUtils.convertToArray(atomicStr); + TypeInformation compositeTypeArrayType = DataTypeUtils.convertToArray(compositeTypeStr); + + + String[] normalFieldNames = new String[] {"id", "name"}; + TypeInformation[] normalTypes = new TypeInformation[] {Types.INT(), Types.STRING()}; + RowTypeInfo rowTypeInfo = new RowTypeInfo(normalTypes, normalFieldNames); + TypeInformation normalAtomicType = Types.OBJECT_ARRAY(Types.STRING()); + TypeInformation normalCompositeType = Types.OBJECT_ARRAY(rowTypeInfo); + + Assert.assertTrue(normalAtomicType.equals(atomicArrayType)); + Assert.assertTrue(normalCompositeType.equals(compositeTypeArrayType)); + } + + @Test + public void convertToRow() { + String string = "ROW"; + RowTypeInfo rowType = DataTypeUtils.convertToRow(string); + + String[] fieldNames = rowType.getFieldNames(); + Assert.assertTrue("id".equals(fieldNames[0])); + Assert.assertTrue("name".equals(fieldNames[1])); + + TypeInformation[] fieldTypes = rowType.getFieldTypes(); + Assert.assertTrue(Types.INT() == fieldTypes[0]); + Assert.assertTrue(Types.STRING() == fieldTypes[1]); + } + + @Test + public void convertToAtomicType() { + TypeInformation type = DataTypeUtils.convertToAtomicType("STRING"); + Assert.assertTrue(type == Types.STRING()); + } +} \ No newline at end of file diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java index 852a381e2..2cc94b9d6 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; +import com.dtstack.flink.sql.util.DataTypeUtils; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; @@ -32,6 +33,7 @@ import org.apache.flink.types.Row; import org.apache.kafka.clients.consumer.ConsumerConfig; +import java.lang.reflect.Array; import java.util.Map; import java.util.Properties; import java.util.stream.Collectors; @@ -71,9 +73,16 @@ protected String generateOperatorName(String tabName, String topicName) { } protected TypeInformation getRowTypeInformation(KafkaSourceTableInfo kafkaSourceTableInfo) { + String[] fieldTypes = kafkaSourceTableInfo.getFieldTypes(); Class[] fieldClasses = kafkaSourceTableInfo.getFieldClasses(); - TypeInformation[] types = IntStream.range(0, fieldClasses.length) - .mapToObj(i -> TypeInformation.of(fieldClasses[i])) + TypeInformation[] types = + IntStream.range(0, fieldClasses.length) + .mapToObj(i -> { + if (fieldClasses[i].isArray()) { + return DataTypeUtils.convertToArray(fieldTypes[i]); + } + return TypeInformation.of(fieldClasses[i]); + }) .toArray(TypeInformation[]::new); return new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); From 42f22145def750f4bf60faebbce06c81742875bb Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 14 Jul 2020 10:13:19 +0800 Subject: [PATCH 414/523] add field split pattern --- .../dtstack/flink/sql/table/AbstractTableParser.java | 2 +- .../com/dtstack/flink/sql/util/DataTypeUtils.java | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 1f210cbf2..5b344d118 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -85,7 +85,7 @@ public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ } public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ - + // TODO 替换 List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); for(String fieldRow : fieldRows){ fieldRow = fieldRow.trim(); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index ba1a76fd4..92e6ca40f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -2,13 +2,11 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import org.apache.flink.api.common.typeinfo.TypeInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.shaded.guava18.com.google.common.base.Splitter; import org.apache.flink.table.api.Types; -import scala.reflect.macros.TypecheckException; import java.util.ArrayList; import java.util.regex.Matcher; @@ -27,6 +25,11 @@ public class DataTypeUtils { private final static char FIELD_DELIMITER = ','; private final static char TYPE_DELIMITER = ' '; + private final static Pattern COMPLEX_TYPE_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+>)\\s*,"); + private final static Pattern ATOMIC_TYPE_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(\\w+)\\s*,"); + private final static Pattern TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+>)\\s*"); + private final static Pattern ATOMIC_TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(\\w+)\\s*"); + private DataTypeUtils() {} public static TypeInformation convertToCollectionType(String string) { @@ -154,4 +157,9 @@ public static TypeInformation convertToAtomicType(String string) { throw new RuntimeException("type " + string + "not supported"); } } + + public static void splitFields(String fieldStmt) { + // TODO 可以把每轮字符流开始的空白字符去掉。 + + } } From f1eafc25724458e76af843b3c7d06b5b6069f3b7 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 14 Jul 2020 14:32:20 +0800 Subject: [PATCH 415/523] fix side table join error msg --- .gitignore | 1 + core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a32168f1a..d7842e1ed 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ target/ *.eclipse.* *.iml plugins/ +sqlplugins/ lib/ .vertx/ .DS_Store diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java index 7b4d3437b..4bc3ecd8f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java @@ -162,7 +162,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName){ } } if(equalFieldIndex == -1){ - throw new RuntimeException("can't find equal field " + rightField); + throw new RuntimeException("can't find equal field " + leftField); } equalValIndex.add(equalFieldIndex); From 30e57d595025a8d2c0fa6e810ac3823c2dd96bea Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 14 Jul 2020 20:30:40 +0800 Subject: [PATCH 416/523] array type support complete --- .../flink/sql/table/AbstractTableParser.java | 38 ++++++++++++++++++- .../com/dtstack/flink/sql/util/ClassUtil.java | 1 + .../dtstack/flink/sql/util/DataTypeUtils.java | 37 +++++++++++++++--- 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 5b344d118..d633c114d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -27,6 +27,8 @@ import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.regex.Matcher; @@ -50,6 +52,9 @@ public abstract class AbstractTableParser { private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); + private static Pattern compositeTypeHeadPattern = Pattern.compile(".+<.+<"); + private static Pattern compositeTypeTailPattern = Pattern.compile(">\\s*>"); + private Map patternMap = Maps.newHashMap(); private Map handlerMap = Maps.newHashMap(); @@ -85,8 +90,11 @@ public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ } public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ - // TODO 替换 + List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); + + ArrayList cache = new ArrayList<>(); + boolean currentIsCompositeType = false; for(String fieldRow : fieldRows){ fieldRow = fieldRow.trim(); @@ -94,7 +102,33 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ throw new RuntimeException(String.format("table [%s],exists field empty.", tableInfo.getName())); } - String[] filedInfoArr = fieldRow.split("\\s+"); + // 处理复合类型，例如 ARRAY> + String[] filedInfoArr; + Matcher headMatcher = compositeTypeHeadPattern.matcher(fieldRow); + Matcher tailMatcher = compositeTypeTailPattern.matcher(fieldRow); + + if (tailMatcher.find()) { + cache.add(fieldRow); + currentIsCompositeType = false; + fieldRow = String.join("", cache); + cache.clear(); + String[] tmp = fieldRow.split("\\s+"); + String[] type = Arrays.copyOfRange(tmp, 1, tmp.length); + filedInfoArr = new String[] { + tmp[0], + String.join(" ", type) + }; + } else if (headMatcher.find() || currentIsCompositeType) { + currentIsCompositeType = true; + StringBuilder builder = new StringBuilder(); + builder.append(fieldRow); + builder.append(","); + cache.add(builder.toString()); + continue; + } else { + filedInfoArr = fieldRow.split("\\s+"); + } + if(filedInfoArr.length < 2 ){ throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow)); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index 7d092141e..5fa96aca9 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -69,6 +69,7 @@ public static Class stringConvertClass(String str) { case "varchar": case "char": case "text": + case "string": return String.class; case "real": diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index 92e6ca40f..1602205b6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -1,5 +1,6 @@ package com.dtstack.flink.sql.util; +import com.google.common.base.Strings; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import org.apache.flink.api.common.typeinfo.TypeInformation; @@ -9,6 +10,7 @@ import org.apache.flink.table.api.Types; import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -25,10 +27,10 @@ public class DataTypeUtils { private final static char FIELD_DELIMITER = ','; private final static char TYPE_DELIMITER = ' '; - private final static Pattern COMPLEX_TYPE_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+>)\\s*,"); private final static Pattern ATOMIC_TYPE_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(\\w+)\\s*,"); - private final static Pattern TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+>)\\s*"); - private final static Pattern ATOMIC_TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(\\w+)\\s*"); + private final static Pattern COMPLEX_TYPE_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+?>)\\s*,"); + private final static Pattern ATOMIC_TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(\\w+)\\s*$"); + private final static Pattern COMPLEX_TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+>)\\s*$"); private DataTypeUtils() {} @@ -158,8 +160,33 @@ public static TypeInformation convertToAtomicType(String string) { } } - public static void splitFields(String fieldStmt) { - // TODO 可以把每轮字符流开始的空白字符去掉。 + public static ArrayList fieldStmtLexer(String fieldStmts) { + + String stmtStream = fieldStmts; + ArrayList tokens = new ArrayList<>(); + while (Strings.isNullOrEmpty(stmtStream)) { + Matcher atomicTypeMatcher = ATOMIC_TYPE_PATTERN.matcher(stmtStream); + Matcher complexTypeMatcher = COMPLEX_TYPE_PATTERN.matcher(stmtStream); + Matcher atomicTypeTailMatcher = ATOMIC_TAIL_PATTERN.matcher(stmtStream); + Matcher complexTypeTailMatcher = COMPLEX_TAIL_PATTERN.matcher(stmtStream); + + String fieldStmt; + + if (atomicTypeMatcher.find()) { + fieldStmt = atomicTypeMatcher.group(0); + } else if (complexTypeMatcher.find()) { + fieldStmt = complexTypeMatcher.group(0); + } else if (atomicTypeTailMatcher.find()) { + fieldStmt = atomicTypeTailMatcher.group(0); + } else if (complexTypeTailMatcher.find()) { + fieldStmt = complexTypeTailMatcher.group(0); + } else { + throw new RuntimeException("field declaration statement error" + fieldStmts); + } + tokens.add(fieldStmt); + stmtStream = stmtStream.substring(fieldStmt.length() + 1); + } + return tokens; } } From 82291f9870f38a32fbeb78c3fa1e550b24cb8871 Mon Sep 17 00:00:00 2001 From: wuren Date: Thu, 16 Jul 2020 14:23:51 +0800 Subject: [PATCH 417/523] add array type AS keyword support --- .../flink/sql/table/AbstractTableParser.java | 123 ++++++++++-------- .../dtstack/flink/sql/util/DataTypeUtils.java | 5 + 2 files changed, 73 insertions(+), 55 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index d633c114d..beed69fcb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -48,12 +48,12 @@ public abstract class AbstractTableParser { private static final String CHAR_TYPE_NO_LENGTH = "CHAR"; private static Pattern primaryKeyPattern = Pattern.compile("(?i)PRIMARY\\s+KEY\\s*\\((.*)\\)"); - private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(\\w+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); + private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); - private static Pattern compositeTypeHeadPattern = Pattern.compile(".+<.+<"); - private static Pattern compositeTypeTailPattern = Pattern.compile(">\\s*>"); + private static Pattern compositeTypeHeadPattern = Pattern.compile(".+<.+<.+"); + private static Pattern compositeTypeTailPattern = Pattern.compile(".*>\\s*>.*"); private Map patternMap = Maps.newHashMap(); @@ -93,8 +93,8 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); - ArrayList cache = new ArrayList<>(); - boolean currentIsCompositeType = false; + ArrayList buffer = new ArrayList<>(); + for(String fieldRow : fieldRows){ fieldRow = fieldRow.trim(); @@ -103,65 +103,58 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ } // 处理复合类型，例如 ARRAY> - String[] filedInfoArr; Matcher headMatcher = compositeTypeHeadPattern.matcher(fieldRow); Matcher tailMatcher = compositeTypeTailPattern.matcher(fieldRow); - - if (tailMatcher.find()) { - cache.add(fieldRow); - currentIsCompositeType = false; - fieldRow = String.join("", cache); - cache.clear(); - String[] tmp = fieldRow.split("\\s+"); - String[] type = Arrays.copyOfRange(tmp, 1, tmp.length); - filedInfoArr = new String[] { - tmp[0], - String.join(" ", type) - }; - } else if (headMatcher.find() || currentIsCompositeType) { - currentIsCompositeType = true; - StringBuilder builder = new StringBuilder(); - builder.append(fieldRow); - builder.append(","); - cache.add(builder.toString()); - continue; + if ( + !tailMatcher.matches() && + (headMatcher.matches() || + !buffer.isEmpty()) + ) { + writeBuffer(buffer, fieldRow); } else { - filedInfoArr = fieldRow.split("\\s+"); - } - - if(filedInfoArr.length < 2 ){ - throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow)); - } - boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo); - if(isMatcherKey){ - continue; - } + String[] fieldInfoArr; + if (tailMatcher.matches()) { + buffer.add(fieldRow); + fieldRow = String.join("", buffer); + fieldInfoArr = readBuffer(buffer); + } else { + fieldInfoArr = fieldRow.split("\\s+"); + } - //Compatible situation may arise in space in the fieldName - String[] filedNameArr = new String[filedInfoArr.length - 1]; - System.arraycopy(filedInfoArr, 0, filedNameArr, 0, filedInfoArr.length - 1); - String fieldName = String.join(" ", filedNameArr); - String fieldType = filedInfoArr[filedInfoArr.length - 1 ].trim(); + if (fieldInfoArr.length < 2) { + throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow)); + } + boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo); + if (isMatcherKey) { + continue; + } - Class fieldClass = null; - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; + //Compatible situation may arise in space in the fieldName + String[] filedNameArr = new String[fieldInfoArr.length - 1]; + System.arraycopy(fieldInfoArr, 0, filedNameArr, 0, fieldInfoArr.length - 1); + String fieldName = String.join(" ", filedNameArr); + String fieldType = fieldInfoArr[fieldInfoArr.length - 1 ].trim(); + + Class fieldClass = null; + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; + + Matcher matcher = charTypePattern.matcher(fieldType); + if (matcher.find()) { + fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); + fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); + fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); + } else { + fieldClass = dbTypeConvertToJavaType(fieldType); + } - Matcher matcher = charTypePattern.matcher(fieldType); - if (matcher.find()) { - fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); - fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); - fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); - } else { - fieldClass = dbTypeConvertToJavaType(fieldType); + tableInfo.addPhysicalMappings(fieldInfoArr[0],fieldInfoArr[0]); + tableInfo.addField(fieldName); + tableInfo.addFieldClass(fieldClass); + tableInfo.addFieldType(fieldType); + tableInfo.addFieldExtraInfo(fieldExtraInfo); } - - tableInfo.addPhysicalMappings(filedInfoArr[0],filedInfoArr[0]); - tableInfo.addField(fieldName); - tableInfo.addFieldClass(fieldClass); - tableInfo.addFieldType(fieldType); - tableInfo.addFieldExtraInfo(fieldExtraInfo); } tableInfo.finish(); @@ -206,4 +199,24 @@ protected void addParserHandler(String parserName, Pattern pattern, ITableFieldD patternMap.put(parserName, pattern); handlerMap.put(parserName, handler); } + + private void writeBuffer(List buffer, String fieldRow) { + StringBuilder builder = new StringBuilder(); + builder.append(fieldRow); + builder.append(","); + buffer.add(builder.toString()); + } + + private String[] readBuffer(List buffer) { + String fieldRow = String.join("", buffer); + buffer.clear(); + String[] tmp = fieldRow.split("\\s+"); + String[] type = Arrays.copyOfRange(tmp, 1, tmp.length); + String[] fieldInfoArr = new String[] { + tmp[0], + String.join(" ", type) + }; + return fieldInfoArr; + } + } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index 1602205b6..88ae95ec8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -160,6 +160,11 @@ public static TypeInformation convertToAtomicType(String string) { } } + /** + * 目前这个方法未使用，设置当初是想字段声明走统一的词法分析器(分词器)。 + * @param fieldStmts + * @return + */ public static ArrayList fieldStmtLexer(String fieldStmts) { String stmtStream = fieldStmts; From 12107d4e7b4711c06112f194a95ced7ab46ab155 Mon Sep 17 00:00:00 2001 From: wuren Date: Thu, 16 Jul 2020 17:24:31 +0800 Subject: [PATCH 418/523] DT_NEST deserialization support ObjeactArray and Row Type --- .../DtNestRowDeserializationSchema.java | 83 +++++++++++++------ 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 5f1b1c6f3..d3ed68945 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -24,6 +24,7 @@ import org.apache.flink.api.common.serialization.AbstractDeserializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; @@ -34,6 +35,7 @@ import org.apache.flink.types.Row; import java.io.IOException; +import java.lang.reflect.Array; import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; @@ -71,30 +73,7 @@ public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map info) { } else if (info.getTypeClass().equals(Types.SQL_TIMESTAMP.getTypeClass())) { // local zone return Timestamp.valueOf(node.asText()); - } else { + } else if (info instanceof RowTypeInfo) { + return convertRow(node, (RowTypeInfo) info); + } else if (info instanceof ObjectArrayTypeInfo) { + return convertObjectArray(node, ((ObjectArrayTypeInfo) info).getComponentInfo()); + } else { // for types that were specified without JSON schema // e.g. POJOs try { @@ -179,5 +162,55 @@ private Object convert(JsonNode node, TypeInformation info) { } } + private Row convertTopRow() { + Row row = new Row(fieldNames.length); + try { + for (int i = 0; i < fieldNames.length; i++) { + JsonNode node = getIgnoreCase(fieldNames[i]); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); + if (node == null) { + if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { + throw new IllegalStateException("Failed to find field with name '" + + fieldNames[i] + "'."); + } else { + row.setField(i, null); + } + } else { + // Read the value as specified type + Object value = convert(node, fieldTypes[i]); + row.setField(i, value); + } + } + return row; + } finally { + nodeAndJsonNodeMapping.clear(); + } + } + + private Row convertRow(JsonNode node, RowTypeInfo info) { + final String[] names = info.getFieldNames(); + final TypeInformation[] types = info.getFieldTypes(); + + final Row row = new Row(names.length); + for (int i = 0; i < names.length; i++) { + final String name = names[i]; + final JsonNode subNode = node.get(name); + if (subNode == null) { + row.setField(i, null); + } else { + row.setField(i, convert(subNode, types[i])); + } + } + + return row; + } + + private Object convertObjectArray(JsonNode node, TypeInformation elementType) { + final Object[] array = (Object[]) Array.newInstance(elementType.getTypeClass(), node.size()); + for (int i = 0; i < node.size(); i++) { + array[i] = convert(node.get(i), elementType); + } + return array; + } } From d5c06c72287491d379ed719cc37fc1d97028e8ee Mon Sep 17 00:00:00 2001 From: wuren Date: Fri, 17 Jul 2020 11:36:29 +0800 Subject: [PATCH 419/523] fix kafka sink doc --- docs/kafkaSink.md | 223 -------------------------------------- docs/plugin/kafkaSink.md | 229 +++++++++++++++++++++++++-------------- docs/pluginsInfo.md | 1 + 3 files changed, 151 insertions(+), 302 deletions(-) delete mode 100644 docs/kafkaSink.md diff --git a/docs/kafkaSink.md b/docs/kafkaSink.md deleted file mode 100644 index 5d7c7b2a7..000000000 --- a/docs/kafkaSink.md +++ /dev/null @@ -1,223 +0,0 @@ -## 1.格式： -``` -CREATE TABLE tableName( - colName colType, - ... - function(colNameX) AS aliasName, - WATERMARK FOR colName AS withOffset( colName , delayTime ) - )WITH( - type ='kafka11', - bootstrapServers ='ip:port,ip:port...', - zookeeperQuorum ='ip:port,ip:port/zkparent', - offsetReset ='latest', - topic ='topicName', - groupId='test', - parallelism ='parllNum', - ); -``` - -## 2.支持的版本 - kafka09,kafka10,kafka11及以上版本 - **kafka读取和写入的版本必须一致，否则会有兼容性错误。** - -## 3.表结构定义 - -|参数名称|含义| -|----|---| -| tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称| -| colName | 列名称| -| colType | 列类型 [colType支持的类型](colType.md)| - -## 4.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|kafka09、kafka10、kafka11、kafka(对应kafka1.0及以上版本)| -|groupId | 需要读取的 groupId 名称|否|| -|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|topic | 需要读取的 topic 名称|是|| -|parallelism | 并行度设置|否|1| -|partitionKeys | 用来分区的字段|否|| -|updateMode | 回溯流数据下发模式，append,upsert.upsert模式下会将是否为回溯信息以字段形式进行下发。|否|append| -|sinkdatatype | 写入kafka数据格式，json,avro,csv|否|json| -|fieldDelimiter | csv数据分隔符|否| , | - - -**kafka相关参数可以自定义，使用kafka.开头即可。** -``` -kafka.consumer.id -kafka.socket.timeout.ms -kafka.fetch.message.max.bytes -kafka.num.consumer.fetchers -kafka.auto.commit.enable -kafka.auto.commit.interval.ms -kafka.queued.max.message.chunks -kafka.rebalance.max.retries -kafka.fetch.min.bytes -kafka.fetch.wait.max.ms -kafka.rebalance.backoff.ms -kafka.refresh.leader.backoff.ms -kafka.consumer.timeout.ms -kafka.exclude.internal.topics -kafka.partition.assignment.strategy -kafka.client.id -kafka.zookeeper.session.timeout.ms -kafka.zookeeper.connection.timeout.ms -kafka.zookeeper.sync.time.ms -kafka.offsets.storage -kafka.offsets.channel.backoff.ms -kafka.offsets.channel.socket.timeout.ms -kafka.offsets.commit.max.retries -kafka.dual.commit.enabled -kafka.partition.assignment.strategy -kafka.socket.receive.buffer.bytes -kafka.fetch.min.bytes -``` - -## 5.样例： - -### json格式： -``` -CREATE TABLE MyResult( - channel varchar, - pv varchar - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - topic='mqTest02', - parallelism ='2', - partitionKeys = 'channel,pv', - updateMode='upsert' - ); - -upsert模式下发的数据格式：{"channel":"zs","pv":"330",retract:true} -append模式下发的数据格式：{"channel":"zs","pv":"330"} - -``` - -### avro格式： - -如果updateMode='upsert',schemaInfo需要包含retract属性信息。 - -``` -CREATE TABLE MyTable( - channel varchar, - pv varchar - --xctime bigint - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - groupId='mqTest01', - offsetReset='latest', - topic='mqTest01', - parallelism ='1', - topicIsPattern ='false' - ); - -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), - PERIOD FOR SYSTEM_TIME - )WITH( - type='mysql', - url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='sidetest', - cache = 'LRU', - cacheTTLMs='10000', - parallelism ='1' - - ); - - -CREATE TABLE MyResult( - channel varchar, - pv varchar - )WITH( - --type='console' - type='kafka', - bootstrapServers='172.16.8.107:9092', - topic='mqTest02', - parallelism ='1', - updateMode='upsert', - sinkdatatype = 'avro', - schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"} - ,{"name":"pv","type":"string"},{"name":"channel","type":"string"}, - {"name":"retract","type":"boolean"}]}' - - ); - - -insert -into - MyResult - select - a.channel as channel, - a.pv as pv - from - MyTable a -``` -### csv格式： - -``` -CREATE TABLE MyTable( - channel varchar, - pv varchar - --xctime bigint - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - groupId='mqTest01', - offsetReset='latest', - topic='mqTest01', - parallelism ='2', - topicIsPattern ='false' - ); - -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), - PERIOD FOR SYSTEM_TIME - )WITH( - type='mysql', - url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='sidetest', - cache = 'LRU', - cacheTTLMs='10000', - parallelism ='1' - - ); - - -CREATE TABLE MyResult( - channel varchar, - pv varchar - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - topic='mqTest02', - parallelism ='2', - updateMode='upsert', - sinkdatatype = 'csv', - fieldDelimiter='*' - - - - ); - - -insert -into - MyResult - select - a.channel as channel, - a.pv as pv - from - MyTable a -``` diff --git a/docs/plugin/kafkaSink.md b/docs/plugin/kafkaSink.md index 1d8d8ce5f..86b01f3ee 100644 --- a/docs/plugin/kafkaSink.md +++ b/docs/plugin/kafkaSink.md @@ -3,46 +3,47 @@ CREATE TABLE tableName( colName colType, ... - colNameX colType, - [primary key (colName)] + function(colNameX) AS aliasName, + WATERMARK FOR colName AS withOffset( colName , delayTime ) )WITH( - type ='kafka09', + type ='kafka11', bootstrapServers ='ip:port,ip:port...', zookeeperQuorum ='ip:port,ip:port/zkparent', offsetReset ='latest', topic ='topicName', groupId='test', parallelism ='parllNum', - timezone='Asia/Shanghai', - sourcedatatype ='json' #可不设置 ); ``` -## 2.支持版本 - kafka09,kafka10,kafka11及以上版本 - +## 2.支持的版本 + kafka09,kafka10,kafka11及以上版本 + **kafka读取和写入的版本必须一致，否则会有兼容性错误。** + ## 3.表结构定义 |参数名称|含义| |----|---| -| tableName| 结果表名称| +| tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| | colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： - + |参数名称|含义|是否必填|默认值| -|----|----|----|----| -|type |表名的输出表类型[kafka09|kafka10|kafka11|kafka]|是|| +|----|---|---|---| +|type | kafka09 | 是|kafka09、kafka10、kafka11、kafka(对应kafka1.0及以上版本)| |groupId | 需要读取的 groupId 名称|否|| |bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| |zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| |topic | 需要读取的 topic 名称|是|| -|topicIsPattern | topic是否是正则表达式格式(true|false) |否| false -|offsetReset | 读取的topic 的offset初始位置[latest|earliest|指定offset值({"0":12312,"1":12321,"2":12312},{"partition_no":offset_value})]|否|latest| |parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型|否|json| -|timezone|时区设置[timezone支持的参数](../timeZone.md)|否|'Asia/Shanghai' +|partitionKeys | 用来分区的字段|否|| +|updateMode | 回溯流数据下发模式，append,upsert.upsert模式下会将是否为回溯信息以字段形式进行下发。|否|append| +|sinkdatatype | 写入kafka数据格式，json,avro,csv|否|json| +|fieldDelimiter | csv数据分隔符|否| , | + + **kafka相关参数可以自定义，使用kafka.开头即可。** ``` kafka.consumer.id @@ -74,79 +75,149 @@ kafka.socket.receive.buffer.bytes kafka.fetch.min.bytes ``` -## 5.完整样例： +## 5.样例： + +### json格式： +``` +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='2', + partitionKeys = 'channel,pv', + updateMode='upsert' + ); + +upsert模式下发的数据格式：{"channel":"zs","pv":"330",retract:true} +append模式下发的数据格式：{"channel":"zs","pv":"330"} + +``` + +### avro格式： + +如果updateMode='upsert',schemaInfo需要包含retract属性信息。 + ``` CREATE TABLE MyTable( - id bigint, - name varchar, - address varchar -)WITH( - type = 'kafka10', - bootstrapServers = '172.16.101.224:9092', - zookeeperQuorm = '172.16.100.188:2181/kafka', - offsetReset = 'latest', - topic = 'tiezhu_test_in2', - groupId = 'flink_sql', - timezone = 'Asia/Shanghai', - topicIsPattern = 'false', - parallelism = '1' -); - -CREATE TABLE sideTable( - id bigint, - school varchar, - home varchar, - PRIMARY KEY(id), + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='1', + topicIsPattern ='false' + ); + +create table sideTable( + channel varchar, + xccount int, + PRIMARY KEY(channel), PERIOD FOR SYSTEM_TIME -)WITH( + )WITH( type='mysql', - url='jdbc:mysql://172.16.8.109:3306/tiezhu', + url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', userName='dtstack', password='abc123', - tableName='stressTest', - cache='ALL', - parallelism='1' -); + tableName='sidetest', + cache = 'LRU', + cacheTTLMs='10000', + parallelism ='1' + + ); + CREATE TABLE MyResult( - id bigint, - name varchar, - address varchar, - home varchar, - school varchar -)WITH( - type = 'kafka10', - bootstrapServers = '172.16.101.224:9092', - zookeeperQuorm = '172.16.100.188:2181/kafka', - offsetReset = 'latest', - topic = 'tiezhu_test_out2', - parallelism = '1' -); - -insert + channel varchar, + pv varchar + )WITH( + --type='console' + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='1', + updateMode='upsert', + sinkdatatype = 'avro', + schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"} + ,{"name":"pv","type":"string"},{"name":"channel","type":"string"}, + {"name":"retract","type":"boolean"}]}' + + ); + + +insert into - MyResult + MyResult select - t1.id AS id, - t1.name AS name, - t1.address AS address, - t2.school AS school, - t2.home AS home - from( - select - id, - name, - address + a.channel as channel, + a.pv as pv from - MyTable - ) t1 - join sideTable t2 - on t1.id = t2.id; - ``` + MyTable a +``` +### csv格式： + +``` +CREATE TABLE MyTable( + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='2', + topicIsPattern ='false' + ); -## 6.结果表数据示例： +create table sideTable( + channel varchar, + xccount int, + PRIMARY KEY(channel), + PERIOD FOR SYSTEM_TIME + )WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', + userName='dtstack', + password='abc123', + tableName='sidetest', + cache = 'LRU', + cacheTTLMs='10000', + parallelism ='1' + + ); + + +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='2', + updateMode='upsert', + sinkdatatype = 'csv', + fieldDelimiter='*' + + + + ); + + +insert +into + MyResult + select + a.channel as channel, + a.pv as pv + from + MyTable a ``` -[root@node002 bin]# ./kafka-console-consumer.sh --bootstrap-server 172.16.101.224:9092 --topic tiezhu_test_out2 -{"id":122,"name":"tiezhu122","address":"hangzhou122","home":"ganzhou122","school":" ecjtu122"} -{"id":123,"name":"tiezhu123","address":"hangzhou123","home":"ganzhou123","school":" ecjtu123"} -``` \ No newline at end of file diff --git a/docs/pluginsInfo.md b/docs/pluginsInfo.md index 60bfc7fc1..88fda90c9 100644 --- a/docs/pluginsInfo.md +++ b/docs/pluginsInfo.md @@ -1,6 +1,7 @@ ### 1 插件列表 #### 1.1 源表插件 * [kafka 源表插件](plugin/kafkaSource.md) +* [kafka 结果表插件](plugin/kafkaSink.md) #### 1.2 结果表插件 * [elasticsearch 结果表插件](plugin/elasticsearchSink.md) From 28269884b84b55cc70e5187de366af70739168ed Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 17 Jul 2020 17:43:14 +0800 Subject: [PATCH 420/523] fix --- docs/plugin/hbaseSide.md | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index 29dc60bf9..550dfbae0 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -43,12 +43,13 @@ | tableName | hbase 的表名称|是|| | cache | 维表缓存策略(NONE/LRU)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| -|kerberosAuthEnable | 是否开启kerberos认证|否|false| -|regionserverPrincipal | regionserver的principal，这个值从hbase-site.xml的hbase.regionserver.kerberos.principal属性中获取|否|| -|clientKeytabFile|client的keytab 文件|否| -|clientPrincipal|client的principal|否|| -|zookeeperSaslClient | zookeeper.sasl.client值|否|true| -|securityKrb5Conf | java.security.krb5.conf值|否|| +|hbase.security.auth.enable | 是否开启kerberos认证|否|false| +|hbase.security.authentication|认证类型|否|kerberos| +|hbase.kerberos.regionserver.principal | regionserver的principal，这个值从hbase-site.xml的hbase.regionserver.kerberos.principal属性中获取|否|| +|hbase.keytab|client的keytab 文件|否| +|hbase.principal|client的principal|否|| +|hbase.sasl.clientconfig | hbase.sasl.clientconfig值|否|Client,和jaas文件中的域对应| +|java.security.krb5.conf | java.security.krb5.conf值|否|| 另外开启Kerberos认证还需要在VM参数中配置krb5, -Djava.security.krb5.conf=/Users/xuchao/Documents/flinkSql/kerberos/krb5.conf 同时在addShipfile参数中添加keytab文件的路径，参数具体细节请看[命令参数说明](../config.md) -------------- @@ -226,11 +227,13 @@ CREATE TABLE sideTable( cacheTTLMs ='60000', asyncTimeoutNum ='0', parallelism ='1', - kerberosAuthEnable='true', - regionserverPrincipal='hbase/_HOST@DTSTACK.COM', - clientKeytabFile='test.keytab', - clientPrincipal='test@DTSTACK.COM', - securityKrb5Conf='krb5.conf', + hbase.security.auth.enable='true', + hbase.security.authentication='kerberos', + hbase.sasl.clientconfig='Client', + hbase.kerberos.regionserver.principal='hbase/_HOST@DTSTACK.COM', + hbase.keytab='yijing.keytab', + hbase.principal='yijing@DTSTACK.COM', + java.security.krb5.conf='krb5.conf' ); insert into From 37969bd663b748905d9f4b1c3e4043ed08a7ae2f Mon Sep 17 00:00:00 2001 From: wuren Date: Fri, 17 Jul 2020 19:52:18 +0800 Subject: [PATCH 421/523] add error msg --- .../dtstack/flink/sql/util/DataTypeUtils.java | 82 +++++-------------- 1 file changed, 20 insertions(+), 62 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index 88ae95ec8..b02ee5504 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -1,5 +1,6 @@ package com.dtstack.flink.sql.util; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; @@ -21,26 +22,21 @@ **/ public class DataTypeUtils { - private final static Pattern COMPOUND_TYPE_PATTERN = Pattern.compile("(.+?)<(.+)>"); + private final static Pattern COMPOSITE_TYPE_PATTERN = Pattern.compile("(.+?)<(.+)>"); private final static String ARRAY = "ARRAY"; private final static String ROW = "ROW"; private final static char FIELD_DELIMITER = ','; private final static char TYPE_DELIMITER = ' '; - private final static Pattern ATOMIC_TYPE_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(\\w+)\\s*,"); - private final static Pattern COMPLEX_TYPE_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+?>)\\s*,"); - private final static Pattern ATOMIC_TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(\\w+)\\s*$"); - private final static Pattern COMPLEX_TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+>)\\s*$"); - private DataTypeUtils() {} - public static TypeInformation convertToCollectionType(String string) { - Matcher matcher = COMPOUND_TYPE_PATTERN.matcher(string); + public static TypeInformation convertToCompositeType(String string) { + Matcher matcher = COMPOSITE_TYPE_PATTERN.matcher(string); // TODO 现在只支持ARRAY类型后续可以加入 MAP等类型 if (matcher.find() && ARRAY.equals(matcher.group(1))) { return convertToArray(string); } else { - throw new RuntimeException(""); + throw new RuntimeException("type " + string + "is not support!"); } } @@ -50,7 +46,7 @@ public static TypeInformation convertToCollectionType(String string) { * @return */ public static TypeInformation convertToArray(String arrayTypeString) { - Matcher matcher = COMPOUND_TYPE_PATTERN.matcher(arrayTypeString); + Matcher matcher = COMPOSITE_TYPE_PATTERN.matcher(arrayTypeString); if (matcher.find() && ARRAY.equals(matcher.group(1))) { String elementTypeString = matcher.group(2); TypeInformation elementType; @@ -61,7 +57,7 @@ public static TypeInformation convertToArray(String arrayTypeString) { } return Types.OBJECT_ARRAY(elementType); } else { - throw new RuntimeException("convert to array type error!"); + throw new RuntimeException(arrayTypeString + "convert to array type error!"); } } @@ -71,19 +67,17 @@ public static TypeInformation convertToArray(String arrayTypeString) { * @param string */ public static RowTypeInfo convertToRow(String string) { - Matcher matcher = COMPOUND_TYPE_PATTERN.matcher(string); + Matcher matcher = COMPOSITE_TYPE_PATTERN.matcher(string); if (matcher.find() && - ROW.equals( - matcher.group(1).toUpperCase() - ) + ROW.equals(matcher.group(1).toUpperCase()) ) { String elementTypeStr = matcher.group(2); Iterable typeInfo = splitCompositeTypeFields(elementTypeStr); Tuple2 tuple = genFieldInfo(typeInfo); return new RowTypeInfo(tuple.f0, tuple.f1); } else { - throw new RuntimeException(""); + throw new RuntimeException(string + "convert to row type error!"); } } @@ -95,28 +89,26 @@ private static Iterable splitCompositeTypeFields(String string) { } private static Tuple2 genFieldInfo(Iterable typeInfo) { - int fieldsSize = Iterators.size(typeInfo.iterator()); ArrayList types = Lists.newArrayList(); ArrayList fieldNames = Lists.newArrayList(); + for (String type : typeInfo) { Iterable fieldInfo = Splitter .on(TYPE_DELIMITER) .trimResults() .omitEmptyStrings() .split(type); + ArrayList array = Lists.newArrayList(fieldInfo.iterator()); - if (array.size() == 2) { - TypeInformation fieldType = convertToAtomicType(array.get(1)); - types.add(fieldType); - fieldNames.add(array.get(0)); - } else { - throw new RuntimeException(); - } + Preconditions.checkState(array.size() == 2, "field info must be name with type"); + TypeInformation fieldType = convertToAtomicType(array.get(1)); + types.add(fieldType); + fieldNames.add(array.get(0)); } - TypeInformation[] typesArray = types.toArray(new TypeInformation[types.size()]); - String[] fieldNamesArray = fieldNames.toArray(new String[fieldNames.size()]); - return Tuple2.of(typesArray, fieldNamesArray); + TypeInformation[] typeArray = types.toArray(new TypeInformation[types.size()]); + String[] fieldNameArray = fieldNames.toArray(new String[fieldNames.size()]); + return Tuple2.of(typeArray, fieldNameArray); } /** @@ -156,42 +148,8 @@ public static TypeInformation convertToAtomicType(String string) { case "TIMESTAMP": return Types.SQL_TIMESTAMP(); default: - throw new RuntimeException("type " + string + "not supported"); + throw new RuntimeException("type " + string + "not supported, please refer to the flink doc!"); } } - /** - * 目前这个方法未使用，设置当初是想字段声明走统一的词法分析器(分词器)。 - * @param fieldStmts - * @return - */ - public static ArrayList fieldStmtLexer(String fieldStmts) { - - String stmtStream = fieldStmts; - ArrayList tokens = new ArrayList<>(); - while (Strings.isNullOrEmpty(stmtStream)) { - Matcher atomicTypeMatcher = ATOMIC_TYPE_PATTERN.matcher(stmtStream); - Matcher complexTypeMatcher = COMPLEX_TYPE_PATTERN.matcher(stmtStream); - Matcher atomicTypeTailMatcher = ATOMIC_TAIL_PATTERN.matcher(stmtStream); - Matcher complexTypeTailMatcher = COMPLEX_TAIL_PATTERN.matcher(stmtStream); - - String fieldStmt; - - if (atomicTypeMatcher.find()) { - fieldStmt = atomicTypeMatcher.group(0); - } else if (complexTypeMatcher.find()) { - fieldStmt = complexTypeMatcher.group(0); - } else if (atomicTypeTailMatcher.find()) { - fieldStmt = atomicTypeTailMatcher.group(0); - } else if (complexTypeTailMatcher.find()) { - fieldStmt = complexTypeTailMatcher.group(0); - } else { - throw new RuntimeException("field declaration statement error" + fieldStmts); - } - - tokens.add(fieldStmt); - stmtStream = stmtStream.substring(fieldStmt.length() + 1); - } - return tokens; - } } From 1ed54a0d93a744d11b6b77e7c3264bac6be44dcf Mon Sep 17 00:00:00 2001 From: wuren Date: Mon, 20 Jul 2020 15:03:29 +0800 Subject: [PATCH 422/523] Optimize code structure --- .../dtstack/flink/sql/util/DataTypeUtils.java | 130 +++++++++++------- .../flink/sql/util/DataTypeUtilsTest.java | 29 ++-- 2 files changed, 97 insertions(+), 62 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index b02ee5504..041696787 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -30,14 +30,20 @@ public class DataTypeUtils { private DataTypeUtils() {} - public static TypeInformation convertToCompositeType(String string) { - Matcher matcher = COMPOSITE_TYPE_PATTERN.matcher(string); - // TODO 现在只支持ARRAY类型后续可以加入 MAP等类型 - if (matcher.find() && ARRAY.equals(matcher.group(1))) { - return convertToArray(string); - } else { - throw new RuntimeException("type " + string + "is not support!"); - } + /** + * 现在只支持ARRAY类型后续可以加入 MAP等类型 + * @param compositeTypeString + * @return + */ + public static TypeInformation convertToCompositeType(String compositeTypeString) { + Matcher matcher = matchCompositeType(compositeTypeString); + final String errorMsg = "type " + compositeTypeString + "is not support!"; + Preconditions.checkState(matcher.find(), errorMsg); + + String normalizedType = normalizeType(matcher.group(1)); + Preconditions.checkState(ARRAY.equals(normalizedType), errorMsg); + + return convertToArray(compositeTypeString); } /** @@ -46,64 +52,56 @@ public static TypeInformation convertToCompositeType(String string) { * @return */ public static TypeInformation convertToArray(String arrayTypeString) { - Matcher matcher = COMPOSITE_TYPE_PATTERN.matcher(arrayTypeString); - if (matcher.find() && ARRAY.equals(matcher.group(1))) { - String elementTypeString = matcher.group(2); - TypeInformation elementType; - if (elementTypeString.toUpperCase().startsWith(ROW)) { - elementType = convertToRow(elementTypeString); - } else { - elementType = convertToAtomicType(elementTypeString); - } - return Types.OBJECT_ARRAY(elementType); + Matcher matcher = matchCompositeType(arrayTypeString); + final String errorMsg = arrayTypeString + "convert to array type error!"; + Preconditions.checkState(matcher.find(), errorMsg); + + String normalizedType = normalizeType(matcher.group(1)); + Preconditions.checkState(ARRAY.equals(normalizedType), errorMsg); + + String elementTypeString = matcher.group(2); + TypeInformation elementType; + String normalizedElementType = normalizeType(elementTypeString); + if (normalizedElementType.startsWith(ROW)) { + elementType = convertToRow(elementTypeString); } else { - throw new RuntimeException(arrayTypeString + "convert to array type error!"); + elementType = convertToAtomicType(elementTypeString); } + return Types.OBJECT_ARRAY(elementType); } /** * 目前ROW里只支持基本类型 - * @param string + * @param rowTypeString */ - public static RowTypeInfo convertToRow(String string) { - Matcher matcher = COMPOSITE_TYPE_PATTERN.matcher(string); - - if (matcher.find() && - ROW.equals(matcher.group(1).toUpperCase()) - ) { - String elementTypeStr = matcher.group(2); - Iterable typeInfo = splitCompositeTypeFields(elementTypeStr); - Tuple2 tuple = genFieldInfo(typeInfo); - return new RowTypeInfo(tuple.f0, tuple.f1); - } else { - throw new RuntimeException(string + "convert to row type error!"); - } + public static RowTypeInfo convertToRow(String rowTypeString) { + Matcher matcher = matchCompositeType(rowTypeString); + final String errorMsg = rowTypeString + "convert to row type error!"; + Preconditions.checkState(matcher.find(), errorMsg); + + String normalizedType = normalizeType(matcher.group(1)); + Preconditions.checkState(ROW.equals(normalizedType), errorMsg); + + String elementTypeStr = matcher.group(2); + Iterable fieldInfos = splitCompositeTypeField(elementTypeStr); + Tuple2 info = genFieldInfo(fieldInfos); + return new RowTypeInfo(info.f0, info.f1); } - private static Iterable splitCompositeTypeFields(String string) { - return Splitter - .on(FIELD_DELIMITER) - .trimResults() - .split(string); - } - private static Tuple2 genFieldInfo(Iterable typeInfo) { + + private static Tuple2 genFieldInfo(Iterable fieldInfos) { ArrayList types = Lists.newArrayList(); ArrayList fieldNames = Lists.newArrayList(); - for (String type : typeInfo) { - Iterable fieldInfo = Splitter - .on(TYPE_DELIMITER) - .trimResults() - .omitEmptyStrings() - .split(type); - - ArrayList array = Lists.newArrayList(fieldInfo.iterator()); - Preconditions.checkState(array.size() == 2, "field info must be name with type"); - TypeInformation fieldType = convertToAtomicType(array.get(1)); + for (String fieldInfo : fieldInfos) { + Iterable splitedInfo = splitTypeInfo(fieldInfo); + ArrayList info = Lists.newArrayList(splitedInfo.iterator()); + Preconditions.checkState(info.size() == 2, "field info must be name with type"); + TypeInformation fieldType = convertToAtomicType(info.get(1)); types.add(fieldType); - fieldNames.add(array.get(0)); + fieldNames.add(info.get(0)); } TypeInformation[] typeArray = types.toArray(new TypeInformation[types.size()]); @@ -117,7 +115,7 @@ private static Tuple2 genFieldInfo(Iterable * @return */ public static TypeInformation convertToAtomicType(String string) { - switch (string.toUpperCase()) { + switch (normalizeType(string)) { case "VARCHAR": case "STRING": return Types.STRING(); @@ -152,4 +150,32 @@ public static TypeInformation convertToAtomicType(String string) { } } + private static Iterable splitTypeInfo(String string) { + return Splitter + .on(TYPE_DELIMITER) + .trimResults() + .omitEmptyStrings() + .split(string); + } + + private static Iterable splitCompositeTypeField(String string) { + return Splitter + .on(FIELD_DELIMITER) + .trimResults() + .split(string); + } + + private static String replaceBlank(String s) { + return s.replaceAll("\\s", " ").trim(); + } + + private static Matcher matchCompositeType(String s) { + return COMPOSITE_TYPE_PATTERN.matcher( + replaceBlank(s) + ); + } + + private static String normalizeType(String s) { + return s.toUpperCase().trim(); + } } diff --git a/core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java b/core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java index 4d5627348..a8fbf88fc 100644 --- a/core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java +++ b/core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java @@ -15,26 +15,35 @@ public class DataTypeUtilsTest { @Test public void convertToArray() { + String[] normalFieldNames = new String[] { "id", "name" }; + TypeInformation[] normalTypes = new TypeInformation[] {Types.INT(), Types.STRING()}; + RowTypeInfo rowTypeInfo = new RowTypeInfo(normalTypes, normalFieldNames); + TypeInformation normalAtomicType = Types.OBJECT_ARRAY(Types.STRING()); + TypeInformation normalCompositeType = Types.OBJECT_ARRAY(rowTypeInfo); + + String atomicStrWithBlank = " ARRAY< STRING\n > "; + String compositeTypeStrWithBlank = " ARRAY< ROW< id INT, name STRING > > "; + + TypeInformation atomicArrayTypeWithBlank = DataTypeUtils.convertToArray(atomicStrWithBlank); + TypeInformation compositeTypeArrayTypeWithBlank = DataTypeUtils.convertToArray(compositeTypeStrWithBlank); + + Assert.assertTrue(normalAtomicType.equals(atomicArrayTypeWithBlank)); + Assert.assertTrue(normalCompositeType.equals(compositeTypeArrayTypeWithBlank)); + + String atomicStr = "ARRAY"; String compositeTypeStr = "ARRAY>"; TypeInformation atomicArrayType = DataTypeUtils.convertToArray(atomicStr); TypeInformation compositeTypeArrayType = DataTypeUtils.convertToArray(compositeTypeStr); - - String[] normalFieldNames = new String[] {"id", "name"}; - TypeInformation[] normalTypes = new TypeInformation[] {Types.INT(), Types.STRING()}; - RowTypeInfo rowTypeInfo = new RowTypeInfo(normalTypes, normalFieldNames); - TypeInformation normalAtomicType = Types.OBJECT_ARRAY(Types.STRING()); - TypeInformation normalCompositeType = Types.OBJECT_ARRAY(rowTypeInfo); - Assert.assertTrue(normalAtomicType.equals(atomicArrayType)); Assert.assertTrue(normalCompositeType.equals(compositeTypeArrayType)); } @Test public void convertToRow() { - String string = "ROW"; + String string = " ROW < id INT, name STRING > "; RowTypeInfo rowType = DataTypeUtils.convertToRow(string); String[] fieldNames = rowType.getFieldNames(); @@ -48,7 +57,7 @@ public void convertToRow() { @Test public void convertToAtomicType() { - TypeInformation type = DataTypeUtils.convertToAtomicType("STRING"); - Assert.assertTrue(type == Types.STRING()); + TypeInformation type = DataTypeUtils.convertToAtomicType(" TIMESTAMP "); + Assert.assertTrue(type == Types.SQL_TIMESTAMP()); } } \ No newline at end of file From f6ce914ee08b787429d2607389f876dc7058f40e Mon Sep 17 00:00:00 2001 From: wuren Date: Mon, 20 Jul 2020 16:43:28 +0800 Subject: [PATCH 423/523] optimize structure --- .../flink/sql/table/AbstractTableParser.java | 18 ++++++++---------- .../dtstack/flink/sql/util/DataTypeUtils.java | 15 +++++++-------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index beed69fcb..5eaee1c6f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -103,16 +103,15 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ } // 处理复合类型，例如 ARRAY> + // 把ARRAY类型的长串字符压入Buffer Matcher headMatcher = compositeTypeHeadPattern.matcher(fieldRow); Matcher tailMatcher = compositeTypeTailPattern.matcher(fieldRow); - if ( - !tailMatcher.matches() && - (headMatcher.matches() || - !buffer.isEmpty()) - ) { + boolean isNotTail = !tailMatcher.matches(); + boolean isToNeedPush = headMatcher.matches() || !buffer.isEmpty(); + + if (isNotTail && isToNeedPush) { writeBuffer(buffer, fieldRow); } else { - String[] fieldInfoArr; if (tailMatcher.matches()) { buffer.add(fieldRow); @@ -122,9 +121,8 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ fieldInfoArr = fieldRow.split("\\s+"); } - if (fieldInfoArr.length < 2) { - throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow)); - } + String errorMsg = String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow); + Preconditions.checkState(fieldInfoArr.length >= 2, errorMsg); boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo); if (isMatcherKey) { @@ -149,7 +147,7 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ fieldClass = dbTypeConvertToJavaType(fieldType); } - tableInfo.addPhysicalMappings(fieldInfoArr[0],fieldInfoArr[0]); + tableInfo.addPhysicalMappings(fieldInfoArr[0], fieldInfoArr[0]); tableInfo.addField(fieldName); tableInfo.addFieldClass(fieldClass); tableInfo.addFieldType(fieldType); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index 041696787..ff368b975 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -84,24 +84,23 @@ public static RowTypeInfo convertToRow(String rowTypeString) { Preconditions.checkState(ROW.equals(normalizedType), errorMsg); String elementTypeStr = matcher.group(2); - Iterable fieldInfos = splitCompositeTypeField(elementTypeStr); - Tuple2 info = genFieldInfo(fieldInfos); + Iterable fieldInfoStrs = splitCompositeTypeField(elementTypeStr); + Tuple2 info = genFieldInfo(fieldInfoStrs); return new RowTypeInfo(info.f0, info.f1); } - - - private static Tuple2 genFieldInfo(Iterable fieldInfos) { + private static Tuple2 genFieldInfo(Iterable fieldInfoStrs) { ArrayList types = Lists.newArrayList(); ArrayList fieldNames = Lists.newArrayList(); - for (String fieldInfo : fieldInfos) { - Iterable splitedInfo = splitTypeInfo(fieldInfo); + for (String fieldStr : fieldInfoStrs) { + Iterable splitedInfo = splitTypeInfo(fieldStr); ArrayList info = Lists.newArrayList(splitedInfo.iterator()); Preconditions.checkState(info.size() == 2, "field info must be name with type"); + + fieldNames.add(info.get(0)); TypeInformation fieldType = convertToAtomicType(info.get(1)); types.add(fieldType); - fieldNames.add(info.get(0)); } TypeInformation[] typeArray = types.toArray(new TypeInformation[types.size()]); From 4e2a1df6a17d5c3f6c6b40fecc6654c043ebda90 Mon Sep 17 00:00:00 2001 From: wuren Date: Mon, 20 Jul 2020 16:44:32 +0800 Subject: [PATCH 424/523] add space --- .../java/com/dtstack/flink/sql/table/AbstractTableParser.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 5eaee1c6f..2bebb75a4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -95,10 +95,10 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ ArrayList buffer = new ArrayList<>(); - for(String fieldRow : fieldRows){ + for (String fieldRow : fieldRows) { fieldRow = fieldRow.trim(); - if(StringUtils.isBlank(fieldRow)){ + if (StringUtils.isBlank(fieldRow)) { throw new RuntimeException(String.format("table [%s],exists field empty.", tableInfo.getName())); } From faafbd0db0a9383b99c20967ba279478a7367e4c Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 21 Jul 2020 14:20:48 +0800 Subject: [PATCH 425/523] add license and doc, update split part --- .../flink/sql/table/AbstractTableParser.java | 84 +++++++------------ .../dtstack/flink/sql/util/DataTypeUtils.java | 21 ++++- .../dtstack/flink/sql/util/DtStringUtil.java | 6 ++ docs/plugin/kafkaSource.md | 80 +++++++++++++++++- 4 files changed, 132 insertions(+), 59 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 2bebb75a4..8173dafe9 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -16,8 +16,6 @@ * limitations under the License. */ - - package com.dtstack.flink.sql.table; import com.dtstack.flink.sql.util.ClassUtil; @@ -27,7 +25,6 @@ import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -52,9 +49,6 @@ public abstract class AbstractTableParser { private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); - private static Pattern compositeTypeHeadPattern = Pattern.compile(".+<.+<.+"); - private static Pattern compositeTypeTailPattern = Pattern.compile(".*>\\s*>.*"); - private Map patternMap = Maps.newHashMap(); private Map handlerMap = Maps.newHashMap(); @@ -89,12 +83,10 @@ public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ return false; } - public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ + public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); - ArrayList buffer = new ArrayList<>(); - for (String fieldRow : fieldRows) { fieldRow = fieldRow.trim(); @@ -102,63 +94,45 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ throw new RuntimeException(String.format("table [%s],exists field empty.", tableInfo.getName())); } - // 处理复合类型，例如 ARRAY> - // 把ARRAY类型的长串字符压入Buffer - Matcher headMatcher = compositeTypeHeadPattern.matcher(fieldRow); - Matcher tailMatcher = compositeTypeTailPattern.matcher(fieldRow); - boolean isNotTail = !tailMatcher.matches(); - boolean isToNeedPush = headMatcher.matches() || !buffer.isEmpty(); + String[] fieldInfoArr = fieldRow.split("\\s+"); - if (isNotTail && isToNeedPush) { - writeBuffer(buffer, fieldRow); - } else { - String[] fieldInfoArr; - if (tailMatcher.matches()) { - buffer.add(fieldRow); - fieldRow = String.join("", buffer); - fieldInfoArr = readBuffer(buffer); - } else { - fieldInfoArr = fieldRow.split("\\s+"); - } + String errorMsg = String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow); + Preconditions.checkState(fieldInfoArr.length >= 2, errorMsg); - String errorMsg = String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow); - Preconditions.checkState(fieldInfoArr.length >= 2, errorMsg); + boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo); + if (isMatcherKey) { + continue; + } - boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo); - if (isMatcherKey) { - continue; - } + //Compatible situation may arise in space in the fieldName + String[] filedNameArr = new String[fieldInfoArr.length - 1]; + System.arraycopy(fieldInfoArr, 0, filedNameArr, 0, fieldInfoArr.length - 1); + String fieldName = String.join(" ", filedNameArr); + String fieldType = fieldInfoArr[fieldInfoArr.length - 1 ].trim(); - //Compatible situation may arise in space in the fieldName - String[] filedNameArr = new String[fieldInfoArr.length - 1]; - System.arraycopy(fieldInfoArr, 0, filedNameArr, 0, fieldInfoArr.length - 1); - String fieldName = String.join(" ", filedNameArr); - String fieldType = fieldInfoArr[fieldInfoArr.length - 1 ].trim(); - - Class fieldClass = null; - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; - - Matcher matcher = charTypePattern.matcher(fieldType); - if (matcher.find()) { - fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); - fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); - fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); - } else { - fieldClass = dbTypeConvertToJavaType(fieldType); - } + Class fieldClass = null; + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; - tableInfo.addPhysicalMappings(fieldInfoArr[0], fieldInfoArr[0]); - tableInfo.addField(fieldName); - tableInfo.addFieldClass(fieldClass); - tableInfo.addFieldType(fieldType); - tableInfo.addFieldExtraInfo(fieldExtraInfo); + Matcher matcher = charTypePattern.matcher(fieldType); + if (matcher.find()) { + fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); + fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); + fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); + } else { + fieldClass = dbTypeConvertToJavaType(fieldType); } + + tableInfo.addPhysicalMappings(fieldInfoArr[0], fieldInfoArr[0]); + tableInfo.addField(fieldName); + tableInfo.addFieldClass(fieldClass); + tableInfo.addFieldType(fieldType); + tableInfo.addFieldExtraInfo(fieldExtraInfo); } tableInfo.finish(); } - public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo){ + public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo) { String primaryFields = matcher.group(1).trim(); String[] splitArry = primaryFields.split(","); List primaryKes = Lists.newArrayList(splitArry); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index ff368b975..72a8b4b66 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -1,8 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.dtstack.flink.sql.util; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -11,7 +27,6 @@ import org.apache.flink.table.api.Types; import java.util.ArrayList; -import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index b4200d733..37fbf9c2b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -89,6 +89,12 @@ public static List splitIgnoreQuota(String str, char delimiter){ } else if (c == ')' && !inSingleQuotes && !inQuotes) { bracketLeftNum--; b.append(c); + } else if (c == '<' && !inSingleQuotes && !inQuotes) { + bracketLeftNum++; + b.append(c); + } else if (c == '>' && !inSingleQuotes && !inQuotes) { + bracketLeftNum--; + b.append(c); } else { b.append(c); } diff --git a/docs/plugin/kafkaSource.md b/docs/plugin/kafkaSource.md index 9e628f287..67afc45a1 100644 --- a/docs/plugin/kafkaSource.md +++ b/docs/plugin/kafkaSource.md @@ -130,7 +130,85 @@ CREATE TABLE MyTable( ); ``` -数组类型字段解析示例 +#### 数组类型字段解析示例 + +声明数据类型为Array可以使用CROSS JOIN UNNEST语句对Array类型进行展开。（新模式，推荐） + +json样例 +``` +{ + "id": 2, + "some_users": [ + "foo", + "bar" + ] +} +``` + +SQL样例 +``` +CREATE TABLE ods ( + id INT, + some_users ARRAY> +) WITH ( + ... +); + +CREATE TABLE dwd ( + id INT, + user_no INT, + user_info VARCHAR +) WITH ( + type ='console', +); + +INSERT INTO dwd + SELECT id, user_info + FROM ods_foo + CROSS JOIN UNNEST(ods_foo.some_users) AS A (user_info); +``` + +json样例 +``` +{ + "id": 4, + "some_users": [ + { + "user_no": 12, + "user_info": "foo" + + }, + { + "user_no": 14, + "user_info": "bar" + } + ] + +``` + +SQL样例 +``` +CREATE TABLE ods ( + id INT, + some_users ARRAY> +) WITH ( + ... +); + +CREATE TABLE dwd ( + id INT, + user_no INT, + user_info VARCHAR +) WITH ( + type ='console', +); + +INSERT INTO dwd + SELECT id, user_no, user_info + FROM ods_foo + CROSS JOIN UNNEST(ods_foo.some_users) AS A (user_no, user_info); +``` +##### 旧模式 json: {"name":"tom", "obj":{"channel": "root"}, "user": [{"pv": 4}, {"pv": 10}], "xctime":1572932485} ``` From 87f272b1e78e3f2b6dd39d600403794f743e7513 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 21 Jul 2020 14:24:08 +0800 Subject: [PATCH 426/523] fix kafka source doc --- docs/plugin/kafkaSource.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugin/kafkaSource.md b/docs/plugin/kafkaSource.md index 67afc45a1..cb7823900 100644 --- a/docs/plugin/kafkaSource.md +++ b/docs/plugin/kafkaSource.md @@ -149,7 +149,7 @@ SQL样例 ``` CREATE TABLE ods ( id INT, - some_users ARRAY> + some_users ARRAY ) WITH ( ... ); @@ -183,7 +183,7 @@ json样例 "user_info": "bar" } ] - +} ``` SQL样例 From 4112f78cc38d632bebc839780e560a7f832d7327 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 21 Jul 2020 14:41:00 +0800 Subject: [PATCH 427/523] remove useless method --- .../flink/sql/table/AbstractTableParser.java | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 8173dafe9..371de6d5d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -25,7 +25,6 @@ import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.regex.Matcher; @@ -172,23 +171,4 @@ protected void addParserHandler(String parserName, Pattern pattern, ITableFieldD handlerMap.put(parserName, handler); } - private void writeBuffer(List buffer, String fieldRow) { - StringBuilder builder = new StringBuilder(); - builder.append(fieldRow); - builder.append(","); - buffer.add(builder.toString()); - } - - private String[] readBuffer(List buffer) { - String fieldRow = String.join("", buffer); - buffer.clear(); - String[] tmp = fieldRow.split("\\s+"); - String[] type = Arrays.copyOfRange(tmp, 1, tmp.length); - String[] fieldInfoArr = new String[] { - tmp[0], - String.join(" ", type) - }; - return fieldInfoArr; - } - } From 0ff459b1686a683bac5ef7bffcb5eb694dcc7123 Mon Sep 17 00:00:00 2001 From: wuren Date: Wed, 5 Aug 2020 14:04:04 +0800 Subject: [PATCH 428/523] fix parse error when much space before AS --- .../java/com/dtstack/flink/sql/table/AbstractTableParser.java | 2 +- core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 371de6d5d..82b653297 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -44,7 +44,7 @@ public abstract class AbstractTableParser { private static final String CHAR_TYPE_NO_LENGTH = "CHAR"; private static Pattern primaryKeyPattern = Pattern.compile("(?i)PRIMARY\\s+KEY\\s*\\((.*)\\)"); - private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); + private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+?)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index 5fa96aca9..ec88bdbe9 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -37,7 +37,7 @@ public class ClassUtil { public static Class stringConvertClass(String str) { // 这部分主要是告诉Class转TypeInfomation的方法，字段是Array类型 - String lowerStr = str.toLowerCase(); + String lowerStr = str.toLowerCase().trim(); if (lowerStr.startsWith("array")) { return Array.newInstance(Integer.class, 0).getClass(); } From 6b28712b934fc69593ef3191b969cb3bef3dee10 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 6 Aug 2020 10:31:15 +0800 Subject: [PATCH 429/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dredis=20=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E5=86=85=E5=AE=B9=E5=92=8C=E6=A0=BC=E5=BC=8F=E4=B8=8D?= =?UTF-8?q?=E5=AF=B9=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index fbb0aa291..d4d1505f8 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -141,7 +141,7 @@ public void accept(Map values) { if (MapUtils.isNotEmpty(values)) { try { Row row = fillData(input.row(), values); - dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.MultiLine, values)); + dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.SingleLine, row)); resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { dealFillDataError(input, resultFuture, e); From fd6fa5f7b22df1a65090bd82f88f2e836ad7b9a3 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 6 Aug 2020 15:16:43 +0800 Subject: [PATCH 430/523] =?UTF-8?q?fix=20join=E9=87=8D=E5=A4=8Dtablename?= =?UTF-8?q?=E4=B8=8D=E6=B3=A8=E5=86=8C=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/JoinInfo.java | 11 ++++---- .../flink/sql/side/JoinNodeDealer.java | 25 +++++++++++-------- .../dtstack/flink/sql/side/SideSQLParser.java | 23 +++++++++-------- .../dtstack/flink/sql/util/TableUtils.java | 17 +++++++++++++ 4 files changed, 49 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java index 997ed417e..f37b82882 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java @@ -69,6 +69,8 @@ public class JoinInfo implements Serializable { private String scope = ""; + private String newTableName = null; + /** * 左表需要查询的字段信息和output的时候对应的列名称 */ @@ -96,13 +98,12 @@ public String getNonSideTable(){ } public String getNewTableName(){ - //兼容左边表是as 的情况 - String leftStr = leftTableName; - leftStr = Strings.isNullOrEmpty(leftStr) ? leftTableAlias : leftStr; - String newName = leftStr + "_" + rightTableName; - return TableUtils.buildTableNameWithScope(newName, scope); + return this.newTableName; } + public void setNewTableName(String newTableName){ + this.newTableName = newTableName; + } public String getNewTableAlias(){ String newName = leftTableAlias + "_" + rightTableAlias; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index e963fae65..57104bd04 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -91,7 +91,8 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, Set> joinFieldSet, Map tableRef, Map fieldRef, - String scope) { + String scope, + Set joinTableNames) { SqlNode leftNode = joinNode.getLeft(); SqlNode rightNode = joinNode.getRight(); @@ -108,13 +109,13 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, if (leftNode.getKind() == JOIN) { //处理连续join dealNestJoin(joinNode, sideTableSet, queueInfo, parentWhere, parentSelectList, - parentGroupByList, joinFieldSet, tableRef, fieldRef, scope); + parentGroupByList, joinFieldSet, tableRef, fieldRef, scope, joinTableNames); leftNode = joinNode.getLeft(); } if (leftNode.getKind() == AS) { AliasInfo aliasInfo = (AliasInfo) sideSQLParser.parseSql(leftNode, sideTableSet, queueInfo, - parentWhere, parentSelectList, parentGroupByList, scope); + parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); leftTbName = aliasInfo.getName(); leftTbAlias = aliasInfo.getAlias(); } else if(leftNode.getKind() == IDENTIFIER){ @@ -126,7 +127,7 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, Preconditions.checkState(!leftIsSide, "side-table must be at the right of join operator"); Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, - parentWhere, parentSelectList, parentGroupByList, scope); + parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); rightTableName = rightTableNameAndAlias.f0; rightTableAlias = rightTableNameAndAlias.f1; @@ -150,7 +151,8 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, tableInfo.setJoinType(joinType); tableInfo.setCondition(joinNode.getCondition()); tableInfo.setScope(scope); - + tableInfo.setNewTableName(TableUtils.buildTableNameWithScope(leftTbName, leftTbAlias, rightTableName, scope, joinTableNames)); + joinTableNames.add(tableInfo.getNewTableName()); TableUtils.replaceJoinFieldRefTableName(joinNode.getCondition(), fieldRef); //extract 需要查询的字段信息 @@ -262,20 +264,21 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, Set> joinFieldSet, Map tableRef, Map fieldRef, - String scope){ + String scope, + Set joinTableNames){ SqlJoin leftJoinNode = (SqlJoin) joinNode.getLeft(); SqlNode parentRightJoinNode = joinNode.getRight(); SqlNode rightNode = leftJoinNode.getRight(); Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, - parentWhere, parentSelectList, parentGroupByList, scope); + parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); Tuple2 parentRightJoinInfo = parseRightNode(parentRightJoinNode, sideTableSet, - queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); + queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); boolean parentRightIsSide = checkIsSideTable(parentRightJoinInfo.f0, sideTableSet); JoinInfo joinInfo = dealJoinNode(leftJoinNode, sideTableSet, queueInfo, parentWhere, parentSelectList, - parentGroupByList, joinFieldSet, tableRef, fieldRef, scope); + parentGroupByList, joinFieldSet, tableRef, fieldRef, scope, joinTableNames); String rightTableName = rightTableNameAndAlias.f0; boolean rightIsSide = checkIsSideTable(rightTableName, sideTableSet); @@ -670,12 +673,12 @@ private void extractSelectField(SqlNode selectNode, private Tuple2 parseRightNode(SqlNode sqlNode, Set sideTableSet, Queue queueInfo, SqlNode parentWhere, SqlNodeList selectList, SqlNodeList parentGroupByList, - String scope) { + String scope, Set joinTableNames) { Tuple2 tabName = new Tuple2<>("", ""); if(sqlNode.getKind() == IDENTIFIER){ tabName.f0 = sqlNode.toString(); }else{ - AliasInfo aliasInfo = (AliasInfo)sideSQLParser.parseSql(sqlNode, sideTableSet, queueInfo, parentWhere, selectList, parentGroupByList, scope); + AliasInfo aliasInfo = (AliasInfo)sideSQLParser.parseSql(sqlNode, sideTableSet, queueInfo, parentWhere, selectList, parentGroupByList, scope, joinTableNames); tabName.f0 = aliasInfo.getName(); tabName.f1 = aliasInfo.getAlias(); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index ba568144d..d94073b69 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -71,7 +71,7 @@ public Queue getExeQueue(String exeSql, Set sideTableSet, String FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); SqlNode sqlNode = flinkPlanner.parse(exeSql); - parseSql(sqlNode, sideTableSet, queueInfo, null, null, null, scope); + parseSql(sqlNode, sideTableSet, queueInfo, null, null, null, scope, Sets.newHashSet()); queueInfo.offer(sqlNode); return queueInfo; } @@ -92,7 +92,8 @@ public Object parseSql(SqlNode sqlNode, SqlNode parentWhere, SqlNodeList parentSelectList, SqlNodeList parentGroupByList, - String scope){ + String scope, + Set joinTableNames){ SqlKind sqlKind = sqlNode.getKind(); switch (sqlKind){ case WITH: { @@ -100,15 +101,15 @@ public Object parseSql(SqlNode sqlNode, SqlNodeList sqlNodeList = sqlWith.withList; for (SqlNode withAsTable : sqlNodeList) { SqlWithItem sqlWithItem = (SqlWithItem) withAsTable; - parseSql(sqlWithItem.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); + parseSql(sqlWithItem.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); queueInfo.add(sqlWithItem); } - parseSql(sqlWith.body, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); + parseSql(sqlWith.body, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); break; } case INSERT: SqlNode sqlSource = ((SqlInsert)sqlNode).getSource(); - return parseSql(sqlSource, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); + return parseSql(sqlSource, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); case SELECT: SqlNode sqlFrom = ((SqlSelect)sqlNode).getFrom(); SqlNode sqlWhere = ((SqlSelect)sqlNode).getWhere(); @@ -116,7 +117,7 @@ public Object parseSql(SqlNode sqlNode, SqlNodeList groupByList = ((SqlSelect) sqlNode).getGroup(); if(sqlFrom.getKind() != IDENTIFIER){ - Object result = parseSql(sqlFrom, sideTableSet, queueInfo, sqlWhere, selectList, groupByList, scope); + Object result = parseSql(sqlFrom, sideTableSet, queueInfo, sqlWhere, selectList, groupByList, scope, joinTableNames); if(result instanceof JoinInfo){ return TableUtils.dealSelectResultWithJoinInfo((JoinInfo) result, (SqlSelect) sqlNode, queueInfo); }else if(result instanceof AliasInfo){ @@ -138,7 +139,7 @@ public Object parseSql(SqlNode sqlNode, Map tableRef = Maps.newHashMap(); Map fieldRef = Maps.newHashMap(); return joinNodeDealer.dealJoinNode((SqlJoin) sqlNode, sideTableSet, queueInfo, - parentWhere, parentSelectList, parentGroupByList, joinFieldSet, tableRef, fieldRef, scope); + parentWhere, parentSelectList, parentGroupByList, joinFieldSet, tableRef, fieldRef, scope, joinTableNames); case AS: SqlNode info = ((SqlBasicCall)sqlNode).getOperands()[0]; SqlNode alias = ((SqlBasicCall) sqlNode).getOperands()[1]; @@ -147,7 +148,7 @@ public Object parseSql(SqlNode sqlNode, if(info.getKind() == IDENTIFIER){ infoStr = info.toString(); } else { - infoStr = parseSql(info, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope).toString(); + infoStr = parseSql(info, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames).toString(); } AliasInfo aliasInfo = new AliasInfo(); @@ -160,12 +161,12 @@ public Object parseSql(SqlNode sqlNode, SqlNode unionLeft = ((SqlBasicCall)sqlNode).getOperands()[0]; SqlNode unionRight = ((SqlBasicCall)sqlNode).getOperands()[1]; - parseSql(unionLeft, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); - parseSql(unionRight, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); + parseSql(unionLeft, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); + parseSql(unionRight, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); break; case ORDER_BY: SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode; - parseSql(sqlOrderBy.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); + parseSql(sqlOrderBy.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); case LITERAL: return LITERAL.toString(); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index c6befb86d..94878355b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -41,6 +41,7 @@ import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.table.api.Table; @@ -48,6 +49,7 @@ import java.util.Map; import java.util.Queue; import java.util.Set; +import java.util.stream.Collector; import static org.apache.calcite.sql.SqlKind.*; import static org.apache.calcite.sql.SqlKind.CASE; @@ -710,4 +712,19 @@ public static String buildTableNameWithScope(String tableName, String scope){ return tableName + "_" + scope; } + public static String buildTableNameWithScope(String leftTableName, String leftTableAlias, String rightTableName, String scope, Set existTableNames){ + //兼容左边表是as 的情况 + String leftStr = Strings.isNullOrEmpty(leftTableName) ? leftTableAlias : leftTableName; + String newName = leftStr + "_" + rightTableName; + String newTableName = TableUtils.buildTableNameWithScope(newName, scope); + if (CollectionUtils.isEmpty(existTableNames)) { + return newName; + } + if (!existTableNames.contains(newName)) { + return newName; + } + return newName + "_" + System.currentTimeMillis(); + + } + } From ff5ba0720bea30d8564a7065a2f092605444242e Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 11 Aug 2020 20:20:15 +0800 Subject: [PATCH 431/523] =?UTF-8?q?[fix]=20=E4=BF=AE=E5=A4=8D=20DataUtil?= =?UTF-8?q?=EF=BC=8CEs6Sink=20=E4=B8=AD=20java.util.Date=20=E5=92=8C=20jav?= =?UTF-8?q?a.sql.Date=20=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtnest/DtNestRowDeserializationSchema.java | 6 +----- .../dtstack/flink/sql/table/AbstractTableInfo.java | 8 +++----- .../java/com/dtstack/flink/sql/util/DateUtil.java | 13 +++++++++++++ .../sql/sink/elasticsearch/CustomerSinkFunc.java | 10 +++++++++- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 3a6ca4829..13cc261ad 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -106,11 +106,7 @@ private void parseTree(JsonNode jsonNode, String prefix) { String nodeKey = getNodeKey(prefix, next); nodeAndJsonNodeMapping.put(nodeKey, child); - if (child.isArray()) { - parseTree(child, nodeKey); - } else { - parseTree(child, nodeKey); - } + parseTree(child, nodeKey); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java index 3b514dd48..40b7a7e82 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java @@ -16,8 +16,6 @@ * limitations under the License. */ - - package com.dtstack.flink.sql.table; import com.google.common.collect.Lists; @@ -178,9 +176,9 @@ public void setPhysicalFields(Map physicalFields) { } public void finish(){ - this.fields = fieldList.toArray(new String[fieldList.size()]); - this.fieldClasses = fieldClassList.toArray(new Class[fieldClassList.size()]); - this.fieldTypes = fieldTypeList.toArray(new String[fieldTypeList.size()]); + this.fields = fieldList.toArray(new String[0]); + this.fieldClasses = fieldClassList.toArray(new Class[0]); + this.fieldTypes = fieldTypeList.toArray(new String[0]); } /** diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index a420c9d82..b05de0f16 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -108,9 +108,22 @@ public static Date localDateTimetoDate(LocalDateTime localDateTime){ } public static LocalDateTime dateToLocalDateTime(Date date){ + date = transformSqlDateToUtilDate(date); return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); } + /** + * 将java.sql.Date 转化为 java.util.Date + * @param date 不知道是java.sql.Date 还是 java.util.Date + * @return 最后返回 java.util.Date + */ + public static Date transformSqlDateToUtilDate(Date date) { + if (date instanceof java.sql.Date) { + date = new Date(date.getTime()); + } + return date; + } + /** * * diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index ec1732d0e..d594ddd93 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.elasticsearch; +import com.dtstack.flink.sql.util.DateUtil; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.metrics.Counter; @@ -31,6 +32,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.sql.Date; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -42,7 +44,9 @@ public class CustomerSinkFunc implements ElasticsearchSinkFunction { private final Logger logger = LoggerFactory.getLogger(CustomerSinkFunc.class); - /** 用作ID的属性值连接符号 */ + /** + * 用作ID的属性值连接符号 + */ private static final String ID_VALUE_SPLIT = "_"; private String index; @@ -107,6 +111,10 @@ private IndexRequest createIndexRequest(Row element) { Map dataMap = Es6Util.rowToJsonMap(element, fieldNames, fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); for (int i = 0; i < length; i++) { + if (element.getField(i) instanceof Date) { + dataMap.put(fieldNames.get(i), DateUtil.transformSqlDateToUtilDate((Date) element.getField(i))); + continue; + } dataMap.put(fieldNames.get(i), element.getField(i)); } From be084d7177fce96294a3914ebe29310e6a604a04 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 11 Aug 2020 20:41:29 +0800 Subject: [PATCH 432/523] =?UTF-8?q?[fix]=20=E4=BF=AE=E5=A4=8D=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E5=A4=A7=E5=B0=8F=E5=86=99=E4=B8=8D=E5=8C=B9=E9=85=8D?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E5=8A=A0=E8=BD=BD=E6=8F=92=E4=BB=B6=E4=B8=8D?= =?UTF-8?q?=E6=AD=A3=E7=A1=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 6f2fbd9ea..f58a0c9a0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -321,7 +321,7 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPathSets.add(sinkTablePathUrl); } else if (tableInfo instanceof AbstractSideTableInfo) { - String sideOperator = ECacheType.ALL.name().equals(((AbstractSideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; + String sideOperator = ECacheType.ALL.name().equalsIgnoreCase(((AbstractSideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; sideTableMap.put(tableInfo.getName(), (AbstractSideTableInfo) tableInfo); URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, AbstractSideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); From d417e519815c9528bfbf8c2caed7f62d0b1f676b Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 13 Aug 2020 11:42:20 +0800 Subject: [PATCH 433/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E8=A1=A8=E6=98=8E=E6=B2=A1=E6=9C=89=E5=B8=A6=E6=9C=89scope?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/exec/ExecuteProcessHelper.java | 2 +- .../dtstack/flink/sql/util/TableUtils.java | 25 ++++--------------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 6f2fbd9ea..24595f902 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -229,7 +229,7 @@ private static void sqlTranslation(String localSqlPluginPath, } if (isSide) { //sql-dimensional table contains the dimension table of execution - sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null, null); + sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null, String.valueOf(scope)); } else { LOG.info("----------exec sql without dimension join-----------"); LOG.info("----------real sql exec is--------------------------\n{}", result.getExecSql()); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 94878355b..2aeb06cf8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -26,8 +26,6 @@ import com.google.common.collect.HashBasedTable; import com.google.common.collect.HashBiMap; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import com.typesafe.config.ConfigException; import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlDataTypeSpec; @@ -49,7 +47,6 @@ import java.util.Map; import java.util.Queue; import java.util.Set; -import java.util.stream.Collector; import static org.apache.calcite.sql.SqlKind.*; import static org.apache.calcite.sql.SqlKind.CASE; @@ -191,25 +188,13 @@ public static void extractSelectFieldToFieldInfo(SqlNode fieldNode, String fromN } } - public static String buildInternalTableName(String left, char split, String right) { - StringBuilder sb = new StringBuilder(); - return sb.append(left).append(split).append(right).toString(); - } - public static SqlBasicCall buildAsNodeByJoinInfo(JoinInfo joinInfo, SqlNode sqlNode0, String tableAlias) { SqlOperator operator = new SqlAsOperator(); SqlParserPos sqlParserPos = new SqlParserPos(0, 0); String newTableName = joinInfo.getNewTableName(); - String lefTbAlias = joinInfo.getLeftTableAlias(); - if(Strings.isNullOrEmpty(lefTbAlias)){ - Set fromTableSet = Sets.newHashSet(); - TableUtils.getFromTableInfo(joinInfo.getLeftNode(), fromTableSet); - lefTbAlias = StringUtils.join(fromTableSet, "_"); - } - - String newTableAlias = !StringUtils.isEmpty(tableAlias) ? tableAlias : buildInternalTableName(lefTbAlias, SPLIT, joinInfo.getRightTableAlias()); + String newTableAlias = !StringUtils.isEmpty(tableAlias) ? tableAlias : joinInfo.getNewTableAlias(); if (null == sqlNode0) { sqlNode0 = new SqlIdentifier(newTableName, null, sqlParserPos); @@ -716,15 +701,15 @@ public static String buildTableNameWithScope(String leftTableName, String leftTa //兼容左边表是as 的情况 String leftStr = Strings.isNullOrEmpty(leftTableName) ? leftTableAlias : leftTableName; String newName = leftStr + "_" + rightTableName; - String newTableName = TableUtils.buildTableNameWithScope(newName, scope); if (CollectionUtils.isEmpty(existTableNames)) { - return newName; + return TableUtils.buildTableNameWithScope(newName, scope); } + if (!existTableNames.contains(newName)) { - return newName; + return TableUtils.buildTableNameWithScope(newName, scope); } - return newName + "_" + System.currentTimeMillis(); + return TableUtils.buildTableNameWithScope(newName, scope) + "_" + System.currentTimeMillis(); } } From 18740de1f166fe0d74f2377473b3ca3f8bd6626d Mon Sep 17 00:00:00 2001 From: wuren Date: Fri, 14 Aug 2020 14:43:11 +0800 Subject: [PATCH 434/523] fix <> parse error --- .../flink/sql/table/AbstractTableParser.java | 2 +- .../dtstack/flink/sql/util/DtStringUtil.java | 52 +++++++++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 371de6d5d..3b9bf7922 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -84,7 +84,7 @@ public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { - List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); + List fieldRows = DtStringUtil.splitField(fieldsInfo); for (String fieldRow : fieldRows) { fieldRow = fieldRow.trim(); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index 37fbf9c2b..3e2ebb82a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -46,14 +46,61 @@ public class DtStringUtil { private static ObjectMapper objectMapper = new ObjectMapper(); - /** * Split the specified string delimiter --- ignored quotes delimiter * @param str * @param delimiter * @return */ - public static List splitIgnoreQuota(String str, char delimiter){ + public static List splitIgnoreQuota(String str, char delimiter) { + List tokensList = new ArrayList<>(); + boolean inQuotes = false; + boolean inSingleQuotes = false; + int bracketLeftNum = 0; + StringBuilder b = new StringBuilder(); + char[] chars = str.toCharArray(); + int idx = 0; + for (char c : chars) { + char flag = 0; + if (idx > 0) { + flag = chars[idx - 1]; + } + if (c == delimiter) { + if (inQuotes) { + b.append(c); + } else if (inSingleQuotes) { + b.append(c); + } else if (bracketLeftNum > 0) { + b.append(c); + } else { + tokensList.add(b.toString()); + b = new StringBuilder(); + } + } else if (c == '\"' && '\\' != flag && !inSingleQuotes) { + inQuotes = !inQuotes; + b.append(c); + } else if (c == '\'' && '\\' != flag && !inQuotes) { + inSingleQuotes = !inSingleQuotes; + b.append(c); + } else if (c == '(' && !inSingleQuotes && !inQuotes) { + bracketLeftNum++; + b.append(c); + } else if (c == ')' && !inSingleQuotes && !inQuotes) { + bracketLeftNum--; + b.append(c); + } else { + b.append(c); + } + idx++; + } + + tokensList.add(b.toString()); + + return tokensList; + } + + public static List splitField(String str) { + final char delimiter = ','; List tokensList = new ArrayList<>(); boolean inQuotes = false; boolean inSingleQuotes = false; @@ -106,7 +153,6 @@ public static List splitIgnoreQuota(String str, char delimiter){ return tokensList; } - public static String replaceIgnoreQuota(String str, String oriStr, String replaceStr){ String splitPatternStr = oriStr + "(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)(?=(?:[^']*'[^']*')*[^']*$)"; return str.replaceAll(splitPatternStr, replaceStr); From b64dd57aab965268d053144c9143e7a1dea242f5 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 14 Aug 2020 17:03:18 +0800 Subject: [PATCH 435/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=BA=E5=B0=91?= =?UTF-8?q?=E5=BC=95=E7=94=A8=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java index f3b983fb4..86bd7c1a7 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.hbase; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.HadoopKerberosName; +import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.util.KerberosName; import org.slf4j.Logger; From eb23e15fac177b4f67b81100f0f7955335e42384 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 19 Aug 2020 11:42:00 +0800 Subject: [PATCH 436/523] =?UTF-8?q?[fix]=20=E4=BF=AE=E5=A4=8Dpostgre=20?= =?UTF-8?q?=E5=81=9A=E7=BB=B4=E8=A1=A8=E6=97=B6schema=E4=B8=8D=E7=94=9F?= =?UTF-8?q?=E6=95=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/postgresql/PostgresqlAllSideInfo.java | 7 +++++++ .../flink/sql/side/postgresql/PostgresqlAsyncReqRow.java | 6 +----- .../flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java | 7 +++++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java index a4a52eabc..fa07d078b 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java @@ -22,6 +22,8 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -38,4 +40,9 @@ public class PostgresqlAllSideInfo extends RdbAllSideInfo { public PostgresqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } + + @Override + public String getTableName(RdbSideTableInfo rdbSideTableInfo) { + return DtStringUtil.getTableFullPath(rdbSideTableInfo.getSchema(), rdbSideTableInfo.getTableName()); + } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index 1ded64371..3470b6b8f 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -19,10 +19,9 @@ package com.dtstack.flink.sql.side.postgresql; -import com.dtstack.flink.sql.factory.DTThreadFactory; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -35,9 +34,6 @@ import org.slf4j.LoggerFactory; import java.util.List; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; /** * Date: 2019-08-11 diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java index f19a488bc..ddb365fe0 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java @@ -22,6 +22,8 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -39,4 +41,9 @@ public class PostgresqlAsyncSideInfo extends RdbAsyncSideInfo { public PostgresqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } + + @Override + public String getTableName(RdbSideTableInfo rdbSideTableInfo) { + return DtStringUtil.getTableFullPath(rdbSideTableInfo.getSchema(), rdbSideTableInfo.getTableName()); + } } From 49e5a257475096bccb72107131338fa6191d8c5d Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 26 Aug 2020 11:35:56 +0800 Subject: [PATCH 437/523] =?UTF-8?q?fix=20=E5=86=85=E5=AD=98=E6=BA=A2?= =?UTF-8?q?=E5=87=BA=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../elasticsearch6/Elasticsearch6AsyncReqRow.java | 11 +++++++++-- .../flink/sql/side/elasticsearch6/util/Es6Util.java | 3 +++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index fa6d507bb..8be1c1f3a 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -70,6 +70,7 @@ public class Elasticsearch6AsyncReqRow extends BaseAsyncReqRow implements Serial private transient RestHighLevelClient rhlClient; private SearchRequest searchRequest; private List sqlJoinCompareOperate = Lists.newArrayList(); + private static final Integer MAX_ROW_NUM = 50000; public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Elasticsearch6AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); @@ -92,7 +93,7 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul String key = buildCacheKey(inputParams); BoolQueryBuilder boolQueryBuilder = Es6Util.setPredicateclause(sideInfo); boolQueryBuilder = setInputParams(inputParams, boolQueryBuilder); - SearchSourceBuilder searchSourceBuilder = initConfiguration(); + SearchSourceBuilder searchSourceBuilder = initConfiguration(inputParams); searchSourceBuilder.query(boolQueryBuilder); searchRequest.source(searchSourceBuilder); @@ -116,6 +117,11 @@ public void onResponse(SearchResponse searchResponse) { if (searchHits.length < getFetchSize()) { break; } + //protect memory + if (rowList.size() >= MAX_ROW_NUM) { + LOG.warn("row size beyond limit"); + break; + } if (tableInfo == null && tmpRhlClient == null) { // create new connection to fetch data tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); @@ -225,10 +231,11 @@ public void close() throws Exception { } - private SearchSourceBuilder initConfiguration() { + private SearchSourceBuilder initConfiguration(Map inputParams) { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.size(getFetchSize()); searchSourceBuilder.sort("_id", SortOrder.DESC); + inputParams.keySet().stream().forEach(k -> searchSourceBuilder.sort(k, SortOrder.DESC)); String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); searchSourceBuilder.fetchSource(sideFieldNames, null); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java index bc5637515..4a25b43d9 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java @@ -171,6 +171,9 @@ public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBu return boolQueryBuilder.must(QueryBuilders.existsQuery(info.getFieldName())); case "=": case "EQUALS": + if(StringUtils.isBlank(info.getCondition())){ + return boolQueryBuilder; + } return boolQueryBuilder.must(QueryBuilders.termQuery(textConvertToKeyword(info.getFieldName(), sideInfo), removeSpaceAndApostrophe(info.getCondition())[0])); case "<>": case "NOT_EQUALS": From 6cc401fefb29eccbcb53c94bdf8cce2be7204daf Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 1 Sep 2020 09:56:43 +0800 Subject: [PATCH 438/523] =?UTF-8?q?=E5=8D=87=E7=BA=A7flink=E7=9A=84?= =?UTF-8?q?=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 374ec3c31..0496c6dd8 100644 --- a/pom.xml +++ b/pom.xml @@ -42,7 +42,7 @@ UTF-8 - 1.8.1 + 1.8.3 From 9e7bd2b4b7b0ed898549dd7185cc62e16d9b1e36 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 1 Sep 2020 16:30:52 +0800 Subject: [PATCH 439/523] fix array type parse error --- .../flink/sql/table/AbstractTableParser.java | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index b5c463cb6..ade1ebb42 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -24,6 +24,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.tuple.Tuple2; import java.util.List; import java.util.Map; @@ -47,6 +48,8 @@ public abstract class AbstractTableParser { private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+?)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); + private static Pattern typePattern = Pattern.compile("(\\S+)\\s+(.+)"); + private Map patternMap = Maps.newHashMap(); @@ -93,23 +96,16 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { throw new RuntimeException(String.format("table [%s],exists field empty.", tableInfo.getName())); } - String[] fieldInfoArr = fieldRow.split("\\s+"); - - String errorMsg = String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow); - Preconditions.checkState(fieldInfoArr.length >= 2, errorMsg); - boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo); if (isMatcherKey) { continue; } - //Compatible situation may arise in space in the fieldName - String[] filedNameArr = new String[fieldInfoArr.length - 1]; - System.arraycopy(fieldInfoArr, 0, filedNameArr, 0, fieldInfoArr.length - 1); - String fieldName = String.join(" ", filedNameArr); - String fieldType = fieldInfoArr[fieldInfoArr.length - 1 ].trim(); + Tuple2 t = extractType(fieldRow, tableInfo.getName()); + String fieldName = t.f0; + String fieldType = t.f1; - Class fieldClass = null; + Class fieldClass; AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; Matcher matcher = charTypePattern.matcher(fieldType); @@ -121,7 +117,7 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { fieldClass = dbTypeConvertToJavaType(fieldType); } - tableInfo.addPhysicalMappings(fieldInfoArr[0], fieldInfoArr[0]); + tableInfo.addPhysicalMappings(fieldName, fieldName); tableInfo.addField(fieldName); tableInfo.addFieldClass(fieldClass); tableInfo.addFieldType(fieldType); @@ -131,11 +127,23 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { tableInfo.finish(); } + private Tuple2 extractType(String fieldRow, String tableName) { + Matcher matcher = typePattern.matcher(fieldRow); + if (matcher.matches()) { + String fieldName = matcher.group(1); + String fieldType = matcher.group(2); + return Tuple2.of(fieldName, fieldType); + } else { + String errorMsg = String.format("table [%s] field [%s] format error.", tableName, fieldRow); + throw new RuntimeException(errorMsg); + } + } + public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo) { String primaryFields = matcher.group(1).trim(); - String[] splitArry = primaryFields.split(","); - List primaryKes = Lists.newArrayList(splitArry); - tableInfo.setPrimaryKeys(primaryKes); + String[] splitArray = primaryFields.split(","); + List primaryKeys = Lists.newArrayList(splitArray); + tableInfo.setPrimaryKeys(primaryKeys); } /** From e7e64e9958a7c05e708ec0191e952dc9ecd6c5fb Mon Sep 17 00:00:00 2001 From: wuren Date: Wed, 2 Sep 2020 09:26:14 +0800 Subject: [PATCH 440/523] fix regular of making error msg invalid --- .../java/com/dtstack/flink/sql/table/AbstractTableParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index ade1ebb42..f0898cfd6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -48,7 +48,7 @@ public abstract class AbstractTableParser { private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+?)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); - private static Pattern typePattern = Pattern.compile("(\\S+)\\s+(.+)"); + private static Pattern typePattern = Pattern.compile("(\\S+)\\s+(\\w+.*)"); private Map patternMap = Maps.newHashMap(); From 533725bf1739667062d61dde97e101972ccff15b Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 3 Sep 2020 16:39:03 +0800 Subject: [PATCH 441/523] hbase sink failed --- .../java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java index 7af51be3d..13329faed 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java @@ -110,10 +110,6 @@ public static String[] splitIgnoreQuotaBrackets(String str, String delimiter){ public ReplaceInfo getReplaceInfo(String field){ field = field.trim(); - if(field.length() <= 2){ - throw new RuntimeException(field + " \n" + - "Format defined exceptions"); - } //判断是不是常量==>''包裹的标识 if(field.startsWith("'") && field.endsWith("'")){ From 3b433faca7bee93e333cf695305b7698016af4a2 Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 3 Sep 2020 17:11:11 +0800 Subject: [PATCH 442/523] hbase sink failed doc fixed --- docs/plugin/hbaseSink.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/plugin/hbaseSink.md b/docs/plugin/hbaseSink.md index 5006f11a2..a5ad12067 100644 --- a/docs/plugin/hbaseSink.md +++ b/docs/plugin/hbaseSink.md @@ -7,7 +7,7 @@ CREATE TABLE MyResult( type ='hbase', zookeeperQuorum ='ip:port[,ip:port]', tableName ='tableName', - rowKey ='colName[,colName]', + rowKey ='colName[+colName]', parallelism ='1', zookeeperParent ='/hbase' ) @@ -34,7 +34,7 @@ hbase2.0 |zookeeperQuorum | hbase zk地址,多个直接用逗号隔开|是|| |zookeeperParent | zkParent 路径|是|| |tableName | 关联的hbase表名称|是|| -|rowkey | hbase的rowkey关联的列信息,多个值以逗号隔开|是|| +|rowkey | hbase的rowkey关联的列信息,多个值以'+'连接|是|| |updateMode|APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|APPEND｜ |parallelism | 并行度设置|否|1| |kerberosAuthEnable | 是否开启kerberos认证|否|false| @@ -76,7 +76,7 @@ CREATE TABLE MyResult( tableName ='myresult', partitionedJoin ='false', parallelism ='1', - rowKey='name,channel' + rowKey='name+channel' ); insert @@ -141,7 +141,7 @@ into ## 6.hbase数据 ### 数据内容说明 -hbase的rowkey 构建规则：以描述的rowkey字段值作为key，多个字段以'-'连接 +hbase的rowkey 构建规则：以描述的rowkey字段值作为key，多个字段以'+'连接 ### 数据内容示例 hbase(main):007:0> scan 'myresult' ROW COLUMN+CELL From 278bcc4e13cea6e207c1ab436a17a258f4d4e2c8 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 8 Sep 2020 15:01:46 +0800 Subject: [PATCH 443/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8Doracle=20=E8=81=94?= =?UTF-8?q?=E5=90=88=E4=B8=BB=E9=94=AE=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java index 3a320658c..590ecc7a1 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java @@ -104,7 +104,7 @@ private String buildConnectionByAllReplace(boolean allReplace, String col) { private String buildConnectionConditions(String[] uniqueKeyFields) { - return Arrays.stream(uniqueKeyFields).map(col -> "T1." + quoteIdentifier(col) + "=T2." + quoteIdentifier(col)).collect(Collectors.joining(",")); + return Arrays.stream(uniqueKeyFields).map(col -> "T1." + quoteIdentifier(col) + "=T2." + quoteIdentifier(col)).collect(Collectors.joining(" AND ")); } /** From 840edc74fec5ef94fce45450d27d8121f80962c2 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 8 Sep 2020 16:40:05 +0800 Subject: [PATCH 444/523] =?UTF-8?q?=E8=A1=A8=E4=B8=8D=E5=AD=98=E5=9C=A8?= =?UTF-8?q?=E6=97=B6=E4=BB=BB=E5=8A=A1=E7=BB=93=E6=9D=9F=E9=80=80=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/rdb/writer/AppendOnlyWriter.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 7c3ff4b09..9dd06d6a0 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -118,11 +118,18 @@ public void executeUpdate(Connection connection) { } catch (SQLException e1) { throw new RuntimeException(e1); } + + if(e.getMessage().contains("doesn't exist")){ + throw new RuntimeException("table not exist"); + + } if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed ,this row is {}", row.toString()); LOG.error("", e); } metricOutputFormat.outDirtyRecords.inc(); + + } }); rows.clear(); From 1586a62f0d97695c5af49b6c290375c616b262ff Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 8 Sep 2020 17:12:53 +0800 Subject: [PATCH 445/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E5=A3=B0=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 9dd06d6a0..b1d601c22 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -120,7 +120,7 @@ public void executeUpdate(Connection connection) { } if(e.getMessage().contains("doesn't exist")){ - throw new RuntimeException("table not exist"); + throw new RuntimeException(e); } if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { From df5fcccb398071cf65c7f361c2d326e575e504ee Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 18 Sep 2020 09:55:21 +0800 Subject: [PATCH 446/523] =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E8=BD=AC=E6=8D=A2?= =?UTF-8?q?=E4=B8=8D=E4=B8=80=E8=87=B4=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index ec88bdbe9..cad1c47cb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -48,8 +48,10 @@ public static Class stringConvertClass(String str) { return Boolean.class; case "smallint": + return Short.class; case "smallintunsigned": case "tinyint": + return Byte.class; case "tinyintunsigned": case "mediumint": case "mediumintunsigned": From 5d0bd0c840a4976bc8392e8ac000c8d68d823366 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 18 Sep 2020 11:07:59 +0800 Subject: [PATCH 447/523] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AE=9A=E6=97=B6flu?= =?UTF-8?q?sh=20=E5=A4=B1=E8=B4=A5=E6=A0=87=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index 4e047492f..b87c2e042 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -41,6 +41,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -76,6 +77,7 @@ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat tuple2) throws IOException { + if(!flushFlag.get()){ + throw new RuntimeException("connect exception,can not write record"); + } checkConnectionOpen(); try { if (outRecords.getCount() % RECEIVEDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { @@ -187,6 +193,7 @@ private void checkConnectionOpen() { public synchronized void flush() throws Exception { jdbcWriter.executeBatch(connection); batchCount = 0; + flushFlag.set(true); } /** From 3b2f3648240113a895bb45e4c71de4e82045c844 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 18 Sep 2020 16:58:11 +0800 Subject: [PATCH 448/523] =?UTF-8?q?impala=20jdbc=20=E6=94=AF=E6=8C=81=20ch?= =?UTF-8?q?ar=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java index ded966c68..b1a31de42 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java @@ -204,6 +204,8 @@ public static int[] buildSqlTypes(List fieldTypeArray) { tmpFieldsType[i] = Types.TINYINT; } else if (fieldType.equals(Short.class.getName())) { tmpFieldsType[i] = Types.SMALLINT; + } else if(fieldType.equals(Character.class.getName())){ + tmpFieldsType[i] = Types.CHAR; } else if (fieldType.equals(String.class.getName())) { tmpFieldsType[i] = Types.CHAR; } else if (fieldType.equals(Byte.class.getName())) { From f33d68ff856f02057215cf8cce9ead3c35261ffa Mon Sep 17 00:00:00 2001 From: wuren Date: Sat, 19 Sep 2020 15:55:59 +0800 Subject: [PATCH 449/523] remove add file statment --- .../dtstack/flink/sql/parser/SqlParser.java | 20 ++++++++ .../flink/sql/parser/SqlParserTest.java | 51 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index 23f8e4942..badcf8d9c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -28,8 +28,11 @@ import com.google.common.collect.Lists; import com.google.common.base.Strings; +import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Reason: @@ -51,6 +54,8 @@ public static void setLocalSqlPluginRoot(String localSqlPluginRoot){ LOCAL_SQL_PLUGIN_ROOT = localSqlPluginRoot; } + private static final Pattern ADD_FIlE_PATTERN = Pattern.compile("(?i).*add\\s+file\\s+.+"); + /** * flink support sql syntax * CREATE TABLE sls_stream() with (); @@ -74,6 +79,7 @@ public static SqlTree parseSql(String sql) throws Exception { .replace("\t", " ").trim(); List sqlArr = DtStringUtil.splitIgnoreQuota(sql, SQL_DELIMITER); + sqlArr = removeAddFileStmt(sqlArr); SqlTree sqlTree = new SqlTree(); AbstractTableInfoParser tableInfoParser = new AbstractTableInfoParser(); for(String childSql : sqlArr){ @@ -154,4 +160,18 @@ public static SqlTree parseSql(String sql) throws Exception { return sqlTree; } + + /** + * remove add file with statment etc. add file /etc/krb5.conf; + */ + private static List removeAddFileStmt(List stmts) { + List cleanedStmts = new ArrayList<>(); + for (String stmt : stmts) { + Matcher matcher = ADD_FIlE_PATTERN.matcher(stmt); + if(!matcher.matches()) { + cleanedStmts.add(stmt); + } + } + return cleanedStmts; + } } diff --git a/core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java b/core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java new file mode 100644 index 000000000..fdbb03a75 --- /dev/null +++ b/core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.parser; + +import org.junit.Assert; +import org.junit.Test; +import org.powermock.reflect.Whitebox; + +import java.util.ArrayList; +import java.util.List; + +/** + * @program: flink.sql + * @author: wuren + * @create: 2020/09/15 + **/ +public class SqlParserTest { + + @Test + public void testRemoveAddFileStmt() throws Exception { + List rawStmts = new ArrayList<>(); + String sql1 = " add file asdasdasd "; + String sql2 = " aDd fIle With asdasdasd "; + String sql3 = " INSERT INTO dwd_foo SELECT id, name FROM ods_foo"; + String sql4 = " ADD FILE asb "; + rawStmts.add(sql1); + rawStmts.add(sql2); + rawStmts.add(sql3); + rawStmts.add(sql4); + + List stmts = Whitebox.invokeMethod(SqlParser.class, "removeAddFileStmt", rawStmts); + Assert.assertEquals(stmts.get(0), sql3); + } + +} \ No newline at end of file From ad867fefd4b099cc06740dd52e96426965af9c7f Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 21 Sep 2020 11:22:41 +0800 Subject: [PATCH 450/523] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E5=80=BC=E4=B8=8D=E5=AF=B9=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index d4d1505f8..88332c21b 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -141,7 +141,7 @@ public void accept(Map values) { if (MapUtils.isNotEmpty(values)) { try { Row row = fillData(input.row(), values); - dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.SingleLine, row)); + dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.SingleLine, values)); resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { dealFillDataError(input, resultFuture, e); From 4864dc79223c507d832b6f562dcd942993508659 Mon Sep 17 00:00:00 2001 From: wuren Date: Mon, 21 Sep 2020 15:00:54 +0800 Subject: [PATCH 451/523] fix hbase async with kafka kerberos login error --- .gitignore | 2 ++ .../sql/side/hbase/HbaseAsyncReqRow.java | 34 +++++++++++++++---- .../side/hbase/utils/HbaseConfigUtils.java | 13 ++----- .../{resource => resources}/log4j.properties | 0 4 files changed, 32 insertions(+), 17 deletions(-) rename launcher/src/main/{resource => resources}/log4j.properties (100%) diff --git a/.gitignore b/.gitignore index d7842e1ed..c8f97e79e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ lib/ bin/nohup.out .DS_Store bin/sideSql.txt +krb5.conf +*.keytab \ No newline at end of file diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 0250decae..8d082a51a 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -31,10 +31,14 @@ import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; import com.stumbleupon.async.Deferred; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.security.DynamicConfiguration; +import org.apache.flink.runtime.security.KerberosUtils; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; @@ -45,7 +49,12 @@ import org.slf4j.LoggerFactory; import sun.security.krb5.KrbException; +import javax.security.auth.login.AppConfigurationEntry; +import java.io.File; +import java.lang.reflect.Field; +import java.security.PrivilegedExceptionAction; import java.sql.Timestamp; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TimeZone; @@ -100,7 +109,7 @@ public void open(Configuration parameters) throws Exception { ExecutorService executorService =new ThreadPoolExecutor(DEFAULT_POOL_SIZE, DEFAULT_POOL_SIZE, 0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(), new DTThreadFactory("hbase-aysnc")); + new LinkedBlockingQueue<>(), new DTThreadFactory("hbase-async")); Config config = new Config(); config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); @@ -111,12 +120,15 @@ public void open(Configuration parameters) throws Exception { }); if (HbaseConfigUtils.asyncOpenKerberos(hbaseConfig)) { - String jaasStr = HbaseConfigUtils.buildJaasStr(hbaseConfig); - String jaasFilePath = HbaseConfigUtils.creatJassFile(jaasStr); - System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_AUTH_LOGIN_CONF, jaasFilePath); - config.overrideConfig(HbaseConfigUtils.KEY_JAVA_SECURITY_AUTH_LOGIN_CONF, jaasFilePath); + String principal = MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_PRINCIPAL); + String keytab = System.getProperty("user.dir") + File.separator + MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_KEY_TAB); + LOG.info("Kerberos login with keytab: {} and principal: {}", keytab, principal); + String name = "HBaseClient"; + config.overrideConfig("hbase.sasl.clientconfig", name); + appendJaasConf(name, keytab, principal); refreshConfig(); } + hBaseClient = new HBaseClient(config, executorService); try { @@ -144,11 +156,21 @@ public void open(Configuration parameters) throws Exception { } } + private void appendJaasConf(String name, String keytab, String principal) { + javax.security.auth.login.Configuration priorConfig = javax.security.auth.login.Configuration.getConfiguration(); + // construct a dynamic JAAS configuration + DynamicConfiguration currentConfig = new DynamicConfiguration(priorConfig); + // wire up the configured JAAS login contexts to use the krb5 entries + AppConfigurationEntry krb5Entry = KerberosUtils.keytabEntry(keytab, principal); + currentConfig.addAppConfigurationEntry(name, krb5Entry); + javax.security.auth.login.Configuration.setConfiguration(currentConfig); + } + private void refreshConfig() throws KrbException { sun.security.krb5.Config.refresh(); KerberosName.resetDefaultRealm(); //reload java.security.auth.login.config - javax.security.auth.login.Configuration.setConfiguration(null); +// javax.security.auth.login.Configuration.setConfiguration(null); } @Override diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index c6399e27a..c588b178b 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -33,7 +33,6 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.UUID; /** * @@ -60,7 +59,7 @@ public class HbaseConfigUtils { private final static String KEY_HBASE_SASL_CLIENTCONFIG = "hbase.sasl.clientconfig"; private final static String KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL = "hbase.kerberos.regionserver.principal"; public static final String KEY_KEY_TAB = "hbase.keytab"; - private static final String KEY_PRINCIPAL = "hbase.principal"; + public static final String KEY_PRINCIPAL = "hbase.principal"; public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "hbase.zookeeper.znode.parent"; @@ -115,9 +114,6 @@ public static boolean asyncOpenKerberos(Map hbaseConfigMap) { return AUTHENTICATION_TYPE.equalsIgnoreCase(MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION)); } - - - public static Configuration getHadoopConfiguration(Map hbaseConfigMap) { for (String key : KEYS_KERBEROS_REQUIRED) { if (StringUtils.isEmpty(MapUtils.getString(hbaseConfigMap, key))) { @@ -176,16 +172,13 @@ public static String creatJassFile(String configStr) throws IOException { return temp.getAbsolutePath(); } - public static String buildJaasStr(Map kerberosConfig) { + public static String buildJaasStr(Map kerberosConfig,String principal,String keyTab) { for (String key : ASYNC_KEYS_KERBEROS_REQUIRED) { if (StringUtils.isEmpty(MapUtils.getString(kerberosConfig, key))) { throw new IllegalArgumentException(String.format("Must provide [%s] when authentication is Kerberos", key)); } } - String keyTab = System.getProperty("user.dir") + File.separator + MapUtils.getString(kerberosConfig, KEY_KEY_TAB); - String principal = MapUtils.getString(kerberosConfig, KEY_PRINCIPAL); - StringBuilder jaasSB = new StringBuilder("Client {\n" + " com.sun.security.auth.module.Krb5LoginModule required\n" + " useKeyTab=true\n" + @@ -196,8 +189,6 @@ public static String buildJaasStr(Map kerberosConfig) { return jaasSB.toString(); } - - public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { if (conf == null) { throw new IllegalArgumentException("kerberos conf can not be null"); diff --git a/launcher/src/main/resource/log4j.properties b/launcher/src/main/resources/log4j.properties similarity index 100% rename from launcher/src/main/resource/log4j.properties rename to launcher/src/main/resources/log4j.properties From bf2e3095a6b1f77e46af04c66287a3f0b08dd283 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 21 Sep 2020 15:51:37 +0800 Subject: [PATCH 452/523] [fix] fix hbase data dislocate --- .../dtstack/flink/sql/table/AbstractTableInfo.java | 2 +- .../flink/sql/sink/hbase/HbaseOutputFormat.java | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java index 40b7a7e82..f8fa9c3ef 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java @@ -49,7 +49,7 @@ public abstract class AbstractTableInfo implements Serializable { private final List fieldList = Lists.newArrayList(); /**key:别名, value: realField */ - private Map physicalFields = Maps.newHashMap(); + private Map physicalFields = Maps.newLinkedHashMap(); private final List fieldTypeList = Lists.newArrayList(); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 904c75a1b..6c31bd8dd 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -27,7 +27,11 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.AuthUtil; +import org.apache.hadoop.hbase.ChoreService; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.ScheduledChore; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Put; @@ -39,9 +43,9 @@ import java.io.File; import java.io.IOException; import java.security.PrivilegedAction; +import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Set; /** * @author: jingzhen@dtstack.com @@ -327,8 +331,8 @@ public HbaseOutputFormat finish() { String[] qualifiers = new String[format.columnNames.length]; if (format.columnNameFamily != null) { - Set keySet = format.columnNameFamily.keySet(); - String[] columns = keySet.toArray(new String[keySet.size()]); + List keyList = new LinkedList<>(format.columnNameFamily.keySet()); + String[] columns = keyList.toArray(new String[0]); for (int i = 0; i < columns.length; ++i) { String col = columns[i]; String[] part = col.split(":"); From e4082d2e25768bc6f678cb7a9a48d1f9533d709c Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 21 Sep 2020 16:27:46 +0800 Subject: [PATCH 453/523] [fix] fix hbase rowkey format defined --- .../java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java index 7af51be3d..d0911dbcb 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java @@ -110,7 +110,7 @@ public static String[] splitIgnoreQuotaBrackets(String str, String delimiter){ public ReplaceInfo getReplaceInfo(String field){ field = field.trim(); - if(field.length() <= 2){ + if(field.length() <= 0){ throw new RuntimeException(field + " \n" + "Format defined exceptions"); } From 22dd152dfabeac6e07ef2a8a1def2fb06804b93f Mon Sep 17 00:00:00 2001 From: wuren Date: Mon, 21 Sep 2020 21:24:03 +0800 Subject: [PATCH 454/523] add error msg and remove useless method --- .../dtstack/flink/sql/util/DtFileUtils.java | 36 +++++++++++++++ .../sql/side/hbase/HbaseAsyncReqRow.java | 12 ++++- .../sql/side/hbase/HbaseAsyncSideInfo.java | 18 ++++++++ .../side/hbase/utils/HbaseConfigUtils.java | 44 +++---------------- 4 files changed, 70 insertions(+), 40 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/util/DtFileUtils.java diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtFileUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DtFileUtils.java new file mode 100644 index 000000000..eb1c974df --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtFileUtils.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.util; + +import org.apache.flink.util.Preconditions; + +import java.io.File; + +/** + * @program: flinkStreamSQL + * @author: wuren + * @create: 2020/09/21 + **/ +public class DtFileUtils { + public static void checkExists(String path) { + File file = new File(path); + String errorMsg = "%s file is not exist!"; + Preconditions.checkState(file.exists(), errorMsg, path); + } +} diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 8d082a51a..f799f3439 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -30,6 +30,9 @@ import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; +import com.dtstack.flink.sql.util.DtFileUtils; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.stumbleupon.async.Deferred; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -114,14 +117,21 @@ public void open(Configuration parameters) throws Exception { Config config = new Config(); config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, hbaseSideTableInfo.getParent()); - HbaseConfigUtils.loadKrb5Conf(hbaseConfig); hbaseConfig.entrySet().forEach(entity -> { config.overrideConfig(entity.getKey(), (String) entity.getValue()); }); if (HbaseConfigUtils.asyncOpenKerberos(hbaseConfig)) { + HbaseConfigUtils.loadKrb5Conf(hbaseConfig); + String principal = MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_PRINCIPAL); + Preconditions.checkState(!Strings.isNullOrEmpty(principal), "%s must be set!", HbaseConfigUtils.KEY_PRINCIPAL); + String regionserver_principal = MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL); + Preconditions.checkState(!Strings.isNullOrEmpty(regionserver_principal), "%s must be set!", HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL); + + MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_KEY_TAB); String keytab = System.getProperty("user.dir") + File.separator + MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_KEY_TAB); + DtFileUtils.checkExists(keytab); LOG.info("Kerberos login with keytab: {} and principal: {}", keytab, principal); String name = "HBaseClient"; config.overrideConfig("hbase.sasl.clientconfig", name); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncSideInfo.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncSideInfo.java index 135aec004..6601458d4 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncSideInfo.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncSideInfo.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.dtstack.flink.sql.side.hbase; import com.dtstack.flink.sql.side.FieldInfo; diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index c588b178b..bdb4ffdf9 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.hbase.utils; +import com.dtstack.flink.sql.util.DtFileUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -26,9 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -57,7 +56,7 @@ public class HbaseConfigUtils { // async side kerberos private final static String KEY_HBASE_SECURITY_AUTH_ENABLE = "hbase.security.auth.enable"; private final static String KEY_HBASE_SASL_CLIENTCONFIG = "hbase.sasl.clientconfig"; - private final static String KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL = "hbase.kerberos.regionserver.principal"; + public final static String KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL = "hbase.kerberos.regionserver.principal"; public static final String KEY_KEY_TAB = "hbase.keytab"; public static final String KEY_PRINCIPAL = "hbase.principal"; @@ -100,13 +99,6 @@ public static Configuration getConfig(Map hbaseConfigMap) { return hConfiguration; } - public static boolean openKerberos(Map hbaseConfigMap) { - if (!MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHORIZATION)) { - return false; - } - return AUTHENTICATION_TYPE.equalsIgnoreCase(MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION)); - } - public static boolean asyncOpenKerberos(Map hbaseConfigMap) { if (!MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTH_ENABLE)) { return false; @@ -156,39 +148,13 @@ public static String getKeytab(Map hbaseConfigMap) { public static void loadKrb5Conf(Map kerberosConfig) { String krb5FilePath = System.getProperty("user.dir") + File.separator + MapUtils.getString(kerberosConfig, KEY_JAVA_SECURITY_KRB5_CONF); + DtFileUtils.checkExists(krb5FilePath); if (!org.apache.commons.lang.StringUtils.isEmpty(krb5FilePath)) { - System.setProperty(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath);; + System.setProperty(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); + LOG.info("{} is set to {}", KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); } } - public static String creatJassFile(String configStr) throws IOException { - String fileName = System.getProperty("user.dir"); - File krbConf = new File(fileName); - File temp = File.createTempFile("JAAS", ".conf", krbConf); - temp.deleteOnExit(); - BufferedWriter out = new BufferedWriter(new FileWriter(temp, false)); - out.write(configStr + "\n"); - out.close(); - return temp.getAbsolutePath(); - } - - public static String buildJaasStr(Map kerberosConfig,String principal,String keyTab) { - for (String key : ASYNC_KEYS_KERBEROS_REQUIRED) { - if (StringUtils.isEmpty(MapUtils.getString(kerberosConfig, key))) { - throw new IllegalArgumentException(String.format("Must provide [%s] when authentication is Kerberos", key)); - } - } - - StringBuilder jaasSB = new StringBuilder("Client {\n" + - " com.sun.security.auth.module.Krb5LoginModule required\n" + - " useKeyTab=true\n" + - " useTicketCache=false\n"); - jaasSB.append(" keyTab=\"").append(keyTab).append("\"").append("\n"); - jaasSB.append(" principal=\"").append(principal).append("\"").append(";\n"); - jaasSB.append("};"); - return jaasSB.toString(); - } - public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { if (conf == null) { throw new IllegalArgumentException("kerberos conf can not be null"); From 215460328796d80ce2f07d62d7951e9b2bb411ed Mon Sep 17 00:00:00 2001 From: wuren Date: Mon, 21 Sep 2020 22:00:07 +0800 Subject: [PATCH 455/523] update krb5 error msg --- .../sql/side/hbase/utils/HbaseConfigUtils.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index bdb4ffdf9..cdd48b640 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -19,6 +19,8 @@ package com.dtstack.flink.sql.side.hbase.utils; import com.dtstack.flink.sql.util.DtFileUtils; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -146,13 +148,13 @@ public static String getKeytab(Map hbaseConfigMap) { throw new IllegalArgumentException(""); } - public static void loadKrb5Conf(Map kerberosConfig) { - String krb5FilePath = System.getProperty("user.dir") + File.separator + MapUtils.getString(kerberosConfig, KEY_JAVA_SECURITY_KRB5_CONF); + public static void loadKrb5Conf(Map config) { + String krb5conf = MapUtils.getString(config, KEY_JAVA_SECURITY_KRB5_CONF); + Preconditions.checkState(!Strings.isNullOrEmpty(krb5conf), "%s must be set!", KEY_JAVA_SECURITY_KRB5_CONF); + String krb5FilePath = System.getProperty("user.dir") + File.separator + MapUtils.getString(config, KEY_JAVA_SECURITY_KRB5_CONF); DtFileUtils.checkExists(krb5FilePath); - if (!org.apache.commons.lang.StringUtils.isEmpty(krb5FilePath)) { - System.setProperty(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); - LOG.info("{} is set to {}", KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); - } + System.setProperty(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); + LOG.info("{} is set to {}", KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); } public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { From 28fa8c76fc18d5b99f5f2fd869941c1f7943aacd Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 22 Sep 2020 10:32:21 +0800 Subject: [PATCH 456/523] optmize code --- .../sql/side/hbase/HbaseAsyncReqRow.java | 16 +++++----- .../side/hbase/utils/HbaseConfigUtils.java | 31 ++++++++++++++++--- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index f799f3439..c36c4affb 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -121,17 +121,17 @@ public void open(Configuration parameters) throws Exception { config.overrideConfig(entity.getKey(), (String) entity.getValue()); }); - if (HbaseConfigUtils.asyncOpenKerberos(hbaseConfig)) { + if (HbaseConfigUtils.isEnableKerberos(hbaseConfig)) { HbaseConfigUtils.loadKrb5Conf(hbaseConfig); - String principal = MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_PRINCIPAL); - Preconditions.checkState(!Strings.isNullOrEmpty(principal), "%s must be set!", HbaseConfigUtils.KEY_PRINCIPAL); - String regionserver_principal = MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL); - Preconditions.checkState(!Strings.isNullOrEmpty(regionserver_principal), "%s must be set!", HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL); + HbaseConfigUtils.checkOpt(principal, HbaseConfigUtils.KEY_PRINCIPAL); + String regionserverPrincipal = MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL); + HbaseConfigUtils.checkOpt(regionserverPrincipal, HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL); + String keytab = MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_KEY_TAB); + HbaseConfigUtils.checkOpt(keytab, HbaseConfigUtils.KEY_KEY_TAB); + String keytabPath = System.getProperty("user.dir") + File.separator + keytab; + DtFileUtils.checkExists(keytabPath); - MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_KEY_TAB); - String keytab = System.getProperty("user.dir") + File.separator + MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_KEY_TAB); - DtFileUtils.checkExists(keytab); LOG.info("Kerberos login with keytab: {} and principal: {}", keytab, principal); String name = "HBaseClient"; config.overrideConfig("hbase.sasl.clientconfig", name); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index cdd48b640..cad4fccc9 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -101,11 +101,27 @@ public static Configuration getConfig(Map hbaseConfigMap) { return hConfiguration; } - public static boolean asyncOpenKerberos(Map hbaseConfigMap) { - if (!MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTH_ENABLE)) { - return false; + public static boolean isEnableKerberos(Map hbaseConfigMap) { + boolean hasAuthorization = AUTHENTICATION_TYPE.equalsIgnoreCase( + MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHORIZATION) + ); + boolean hasAuthentication = AUTHENTICATION_TYPE.equalsIgnoreCase( + MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION) + ); + boolean hasAuthEnable = MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTH_ENABLE); + + if(hasAuthentication || hasAuthorization || hasAuthEnable) { + LOG.info("Enable kerberos for hbase."); + setKerberosConf(hbaseConfigMap); + return true; } - return AUTHENTICATION_TYPE.equalsIgnoreCase(MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION)); + return false; + } + + private static void setKerberosConf(Map hbaseConfigMap) { + hbaseConfigMap.put(KEY_HBASE_SECURITY_AUTHORIZATION, AUTHENTICATION_TYPE); + hbaseConfigMap.put(KEY_HBASE_SECURITY_AUTHENTICATION, AUTHENTICATION_TYPE); + hbaseConfigMap.put(KEY_HBASE_SECURITY_AUTH_ENABLE, true); } public static Configuration getHadoopConfiguration(Map hbaseConfigMap) { @@ -150,13 +166,18 @@ public static String getKeytab(Map hbaseConfigMap) { public static void loadKrb5Conf(Map config) { String krb5conf = MapUtils.getString(config, KEY_JAVA_SECURITY_KRB5_CONF); - Preconditions.checkState(!Strings.isNullOrEmpty(krb5conf), "%s must be set!", KEY_JAVA_SECURITY_KRB5_CONF); + checkOpt(krb5conf, KEY_JAVA_SECURITY_KRB5_CONF); String krb5FilePath = System.getProperty("user.dir") + File.separator + MapUtils.getString(config, KEY_JAVA_SECURITY_KRB5_CONF); DtFileUtils.checkExists(krb5FilePath); System.setProperty(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); LOG.info("{} is set to {}", KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); } + // TODO 日后改造可以下沉到Core模块 + public static void checkOpt(String opt, String key) { + Preconditions.checkState(!Strings.isNullOrEmpty(opt), "%s must be set!", key); + } + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { if (conf == null) { throw new IllegalArgumentException("kerberos conf can not be null"); From bf17ce0568d0136c6c9a333a575730e086c32baa Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 22 Sep 2020 20:23:30 +0800 Subject: [PATCH 457/523] [fix] fix hbase rowkey format defined --- .../java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java index 6acfcb760..f80bf9f22 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java @@ -101,7 +101,7 @@ public static String[] splitIgnoreQuotaBrackets(String str, String delimiter){ public ReplaceInfo getReplaceInfo(String field){ field = field.trim(); - if(field.length() <= 2){ + if(field.length() <= 0){ throw new RuntimeException(field + " \n" + "Format defined exceptions"); } From e1f0662777282e5bf6138205f90c2ab459617173 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 22 Sep 2020 22:13:17 +0800 Subject: [PATCH 458/523] =?UTF-8?q?[fix]=20fix=20=E7=BB=B4=E8=A1=A8join?= =?UTF-8?q?=E5=87=BA=E7=8E=B0Table=20not=20Found?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/side/JoinNodeDealer.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index 57104bd04..6afc12462 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -489,7 +489,7 @@ private Set extractSelectFieldFromJoinCondition(Set extractSelectFieldFromJoinCondition(Set fromTableNameSet, String checkTableName, Map mappingTableName) { + for (int i = 0; i < mappingTableName.size() + 1; i++) { + if (fromTableNameSet.contains(checkTableName)) { + return true; + } + + checkTableName = mappingTableName.get(checkTableName); + if (checkTableName == null) { + return false; + } + } + return true; + } + private Set extractFieldFromGroupByList(SqlNodeList parentGroupByList, Set fromTableNameSet, Map tableRef){ From 79e0842e796cd5aa0f454c3aa83564dd48aedd2b Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 24 Sep 2020 09:31:49 +0800 Subject: [PATCH 459/523] [30589][kafka-sink]remove kafka sink retract data --- .../java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java | 1 + 1 file changed, 1 insertion(+) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index 7234216a7..1d0c4d0e2 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -100,6 +100,7 @@ protected TableSchema buildTableSchema(String[] fieldNames, TypeInformation[] public void emitDataStream(DataStream> dataStream) { DataStream mapDataStream = dataStream .map((Tuple2 record) -> new CRow(record.f1, record.f0)) + .filter(x -> x.change()) .returns(getRowTypeInfo()) .setParallelism(parallelism); From c90af4d8de11d00b7d6667169f3079aa8a24dc1c Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 24 Sep 2020 17:26:42 +0800 Subject: [PATCH 460/523] [fix] sink add parallelism --- .../flink/sql/sink/cassandra/CassandraSink.java | 9 ++++++++- .../sql/sink/elasticsearch/ElasticsearchSink.java | 10 ++++++++-- .../elasticsearch/table/ElasticsearchSinkParser.java | 3 +++ .../sql/sink/elasticsearch/ElasticsearchSink.java | 10 ++++++++-- .../elasticsearch/table/ElasticsearchSinkParser.java | 3 +++ .../com/dtstack/flink/sql/sink/hbase/HbaseSink.java | 4 ++-- .../flink/sql/sink/kafka/AbstractKafkaSink.java | 2 +- .../flink/sql/sink/kafka/table/KafkaSinkParser.java | 2 +- .../com/dtstack/flink/sql/sink/kudu/KuduSink.java | 5 ++++- .../com/dtstack/flink/sql/sink/mongo/MongoSink.java | 11 ++++++++++- .../com/dtstack/flink/sql/sink/redis/RedisSink.java | 11 ++++++++++- .../flink/sql/sink/redis/table/RedisSinkParser.java | 5 +++-- 12 files changed, 61 insertions(+), 14 deletions(-) diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java index 26152a7d3..a78ef97f7 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java @@ -34,6 +34,8 @@ import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; +import java.util.Objects; + /** * Reason: * Date: 2018/11/22 @@ -57,6 +59,8 @@ public class CassandraSink implements RetractStreamTableSink, IStreamSinkGe protected Integer readTimeoutMillis; protected Integer connectTimeoutMillis; protected Integer poolTimeoutMillis; + protected Integer parallelism = 1; + protected String registerTableName; public CassandraSink() { // TO DO NOTHING @@ -77,6 +81,9 @@ public CassandraSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.readTimeoutMillis = cassandraTableInfo.getReadTimeoutMillis(); this.connectTimeoutMillis = cassandraTableInfo.getConnectTimeoutMillis(); this.poolTimeoutMillis = cassandraTableInfo.getPoolTimeoutMillis(); + this.parallelism = Objects.isNull(cassandraTableInfo.getParallelism()) ? + parallelism : cassandraTableInfo.getParallelism(); + this.registerTableName = cassandraTableInfo.getTableName(); return this; } @@ -100,7 +107,7 @@ public void emitDataStream(DataStream> dataStream) { CassandraOutputFormat outputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(outputFormat); - dataStream.addSink(richSinkFunction); + dataStream.addSink(richSinkFunction).setParallelism(parallelism).name(registerTableName); } @Override diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 5ca81c5ed..6958804fa 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -45,6 +45,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; /** * table output elastic5plugin @@ -75,7 +76,9 @@ public class ElasticsearchSink implements RetractStreamTableSink, IStreamSi private TypeInformation[] fieldTypes; - private int parallelism = -1; + private int parallelism = 1; + + private String registerTableName; private ElasticsearchTableInfo esTableInfo; @@ -149,7 +152,7 @@ private RichSinkFunction createEsSinkFunction(){ @Override public void emitDataStream(DataStream> dataStream) { RichSinkFunction richSinkFunction = createEsSinkFunction(); - DataStreamSink streamSink = dataStream.addSink(richSinkFunction); + DataStreamSink streamSink = dataStream.addSink(richSinkFunction).name(registerTableName); if(parallelism > 0){ streamSink.setParallelism(parallelism); } @@ -176,6 +179,9 @@ public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) String id = elasticsearchTableInfo.getId(); String[] idField = StringUtils.split(id, ","); idIndexList = new ArrayList<>(); + parallelism = Objects.isNull(elasticsearchTableInfo.getParallelism()) ? + parallelism : elasticsearchTableInfo.getParallelism(); + registerTableName = elasticsearchTableInfo.getName(); for(int i = 0; i < idField.length; ++i) { idIndexList.add(Integer.valueOf(idField[i])); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 7988e597c..4117ad961 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -50,6 +50,8 @@ public class ElasticsearchSinkParser extends AbstractTableParser { private static final String KEY_ES_PASSWORD = "password"; + private static final String KEY_ES_PARALLELISM = "parallelism"; + @Override protected boolean fieldNameNeedsUpperCase() { return false; @@ -65,6 +67,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map, IStreamSi private TypeInformation[] fieldTypes; - private int parallelism = -1; + private int parallelism = 1; + + private String registerTableName; private ElasticsearchTableInfo esTableInfo; @@ -121,7 +124,7 @@ private RichSinkFunction createEsSinkFunction() { @Override public void emitDataStream(DataStream> dataStream) { RichSinkFunction richSinkFunction = createEsSinkFunction(); - DataStreamSink streamSink = dataStream.addSink(richSinkFunction); + DataStreamSink streamSink = dataStream.addSink(richSinkFunction).name(registerTableName); if (parallelism > 0) { streamSink.setParallelism(parallelism); } @@ -136,6 +139,9 @@ public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) columnTypes = esTableInfo.getFieldTypes(); esAddressList = Arrays.asList(esTableInfo.getAddress().split(",")); String id = esTableInfo.getId(); + parallelism = Objects.isNull(esTableInfo.getParallelism()) ? + parallelism : esTableInfo.getParallelism(); + registerTableName = esTableInfo.getName(); if (!StringUtils.isEmpty(id)) { idIndexList = Arrays.stream(StringUtils.split(id, ",")).map(Integer::valueOf).collect(Collectors.toList()); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 22c2b72bc..e1e26e1ac 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -49,6 +49,8 @@ public class ElasticsearchSinkParser extends AbstractTableParser { private static final String KEY_TRUE = "true"; + private static final String KEY_PARALLELISM = "parallelism"; + @Override protected boolean fieldNameNeedsUpperCase() { return false; @@ -64,6 +66,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map, IStreamSinkGener< private String clientPrincipal; private String clientKeytabFile; - private int parallelism = -1; + private int parallelism = 1; public HbaseSink() { @@ -115,7 +115,7 @@ public void emitDataStream(DataStream> dataStream) { HbaseOutputFormat outputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(outputFormat); - dataStream.addSink(richSinkFunction); + dataStream.addSink(richSinkFunction).setParallelism(parallelism).name(registerTabName); } @Override diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index 7234216a7..427f5772c 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -56,7 +56,7 @@ public abstract class AbstractKafkaSink implements RetractStreamTableSink, protected String[] partitionKeys; protected String sinkOperatorName; protected Properties properties; - protected int parallelism; + protected int parallelism = 1; protected String topic; protected String tableName; diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index 4ad8947a8..58bf8bc78 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -57,7 +57,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map, Serializable, IStreamSinkGener { @@ -36,7 +37,7 @@ public class KuduSink implements RetractStreamTableSink, Serializable, IStr private Integer defaultSocketReadTimeoutMs; - private int parallelism = -1; + private int parallelism = 1; @Override public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { @@ -47,6 +48,8 @@ public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.defaultSocketReadTimeoutMs = kuduTableInfo.getDefaultSocketReadTimeoutMs(); this.workerCount = kuduTableInfo.getWorkerCount(); this.writeMode = kuduTableInfo.getWriteMode(); + this.parallelism = Objects.isNull(kuduTableInfo.getParallelism()) ? + parallelism : kuduTableInfo.getParallelism(); return this; } diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java index 3f34055ec..39c12a789 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java @@ -33,6 +33,8 @@ import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; +import java.util.Objects; + /** * Reason: * Date: 2018/11/6 @@ -48,6 +50,8 @@ public class MongoSink implements RetractStreamTableSink, IStreamSinkGener< protected String userName; protected String password; protected String database; + protected Integer parallelism = 1; + protected String registerTableName; public MongoSink() { // TO DO NOTHING @@ -61,6 +65,9 @@ public MongoSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.userName = mongoTableInfo.getUserName(); this.password = mongoTableInfo.getPassword(); this.database = mongoTableInfo.getDatabase(); + this.parallelism = Objects.isNull(mongoTableInfo.getParallelism()) ? + parallelism : mongoTableInfo.getParallelism(); + this.registerTableName = mongoTableInfo.getName(); return this; } @@ -77,7 +84,9 @@ public void emitDataStream(DataStream> dataStream) { MongoOutputFormat outputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(outputFormat); - dataStream.addSink(richSinkFunction); + dataStream.addSink(richSinkFunction) + .setParallelism(parallelism) + .name(registerTableName); } @Override diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java index cc49a3ba8..f99e69725 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java @@ -33,6 +33,8 @@ import org.apache.flink.types.Row; import java.util.List; +import java.util.Objects; + /** * @author yanxi */ @@ -64,6 +66,10 @@ public class RedisSink implements RetractStreamTableSink, IStreamSinkGener< protected String masterName; + protected Integer parallelism = 1; + + protected String registerTableName; + public RedisSink(){ } @@ -82,6 +88,9 @@ public RedisSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.minIdle = redisTableInfo.getMinIdle(); this.masterName = redisTableInfo.getMasterName(); this.timeout = redisTableInfo.getTimeout(); + this.parallelism = Objects.isNull(redisTableInfo.getParallelism()) ? + parallelism : redisTableInfo.getParallelism(); + this.registerTableName = redisTableInfo.getName(); return this; } @@ -108,7 +117,7 @@ public void emitDataStream(DataStream> dataStream) { .setMasterName(this.masterName); RedisOutputFormat redisOutputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(redisOutputFormat); - dataStream.addSink(richSinkFunction); + dataStream.addSink(richSinkFunction).setParallelism(parallelism).name(registerTableName); } @Override diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 8961f7da9..91e49c5fc 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -50,11 +50,12 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map primaryKeysList = Lists.newArrayList(); if (!StringUtils.isEmpty(primaryKeysStr)) { - List primaryKeysList = Lists.newArrayList(); primaryKeysList = Arrays.asList(StringUtils.split(primaryKeysStr, ",")); - redisTableInfo.setPrimaryKeys(primaryKeysList); } + redisTableInfo.setPrimaryKeys(primaryKeysList); + redisTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(RedisTableInfo.PARALLELISM_KEY.toLowerCase()))); return redisTableInfo; } From 3d7cbb46e8feee135b759c1eb4c6466a799a2f28 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 25 Sep 2020 11:22:28 +0800 Subject: [PATCH 461/523] [fix] fix when JOIN xxx then table not register error --- .../main/java/com/dtstack/flink/sql/util/TableUtils.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 2aeb06cf8..879b9204b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -232,6 +232,13 @@ public static String dealSelectResultWithJoinInfo(JoinInfo joinInfo, SqlSelect s queueInfo.offer(joinInfo.getLeftNode()); } + if (joinInfo.getLeftNode().getKind() == AS) { + SqlNode leftSqlNode = ((SqlBasicCall) joinInfo.getLeftNode()).getOperands()[0]; + if (leftSqlNode.getKind() == UNION) { + queueInfo.offer(joinInfo.getLeftNode()); + } + } + queueInfo.offer(joinInfo); } replaceFromNodeForJoin(joinInfo, sqlNode); From df6dc408ba92125eeed0ca5071c2df04201a3a84 Mon Sep 17 00:00:00 2001 From: chuixue Date: Sun, 27 Sep 2020 10:09:51 +0800 Subject: [PATCH 462/523] [30589][kafka-sink]remove kafka retract data in append mode --- .../com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java | 6 +++++- .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 1 + .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 1 + .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 1 + .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 1 + 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index 1d0c4d0e2..55d1fa7f8 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.kafka; +import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import org.apache.commons.lang3.StringUtils; @@ -59,6 +60,7 @@ public abstract class AbstractKafkaSink implements RetractStreamTableSink, protected int parallelism; protected String topic; protected String tableName; + protected String updateMode; protected TableSchema schema; protected SinkFunction kafkaProducer; @@ -98,9 +100,11 @@ protected TableSchema buildTableSchema(String[] fieldNames, TypeInformation[] @Override public void emitDataStream(DataStream> dataStream) { + if (updateMode.equalsIgnoreCase(EUpdateMode.APPEND.name())) { + dataStream = dataStream.filter((Tuple2 record) -> record.f0); + } DataStream mapDataStream = dataStream .map((Tuple2 record) -> new CRow(record.f1, record.f0)) - .filter(x -> x.change()) .returns(getRowTypeInfo()) .setParallelism(parallelism); diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 632bb720e..3f92366b8 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -36,6 +36,7 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { Properties kafkaProperties = getKafkaProperties(kafkaSinkTableInfo); this.tableName = kafkaSinkTableInfo.getName(); + this.updateMode = kafkaSinkTableInfo.getUpdateMode(); this.topic = kafkaSinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafkaSinkTableInfo); diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index d22be3d59..402b6ed9b 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -38,6 +38,7 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { Properties kafkaProperties = getKafkaProperties(kafka09SinkTableInfo); this.tableName = kafka09SinkTableInfo.getName(); + this.updateMode = kafka09SinkTableInfo.getUpdateMode(); this.topic = kafka09SinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka09SinkTableInfo); diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index eea78e121..09a1bd781 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -40,6 +40,7 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { Properties kafkaProperties = getKafkaProperties(kafka10SinkTableInfo); this.tableName = kafka10SinkTableInfo.getName(); + this.updateMode = kafka10SinkTableInfo.getUpdateMode(); this.topic = kafka10SinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka10SinkTableInfo); diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index ea45280c7..eee1a4ce5 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -41,6 +41,7 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { Properties kafkaProperties = getKafkaProperties(kafka11SinkTableInfo); this.tableName = kafka11SinkTableInfo.getName(); + this.updateMode = kafka11SinkTableInfo.getUpdateMode(); this.topic = kafka11SinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka11SinkTableInfo); From 1a05ab141fd76031764e6c18006dfed042ed24ba Mon Sep 17 00:00:00 2001 From: dapeng Date: Sun, 27 Sep 2020 16:59:52 +0800 Subject: [PATCH 463/523] =?UTF-8?q?elasticsearch=20bulk=20=E7=9A=84?= =?UTF-8?q?=E5=A4=A7=E5=B0=8F=E5=86=99=E6=AD=BB=E4=BA=861?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/elasticsearch/ElasticsearchSink.java | 1 + .../elasticsearch/table/ElasticsearchSinkParser.java | 3 +++ .../elasticsearch/table/ElasticsearchTableInfo.java | 10 ++++++++++ 3 files changed, 14 insertions(+) diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index b7d9de6fc..602d4cfba 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -135,6 +135,7 @@ public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) type = esTableInfo.getEsType(); columnTypes = esTableInfo.getFieldTypes(); esAddressList = Arrays.asList(esTableInfo.getAddress().split(",")); + this.bulkFlushMaxActions = esTableInfo.getBatchSize(); String id = esTableInfo.getId(); if (!StringUtils.isEmpty(id)) { diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 22c2b72bc..560872b05 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -47,6 +47,8 @@ public class ElasticsearchSinkParser extends AbstractTableParser { private static final String KEY_ES6_PASSWORD = "password"; + public static final String BATCH_SIZE_KEY = "batchSize"; + private static final String KEY_TRUE = "true"; @Override @@ -64,6 +66,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Sun, 27 Sep 2020 17:56:46 +0800 Subject: [PATCH 464/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dredis=E5=AF=86?= =?UTF-8?q?=E7=A0=81=E4=B8=8D=E8=83=BD=E5=8C=85=E5=90=AB#=EF=BC=8C?= =?UTF-8?q?=EF=BC=9F=E7=AD=89=E7=89=B9=E6=AE=8A=E5=AD=97=E7=AC=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/redis/RedisAsyncReqRow.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index d4d1505f8..e0a348c38 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -21,7 +21,9 @@ import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import io.lettuce.core.KeyValue; +import io.lettuce.core.api.async.RedisAsyncCommands; import io.lettuce.core.api.async.RedisStringAsyncCommands; +import io.lettuce.core.cluster.api.async.RedisAdvancedClusterAsyncCommands; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -87,11 +89,6 @@ public void open(Configuration parameters) throws Exception { private void buildRedisClient(RedisSideTableInfo tableInfo){ String url = redisSideTableInfo.getUrl(); String password = redisSideTableInfo.getPassword(); - if (password != null){ - password = password + "@"; - } else { - password = ""; - } String database = redisSideTableInfo.getDatabase(); if (database == null){ database = "0"; @@ -99,25 +96,28 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ switch (RedisType.parse(tableInfo.getRedisType())){ case STANDALONE: StringBuilder redisUri = new StringBuilder(); - redisUri.append("redis://").append(password).append(url).append("/").append(database); + redisUri.append("redis://").append(url).append("/").append(database); redisClient = RedisClient.create(redisUri.toString()); connection = redisClient.connect(); async = connection.async(); + ((RedisAsyncCommands)async).auth(password); break; case SENTINEL: StringBuilder sentinelUri = new StringBuilder(); - sentinelUri.append("redis-sentinel://").append(password) + sentinelUri.append("redis-sentinel://") .append(url).append("/").append(database).append("#").append(redisSideTableInfo.getMasterName()); redisClient = RedisClient.create(sentinelUri.toString()); connection = redisClient.connect(); async = connection.async(); + ((RedisAsyncCommands)async).auth(password); break; case CLUSTER: StringBuilder clusterUri = new StringBuilder(); - clusterUri.append("redis://").append(password).append(url); + clusterUri.append("redis://").append(url); clusterClient = RedisClusterClient.create(clusterUri.toString()); clusterConnection = clusterClient.connect(); async = clusterConnection.async(); + ((RedisAdvancedClusterAsyncCommands)async).auth(password); default: break; } From 09f97a9c9389e0bff756f3dd5aaea477832b68b8 Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Tue, 29 Sep 2020 23:03:52 +0800 Subject: [PATCH 465/523] =?UTF-8?q?=E4=BF=AE=E6=94=B94.1.X=20=E5=AF=B9?= =?UTF-8?q?=E5=BA=94=E5=88=B0flink=E7=89=88=E6=9C=AC=E5=8F=B7=E5=88=B01.8.?= =?UTF-8?q?1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0496c6dd8..374ec3c31 100644 --- a/pom.xml +++ b/pom.xml @@ -42,7 +42,7 @@ UTF-8 - 1.8.3 + 1.8.1 From 64b681fb6f4396d879304d00e94165be21a87666 Mon Sep 17 00:00:00 2001 From: wuren Date: Fri, 9 Oct 2020 10:08:47 +0800 Subject: [PATCH 466/523] remove add file statment --- .../dtstack/flink/sql/parser/SqlParser.java | 20 ++++++++ .../flink/sql/parser/SqlParserTest.java | 51 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index 23f8e4942..badcf8d9c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -28,8 +28,11 @@ import com.google.common.collect.Lists; import com.google.common.base.Strings; +import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Reason: @@ -51,6 +54,8 @@ public static void setLocalSqlPluginRoot(String localSqlPluginRoot){ LOCAL_SQL_PLUGIN_ROOT = localSqlPluginRoot; } + private static final Pattern ADD_FIlE_PATTERN = Pattern.compile("(?i).*add\\s+file\\s+.+"); + /** * flink support sql syntax * CREATE TABLE sls_stream() with (); @@ -74,6 +79,7 @@ public static SqlTree parseSql(String sql) throws Exception { .replace("\t", " ").trim(); List sqlArr = DtStringUtil.splitIgnoreQuota(sql, SQL_DELIMITER); + sqlArr = removeAddFileStmt(sqlArr); SqlTree sqlTree = new SqlTree(); AbstractTableInfoParser tableInfoParser = new AbstractTableInfoParser(); for(String childSql : sqlArr){ @@ -154,4 +160,18 @@ public static SqlTree parseSql(String sql) throws Exception { return sqlTree; } + + /** + * remove add file with statment etc. add file /etc/krb5.conf; + */ + private static List removeAddFileStmt(List stmts) { + List cleanedStmts = new ArrayList<>(); + for (String stmt : stmts) { + Matcher matcher = ADD_FIlE_PATTERN.matcher(stmt); + if(!matcher.matches()) { + cleanedStmts.add(stmt); + } + } + return cleanedStmts; + } } diff --git a/core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java b/core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java new file mode 100644 index 000000000..fdbb03a75 --- /dev/null +++ b/core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.parser; + +import org.junit.Assert; +import org.junit.Test; +import org.powermock.reflect.Whitebox; + +import java.util.ArrayList; +import java.util.List; + +/** + * @program: flink.sql + * @author: wuren + * @create: 2020/09/15 + **/ +public class SqlParserTest { + + @Test + public void testRemoveAddFileStmt() throws Exception { + List rawStmts = new ArrayList<>(); + String sql1 = " add file asdasdasd "; + String sql2 = " aDd fIle With asdasdasd "; + String sql3 = " INSERT INTO dwd_foo SELECT id, name FROM ods_foo"; + String sql4 = " ADD FILE asb "; + rawStmts.add(sql1); + rawStmts.add(sql2); + rawStmts.add(sql3); + rawStmts.add(sql4); + + List stmts = Whitebox.invokeMethod(SqlParser.class, "removeAddFileStmt", rawStmts); + Assert.assertEquals(stmts.get(0), sql3); + } + +} \ No newline at end of file From badc105562e054dc94664f1f03d019e36f1c93f1 Mon Sep 17 00:00:00 2001 From: wuren Date: Fri, 9 Oct 2020 10:54:31 +0800 Subject: [PATCH 467/523] add unit test dependence --- pom.xml | 53 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 0496c6dd8..d10954d3e 100644 --- a/pom.xml +++ b/pom.xml @@ -39,11 +39,56 @@ + + UTF-8 + 1.8.3 + 2.7.3 + 4.12 + 2.21.0 + 2.0.4 + 0.7.8 + - - UTF-8 - 1.8.3 - + + + + junit + junit + ${junit.version} + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + org.powermock + powermock-module-junit4 + ${powermock.version} + test + + + org.powermock + powermock-api-mockito2 + ${powermock.version} + test + + + org.mockito + mockito-core + + + + + org.jacoco + org.jacoco.agent + runtime + test + ${jacoco.version} + + From a786cf8b622a729ccaac594b283a251a16ad1ab5 Mon Sep 17 00:00:00 2001 From: wuren Date: Fri, 9 Oct 2020 13:55:15 +0800 Subject: [PATCH 468/523] add unit test dependences --- pom.xml | 53 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 374ec3c31..d10954d3e 100644 --- a/pom.xml +++ b/pom.xml @@ -39,11 +39,56 @@ + + UTF-8 + 1.8.3 + 2.7.3 + 4.12 + 2.21.0 + 2.0.4 + 0.7.8 + - - UTF-8 - 1.8.1 - + + + + junit + junit + ${junit.version} + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + org.powermock + powermock-module-junit4 + ${powermock.version} + test + + + org.powermock + powermock-api-mockito2 + ${powermock.version} + test + + + org.mockito + mockito-core + + + + + org.jacoco + org.jacoco.agent + runtime + test + ${jacoco.version} + + From ea2d6431db113d0d7122edeeab0a55e6f7762705 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Sat, 10 Oct 2020 12:42:28 +0800 Subject: [PATCH 469/523] resolve conflict --- .../java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java index 6acfcb760..1269b2844 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java @@ -101,10 +101,6 @@ public static String[] splitIgnoreQuotaBrackets(String str, String delimiter){ public ReplaceInfo getReplaceInfo(String field){ field = field.trim(); - if(field.length() <= 2){ - throw new RuntimeException(field + " \n" + - "Format defined exceptions"); - } //判断是不是常量==>''包裹的标识 if(field.startsWith("'") && field.endsWith("'")){ From 3f7488d0120439e0045e26388bfb4c6a19326f79 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Sat, 10 Oct 2020 15:00:58 +0800 Subject: [PATCH 470/523] =?UTF-8?q?[fix]=20Flink=E4=B8=8D=E6=94=AF?= =?UTF-8?q?=E6=8C=81text=EF=BC=8C=E5=AF=B9=E7=94=A8=E6=88=B7=E5=81=9A?= =?UTF-8?q?=E6=8F=90=E7=A4=BA=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index ec88bdbe9..21551364f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -68,9 +68,10 @@ public static Class stringConvertClass(String str) { case "varchar": case "char": - case "text": case "string": return String.class; + case "text": + throw new IllegalArgumentException(str + " type is not support, please use STRING. "); case "real": case "float": From c8a367483cb25dde581588281686a982264ef64e Mon Sep 17 00:00:00 2001 From: tiezhu Date: Sat, 10 Oct 2020 15:36:59 +0800 Subject: [PATCH 471/523] =?UTF-8?q?[fix]=20Flink=E4=B8=8D=E6=94=AF?= =?UTF-8?q?=E6=8C=81text=EF=BC=8C=E5=AF=B9=E7=94=A8=E6=88=B7=E5=81=9A?= =?UTF-8?q?=E6=8F=90=E7=A4=BA=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../side/elasticsearch6/util/ClassUtil.java | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/ClassUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/ClassUtil.java index 90ac58229..4dd1c4300 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/ClassUtil.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/ClassUtil.java @@ -15,12 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - package com.dtstack.flink.sql.side.elasticsearch6.util; import com.dtstack.flink.sql.util.DateUtil; +import java.lang.reflect.Array; import java.math.BigDecimal; import java.sql.Date; import java.sql.Time; @@ -30,13 +29,19 @@ * Reason: TODO ADD REASON(可选) * Date: 2017年03月10日 下午1:16:37 * Company: www.dtstack.com - * * @author sishu.yss */ public class ClassUtil { public static Class stringConvertClass(String str) { - switch (str.toLowerCase()) { + + // 这部分主要是告诉Class转TypeInfomation的方法，字段是Array类型 + String lowerStr = str.toLowerCase().trim(); + if (lowerStr.startsWith("array")) { + return Array.newInstance(Integer.class, 0).getClass(); + } + + switch (lowerStr) { case "boolean": case "bit": return Boolean.class; @@ -54,7 +59,6 @@ public static Class stringConvertClass(String str) { case "blob": return Byte.class; - case "long": case "bigint": case "intunsigned": case "integerunsigned": @@ -63,8 +67,10 @@ public static Class stringConvertClass(String str) { case "varchar": case "char": - case "text": + case "string": return String.class; + case "text": + throw new IllegalArgumentException(str + " type is not support, please use STRING. "); case "real": case "float": @@ -90,9 +96,10 @@ public static Class stringConvertClass(String str) { case "decimalunsigned": return BigDecimal.class; default: - throw new RuntimeException("不支持 " + str + " 类型"); + break; } + throw new RuntimeException("不支持 " + str + " 类型"); } public static Object convertType(Object field, String fromType, String toType) { From 1284859eced6f93556b40feef8284d283fc82539 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 13 Oct 2020 09:58:15 +0800 Subject: [PATCH 472/523] rollback flink version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d10954d3e..d21ae0f9a 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ UTF-8 - 1.8.3 + 1.8.1 2.7.3 4.12 2.21.0 From 42c7c18c23d60ce783c5b257a23bf96797711814 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 13 Oct 2020 20:56:36 +0800 Subject: [PATCH 473/523] =?UTF-8?q?=E5=8D=87=E7=BA=A7flink=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E8=87=B31.8.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d21ae0f9a..d10954d3e 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ UTF-8 - 1.8.1 + 1.8.3 2.7.3 4.12 2.21.0 From d87145c8fc582ec87665ff4ba103d532dbe2a372 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 21 Oct 2020 10:32:34 +0800 Subject: [PATCH 474/523] =?UTF-8?q?fix=20=E8=BF=9E=E6=8E=A5=E6=96=AD?= =?UTF-8?q?=E5=BC=80=E4=BB=BB=E5=8A=A1=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/rdb/writer/AbstractUpsertWriter.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java index 42ed545a6..bc4f763a5 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -170,8 +170,16 @@ public void executeUpdate(Connection connection) throws SQLException { connection.commit(); } catch (Exception e) { // deal pg error: current transaction is aborted, commands ignored until end of transaction block - connection.rollback(); - connection.commit(); + try { + connection.rollback(); + connection.commit(); + } catch (SQLException e1) { + throw new RuntimeException(e1); + } + + if(e.getMessage().contains("doesn't exist")){ + throw new RuntimeException(e); + } if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed ,this row is {}", entry.getValue()); LOG.error("", e); From ae223316115642e075a6db8169817d14c2603df2 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 21 Oct 2020 11:39:03 +0800 Subject: [PATCH 475/523] =?UTF-8?q?fix=20=E7=89=88=E6=9C=AC=E4=B8=8D?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=E7=9A=84api?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/redis/RedisAsyncReqRow.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index e0a348c38..c0d418fc1 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import io.lettuce.core.KeyValue; +import io.lettuce.core.RedisURI; import io.lettuce.core.api.async.RedisAsyncCommands; import io.lettuce.core.api.async.RedisStringAsyncCommands; import io.lettuce.core.cluster.api.async.RedisAdvancedClusterAsyncCommands; @@ -95,29 +96,28 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ } switch (RedisType.parse(tableInfo.getRedisType())){ case STANDALONE: - StringBuilder redisUri = new StringBuilder(); - redisUri.append("redis://").append(url).append("/").append(database); - redisClient = RedisClient.create(redisUri.toString()); + RedisURI redisURI = RedisURI.create("redis://" + url); + redisURI.setPassword(password); + redisURI.setDatabase(Integer.valueOf(database)); + redisClient = RedisClient.create(redisURI); connection = redisClient.connect(); async = connection.async(); - ((RedisAsyncCommands)async).auth(password); break; case SENTINEL: - StringBuilder sentinelUri = new StringBuilder(); - sentinelUri.append("redis-sentinel://") - .append(url).append("/").append(database).append("#").append(redisSideTableInfo.getMasterName()); - redisClient = RedisClient.create(sentinelUri.toString()); + RedisURI redisSentinelURI = RedisURI.create("redis-sentinel://" + url); + redisSentinelURI.setPassword(password); + redisSentinelURI.setDatabase(Integer.valueOf(database)); + redisSentinelURI.setSentinelMasterId(redisSideTableInfo.getMasterName()); + redisClient = RedisClient.create(redisSentinelURI); connection = redisClient.connect(); async = connection.async(); - ((RedisAsyncCommands)async).auth(password); break; case CLUSTER: - StringBuilder clusterUri = new StringBuilder(); - clusterUri.append("redis://").append(url); - clusterClient = RedisClusterClient.create(clusterUri.toString()); + RedisURI clusterURI = RedisURI.create("redis://" + url); + clusterURI.setPassword(password); + clusterClient = RedisClusterClient.create(clusterURI); clusterConnection = clusterClient.connect(); async = clusterConnection.async(); - ((RedisAdvancedClusterAsyncCommands)async).auth(password); default: break; } From 44858b19b1190f2baf203ad3d9d4c8da4cb31938 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 26 Oct 2020 09:36:38 +0800 Subject: [PATCH 476/523] [feat] impala and kudu with kerberos [feat] impala support dynamic and static partition [feat] complete kudu with kerberos [fix] import and override error [feat] impala with kerberos --- .../sql/constrant/PluginParamConsts.java | 30 + .../dtstack/flink/sql/krb/KerberosTable.java | 65 ++ .../flink/sql/side/BaseAsyncReqRow.java | 7 +- .../com/dtstack/flink/sql/util/KrbUtils.java | 54 ++ .../dtstack/flink/sql/util/KrbUtilsTest.java | 44 + docs/plugin/kuduSide.md | 22 + docs/plugin/kuduSink.md | 149 +++- .../sql/side/impala/ImpalaAllReqRow.java | 37 +- .../sql/side/impala/ImpalaAsyncReqRow.java | 79 +- .../sql/sink/impala/ImpalaOutputFormat.java | 754 ++++++++++++++++++ .../flink/sql/sink/impala/ImpalaSink.java | 206 +++-- .../sink/impala/table/ImpalaSinkParser.java | 150 ++-- .../sink/impala/table/ImpalaTableInfo.java | 172 +++- .../flink/sql/side/kudu/KuduAllReqRow.java | 52 +- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 57 +- .../sql/side/kudu/table/KuduSideParser.java | 13 + .../side/kudu/table/KuduSideTableInfo.java | 51 +- .../flink/sql/sink/kudu/KuduOutputFormat.java | 52 +- .../dtstack/flink/sql/sink/kudu/KuduSink.java | 20 +- .../sql/sink/kudu/table/KuduSinkParser.java | 13 + .../sql/sink/kudu/table/KuduTableInfo.java | 53 +- .../sql/side/rdb/async/RdbAsyncReqRow.java | 112 ++- .../sql/sink/rdb/JDBCTypeConvertUtils.java | 55 ++ .../rdb/format/JDBCUpsertOutputFormat.java | 16 +- 24 files changed, 1965 insertions(+), 298 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/constrant/PluginParamConsts.java create mode 100644 core/src/main/java/com/dtstack/flink/sql/krb/KerberosTable.java create mode 100644 core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java create mode 100644 core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java create mode 100644 impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/PluginParamConsts.java b/core/src/main/java/com/dtstack/flink/sql/constrant/PluginParamConsts.java new file mode 100644 index 000000000..baf314c19 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/PluginParamConsts.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.constrant; + +/** + * @program: flinkStreamSQL + * @author: wuren + * @create: 2020/09/15 + **/ +public class PluginParamConsts { + public static final String PRINCIPAL = "principal"; + public static final String KEYTAB = "keytab"; + public static final String KRB5_CONF = "krb5conf"; +} diff --git a/core/src/main/java/com/dtstack/flink/sql/krb/KerberosTable.java b/core/src/main/java/com/dtstack/flink/sql/krb/KerberosTable.java new file mode 100644 index 000000000..ce6691294 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/krb/KerberosTable.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.krb; + +import com.google.common.base.Strings; + +/** + * @program: flinkStreamSQL + * @author: wuren + * @create: 2020/09/15 + **/ +public interface KerberosTable { + + String getPrincipal(); + + void setPrincipal(String principal); + + String getKeytab(); + + void setKeytab(String keytab); + + String getKrb5conf(); + + void setKrb5conf(String krb5conf); + + boolean isEnableKrb(); + + void setEnableKrb(boolean enableKrb); + + default void judgeKrbEnable() { + boolean allSet = + !Strings.isNullOrEmpty(getPrincipal()) && + !Strings.isNullOrEmpty(getKeytab()) && + !Strings.isNullOrEmpty(getKrb5conf()); + + boolean allNotSet = + Strings.isNullOrEmpty(getPrincipal()) && + Strings.isNullOrEmpty(getKeytab()) && + Strings.isNullOrEmpty(getKrb5conf()); + + if (allSet) { + setEnableKrb(true); + } else if (allNotSet) { + setEnableKrb(false); + } else { + throw new RuntimeException("Missing kerberos parameter! all kerberos params must be set, or all kerberos params are not set"); + } + } +} diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index ae8c75f7e..3b6a7f88c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -243,12 +243,7 @@ protected ScheduledFuture registerTimer(CRow input, ResultFuture result long timeoutTimestamp = sideInfo.getSideTableInfo().getAsyncTimeout() + getProcessingTimeService().getCurrentProcessingTime(); return getProcessingTimeService().registerTimer( timeoutTimestamp, - new ProcessingTimeCallback() { - @Override - public void onProcessingTime(long timestamp) throws Exception { - timeout(input, resultFuture); - } - }); + timestamp -> timeout(input, resultFuture)); } protected void cancelTimerWhenComplete(ResultFuture resultFuture, ScheduledFuture timerFuture){ diff --git a/core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java new file mode 100644 index 000000000..2a83359c4 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +/** + * @program: flinkStreamSQL + * @author: wuren + * @create: 2020/09/14 + **/ +public class KrbUtils { + + private static final Logger LOG = LoggerFactory.getLogger(KrbUtils.class); + + public static final String KRB5_CONF_KEY = "java.security.krb5.conf"; + public static final String HADOOP_AUTH_KEY = "hadoop.security.authentication"; + public static final String KRB_STR = "Kerberos"; +// public static final String FALSE_STR = "false"; +// public static final String SUBJECT_ONLY_KEY = "javax.security.auth.useSubjectCredsOnly"; + + public static UserGroupInformation getUgi(String principal, String keytabPath, String krb5confPath) throws IOException { + LOG.info("Kerberos login with principal: {} and keytab: {}", principal, keytabPath); + System.setProperty(KRB5_CONF_KEY, krb5confPath); + // TODO 尚未探索出此选项的意义，以后研究明白方可打开 +// System.setProperty(SUBJECT_ONLY_KEY, FALSE_STR); + Configuration configuration = new Configuration(); + configuration.set(HADOOP_AUTH_KEY , KRB_STR); + UserGroupInformation.setConfiguration(configuration); + return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytabPath); + } + +} diff --git a/core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java b/core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java new file mode 100644 index 000000000..6c7359e14 --- /dev/null +++ b/core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.util; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; + +/** + * @program: flinkStreamSQL + * @author: wuren + * @create: 2020/09/14 + **/ +public class KrbUtilsTest { + @Test + public void testGetUgi() throws IOException { + String principal = ""; + String keytabPath = ""; + String krb5confPath = ""; + try { + KrbUtils.getUgi(principal, keytabPath, krb5confPath); + } catch (IllegalArgumentException e) { + Assert.assertEquals(e.getMessage(), "Can't get Kerberos realm"); + } + + } +} \ No newline at end of file diff --git a/docs/plugin/kuduSide.md b/docs/plugin/kuduSide.md index a2636d4aa..3075c3f35 100644 --- a/docs/plugin/kuduSide.md +++ b/docs/plugin/kuduSide.md @@ -60,6 +60,11 @@ | isFaultTolerant |查询是否容错 查询失败是否扫描第二个副本 默认false 容错 | 否|| | cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| +| principal |kerberos用于登录的principal | 否|| +| keytab |keytab文件的路径 | 否|| +| krb5conf |conf文件路径 | 否|| +Kerberos三个参数全部设置则开启Kerberos认证，如果缺少任何一个则会提示缺少参数错误。 +如果全部未设置则不开启Kerberos连接Kudu集群。 -------------- ## 5.样例 @@ -163,3 +168,20 @@ into on t1.id = t2.id; ``` +## 7.kerberos示例 +``` +create table dim ( + name varchar, + id int, + PERIOD FOR SYSTEM_TIME +) WITH ( + type='kudu', + kuduMasters='host1', + tableName='foo', + parallelism ='1', + cache ='ALL', + keytab='foo/foobar.keytab', + krb5conf='bar/krb5.conf', + principal='kudu/host1@DTSTACK.COM' +); +``` \ No newline at end of file diff --git a/docs/plugin/kuduSink.md b/docs/plugin/kuduSink.md index cc8f8535a..7eb907770 100644 --- a/docs/plugin/kuduSink.md +++ b/docs/plugin/kuduSink.md @@ -11,7 +11,154 @@ CREATE TABLE tableName( writeMode='upsert', workerCount='1', defaultOperationTimeoutMs='600000', - defaultSocketReadTimeoutMs='6000000', + defaultSocketReadT## 1.格式： + ``` + CREATE TABLE tableName( + colName colType, + ... + colNameX colType + )WITH( + type ='kudu', + kuduMasters ='ip1,ip2,ip3', + tableName ='impala::default.test', + writeMode='upsert', + workerCount='1', + defaultOperationTimeoutMs='600000', + defaultSocketReadTimeoutMs='6000000', + parallelism ='parllNum' + ); + + + ``` + + ## 2.支持版本 + kudu 1.9.0+cdh6.2.0 + + ## 3.表结构定义 + + |参数名称|含义| + |----|---| + | tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称 + | colName | 列名称，redis中存储为 表名:主键名:主键值:列名]| + | colType | 列类型 [colType支持的类型](../colType.md)| + + + ## 4.参数： + + |参数名称|含义|是否必填|默认值| + |----|---|---|-----| + | type | 表名 输出表类型[mysq|hbase|elasticsearch|redis|kudu]|是|| + | kuduMasters | kudu master节点的地址;格式ip[ip，ip2]|是|| + | tableName | kudu 的表名称|是|| + | writeMode | 写入kudu的模式 insert|update|upsert |否 |upsert + | workerCount | 工作线程数 |否| + | defaultOperationTimeoutMs | 操作超时时间 |否| + | defaultSocketReadTimeoutMs | socket读取超时时间 |否| + | parallelism | 并行度设置|否|1| + | principal |kerberos用于登录的principal | 否|| + | keytab |keytab文件的路径 | 否|| + | krb5conf |conf文件路径 | 否|| + Kerberos三个参数全部设置则开启Kerberos认证，如果缺少任何一个则会提示缺少参数错误。 + 如果全部未设置则不开启Kerberos连接Kudu集群。 + + ## 5.样例： + ``` + CREATE TABLE MyTable( + channel varchar, + name varchar, + pv varchar, + a varchar, + b varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='es_test', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + + CREATE TABLE MyResult( + a string, + b string, + c string, + d string + )WITH( + type ='kudu', + kuduMasters ='cdh03.cdhsite:7051', + tableName ='myresult', + writeMode='insert', + parallelism ='1' + ); + + CREATE TABLE sideTable( + c string, + d string, + PRIMARY KEY(c) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='kudu', + kuduMasters ='cdh03.cdhsite:7051', + tableName ='sidetest4', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1', + primaryKey ='c', + isFaultTolerant ='false' + ); + + insert + into + MyResult + select + MyTable.a, + MyTable.b, + s.c, + s.d + from + MyTable + join + sideTable s + on MyTable.a = s.c + where + MyTable.a='2' + and s.d='2' + + ``` + + ## 6.数据示例 + ### 输入数据 + ``` + {"channel":"daishuyun","name":"roc","pv":"10","a":"2","b":"2"} + ``` + ### 结果数据 + ``` + {"a":"2","b":"2","c":"3","d":"4"} + ``` + + ## 7.kerberos示例 + ``` + create table dwd ( + name varchar, + id int + ) WITH ( + type='kudu', + kuduMasters='host1', + tableName='foo', + writeMode='insert', + parallelism ='1', + keytab='foo/foobar.keytab', + krb5conf='bar/krb5.conf', + principal='kudu/host1@DTSTACK.COM' + ); + ``` +imeoutMs='6000000', parallelism ='parllNum' ); diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java index 10938308a..179e52e56 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java @@ -18,21 +18,23 @@ package com.dtstack.flink.sql.side.impala; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.JDBCUtils; +import com.dtstack.flink.sql.util.KrbUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.security.PrivilegedAction; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.SQLException; import java.util.List; /** @@ -61,10 +63,26 @@ public ImpalaAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List) () -> { + try { + return DriverManager.getConnection(url); + } catch (SQLException e) { + e.printStackTrace(); + } + return null; + }); + } else { + connection = DriverManager.getConnection(url); + } connection.setAutoCommit(false); return connection; } catch (Exception e) { @@ -78,14 +96,11 @@ public String getUrl() throws IOException { String newUrl = ""; Integer authMech = impalaSideTableInfo.getAuthMech(); - StringBuffer urlBuffer = new StringBuffer(impalaSideTableInfo.getUrl()); + StringBuilder urlBuffer = new StringBuilder(impalaSideTableInfo.getUrl()); if (authMech == 0) { newUrl = urlBuffer.toString(); } else if (authMech == 1) { - String keyTabFilePath = impalaSideTableInfo.getKeyTabFilePath(); - String krb5FilePath = impalaSideTableInfo.getKrb5FilePath(); - String principal = impalaSideTableInfo.getPrincipal(); String krbRealm = impalaSideTableInfo.getKrbRealm(); String krbHostFQDN = impalaSideTableInfo.getKrbHostFQDN(); String krbServiceName = impalaSideTableInfo.getKrbServiceName(); @@ -96,11 +111,7 @@ public String getUrl() throws IOException { .concat("KrbServiceName=").concat(krbServiceName).concat(";") ); newUrl = urlBuffer.toString(); - System.setProperty("java.security.krb5.conf", krb5FilePath); - Configuration configuration = new Configuration(); - configuration.set("hadoop.security.authentication" , "Kerberos"); - UserGroupInformation.setConfiguration(configuration); - UserGroupInformation.loginUserFromKeytab(principal, keyTabFilePath); + } else if (authMech == 2) { String uName = impalaSideTableInfo.getUserName(); diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 3f4817580..ae8ac2be9 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -18,27 +18,32 @@ package com.dtstack.flink.sql.side.impala; -import com.dtstack.flink.sql.factory.DTThreadFactory; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; +import com.dtstack.flink.sql.util.KrbUtils; import io.vertx.core.Vertx; import io.vertx.core.VertxOptions; import io.vertx.core.json.JsonObject; import io.vertx.ext.jdbc.JDBCClient; +import io.vertx.ext.sql.SQLClient; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; +import java.security.PrivilegedAction; import java.util.List; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; /** * Date: 2019/11/12 @@ -53,6 +58,7 @@ public class ImpalaAsyncReqRow extends RdbAsyncReqRow { private final static String IMPALA_DRIVER = "com.cloudera.impala.jdbc41.Driver"; + protected UserGroupInformation ugi = null; public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ImpalaAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); @@ -60,9 +66,19 @@ public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List inputParams, + CRow input, + ResultFuture resultFuture, + SQLClient rdbSqlClient, + AtomicLong failCounter, + AtomicBoolean finishFlag, + CountDownLatch latch) { + if (ugi == null) { + doAsyncQueryData(inputParams, + input, resultFuture, + rdbSqlClient, + failCounter, + finishFlag, + latch); + } else { + // Kerberos + ugi.doAs((PrivilegedAction) () -> { + doAsyncQueryData(inputParams, + input, resultFuture, + rdbSqlClient, + failCounter, + finishFlag, + latch); + return null; + }); + } + } } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java new file mode 100644 index 000000000..19e904ef2 --- /dev/null +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -0,0 +1,754 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.impala; + +import com.dtstack.flink.sql.factory.DTThreadFactory; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.sink.rdb.JDBCTypeConvertUtils; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.util.JDBCUtils; +import com.dtstack.flink.sql.util.KrbUtils; +import com.google.common.collect.Maps; +import org.apache.commons.collections.CollectionUtils; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.types.Row; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.rmi.RemoteException; +import java.security.PrivilegedExceptionAction; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static com.dtstack.flink.sql.sink.rdb.JDBCTypeConvertUtils.setRecordToStatement; +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * Date: 2020/10/14 + * Company: www.dtstack.com + * + * @author tiezhu + */ +public class ImpalaOutputFormat extends AbstractDtRichOutputFormat> { + + private static final Logger LOG = LoggerFactory.getLogger(ImpalaOutputFormat.class); + + private static final long serialVersionUID = 1L; + + // ${field} + private static final Pattern STATIC_PARTITION_PATTERN = Pattern.compile("\\$\\{([^}]*)}"); + // cast(value as string) -> cast('value' as string) cast(value as timestamp) -> cast('value' as timestamp) + private static final Pattern TYPE_PATTERN = Pattern.compile("cast\\((.*) as (.*)\\)"); + //specific type which values need to be quoted + private static final String[] NEED_QUOTE_TYPE = {"string", "timestamp"}; + + private static final Integer DEFAULT_CONN_TIME_OUT = 60; + private static final int RECEIVE_DATA_PRINT_FREQUENCY = 1000; + private static final int DIRTY_DATA_PRINT_FREQUENCY = 1000; + + private static final String KUDU_TYPE = "kudu"; + private static final String UPDATE_MODE = "update"; + private static final String PARTITION_CONSTANT = "PARTITION"; + private static final String DRIVER_NAME = "com.cloudera.impala.jdbc41.Driver"; + + private static final String VALUES_CONDITION = "${valuesCondition}"; + private static final String PARTITION_CONDITION = "${partitionCondition}"; + private static final String TABLE_FIELDS_CONDITION = "${tableFieldsCondition}"; + private static final String NO_PARTITION = "noPartition"; + + protected transient Connection connection; + protected transient Statement statement; + protected transient PreparedStatement updateStatement; + + private transient volatile boolean closed = false; + private int batchCount = 0; + + // |------------------------------------------------| + // | partitionCondition |Array of valueCondition| + // |------------------------------------------------| + // | ptOne, ptTwo, ptThree | [(v1, v2, v3, v4, v5)]| DP + // |------------------------------------------------| + // | ptOne = v1, ptTwo = v2 | [(v3, v4, v5)] | SP + // |------------------------------------------------| + // | ptOne, ptTwo = v2 | [(v1, v3, v4, v5)] | DP and SP + // |------------------------------------------------| + // | noPartition | [(v1, v2, v3, v4, v5)]| kudu or disablePartition + // |------------------------------------------------| + private transient Map> rowDataMap; + + protected String keytabPath; + protected String krb5confPath; + protected String principal; + protected Integer authMech; + protected String dbUrl; + protected String userName; + protected String password; + protected int batchSize = 100; + protected long batchWaitInterval = 60 * 1000L; + protected String tableName; + protected List primaryKeys; + protected String partitionFields; + protected Boolean enablePartition; + protected String schema; + protected String storeType; + protected String updateMode; + public List fieldNames; + public List fieldTypes; + public List fieldExtraInfoList; + + // partition field of static partition which matched by ${field} + private final List staticPartitionFields = new ArrayList<>(); + + // valueFieldsName -> 重组之后的fieldNames，为了重组row data字段值对应 + // 需要对partition字段做特殊处理，比如原来的字段顺序为(age, name, id)，但是因为partition，写入的SQL为 + // INSERT INTO tableName(name, id) PARTITION(age) VALUES(?, ?, ?) + // 那么实际executeSql设置字段的顺序应该为(name, id, age)，同时，字段对应的type顺序也需要重组 + private List valueFieldNames; + private transient AbstractDtRichOutputFormat metricOutputFormat; + private List rows; + + private transient ScheduledExecutorService scheduler; + private transient ScheduledFuture scheduledFuture; + + @Override + public void configure(Configuration parameters) { + } + + @Override + public void open(int taskNumber, int numTasks) throws IOException { + try { + rowDataMap = new HashMap<>(); + rows = new ArrayList<>(); + metricOutputFormat = this; + openConnect(); + initScheduledTask(batchWaitInterval); + init(); + initMetric(); + } catch (Exception e) { + throw new RemoteException("impala output format open error!", e); + } + } + + private void init() throws SQLException { + if (Objects.nonNull(partitionFields)) { + // match ${field} from partitionFields + Matcher matcher = STATIC_PARTITION_PATTERN.matcher(partitionFields); + while (matcher.find()) { + LOG.info("find static partition field: {}", matcher.group(1)); + staticPartitionFields.add(matcher.group(1)); + } + } + + if (updateMode.equalsIgnoreCase(UPDATE_MODE)) { + if (!storeType.equalsIgnoreCase(KUDU_TYPE)) { + throw new IllegalArgumentException("update mode not support for non-kudu table!"); + } + + updateStatement = connection.prepareStatement(buildUpdateSql(schema, tableName, fieldNames, primaryKeys)); + return; + } + + valueFieldNames = rebuildFieldNameListAndTypeList(fieldNames, staticPartitionFields, fieldTypes, partitionFields); + } + + private void initScheduledTask(Long batchWaitInterval) { + if (batchWaitInterval != 0) { + this.scheduler = new ScheduledThreadPoolExecutor(1, + new DTThreadFactory("impala-upsert-output-format")); + this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(() -> { + synchronized (ImpalaOutputFormat.this) { + try { + flush(); + } catch (Exception e) { + LOG.error("Writing records to impala jdbc failed.", e); + throw new RuntimeException("Writing records to impala jdbc failed.", e); + } + } + }, batchWaitInterval, batchWaitInterval, TimeUnit.MILLISECONDS); + } + } + + private void openConnect() throws IOException { + if (authMech == 1) { + UserGroupInformation ugi = KrbUtils.getUgi(principal, keytabPath, krb5confPath); + try { + ugi.doAs((PrivilegedExceptionAction) () -> { + openJdbc(); + return null; + }); + } catch (InterruptedException | IOException e) { + throw new IllegalArgumentException("connect impala error!", e); + } + } else { + openJdbc(); + } + } + + /** + * get jdbc connection + */ + private void openJdbc() { + JDBCUtils.forName(DRIVER_NAME, getClass().getClassLoader()); + try { + connection = DriverManager.getConnection(dbUrl, userName, password); + statement = connection.createStatement(); + connection.setAutoCommit(false); + } catch (SQLException sqlException) { + throw new RuntimeException("get impala jdbc connection failed!", sqlException); + } + } + + private synchronized void flush() throws SQLException { + if (batchCount > 0) { + if (updateMode.equalsIgnoreCase(UPDATE_MODE)) { + executeUpdateBatch(); + } + if (!rowDataMap.isEmpty()) { + String templateSql = + "INSERT INTO tableName ${tableFieldsCondition} PARTITION ${partitionCondition} VALUES ${valuesCondition}"; + executeBatchSql( + statement, + templateSql, + schema, + tableName, + storeType, + enablePartition, + valueFieldNames, + partitionFields, + rowDataMap + ); + rowDataMap.clear(); + } + } + batchCount = 0; + + } + + /** + * execute batch update statement + * + * @throws SQLException throw sql exception + */ + private void executeUpdateBatch() throws SQLException { + try { + rows.forEach(row -> { + try { + JDBCTypeConvertUtils.setRecordToStatement( + updateStatement, + JDBCTypeConvertUtils.getSqlTypeFromFieldType(fieldTypes), + row, + primaryKeys.stream().mapToInt(fieldNames::indexOf).toArray() + ); + updateStatement.addBatch(); + } catch (Exception e) { + throw new RuntimeException("impala jdbc execute batch error!", e); + } + }); + updateStatement.executeBatch(); + connection.commit(); + rows.clear(); + } catch (Exception e) { + LOG.debug("impala jdbc execute batch error ", e); + connection.rollback(); + connection.commit(); + updateStatement.clearBatch(); + executeUpdate(connection); + } + } + + public void executeUpdate(Connection connection) { + rows.forEach(row -> { + try { + setRecordToStatement(updateStatement, JDBCTypeConvertUtils.getSqlTypeFromFieldType(fieldTypes), row); + updateStatement.executeUpdate(); + connection.commit(); + } catch (Exception e) { + try { + connection.rollback(); + connection.commit(); + } catch (SQLException e1) { + throw new RuntimeException(e1); + } + if (metricOutputFormat.outDirtyRecords.getCount() % DIRTY_DATA_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { + LOG.error("record insert failed ,this row is {}", row.toString()); + LOG.error("", e); + } + metricOutputFormat.outDirtyRecords.inc(); + } + }); + rows.clear(); + } + + private void putRowIntoMap(Map> rowDataMap, Tuple2 rowData) { + Set keySet = rowDataMap.keySet(); + ArrayList tempRowArray; + if (keySet.contains(rowData.f0)) { + tempRowArray = rowDataMap.get(rowData.f0); + } else { + tempRowArray = new ArrayList<>(); + } + tempRowArray.add(rowData.f1); + rowDataMap.put(rowData.f0, tempRowArray); + } + + private List rebuildFieldNameListAndTypeList(List fieldNames, + List staticPartitionFields, + List fieldTypes, + String partitionFields) { + if (partitionFields.isEmpty()) { + return fieldNames; + } + + List valueFields = new ArrayList<>(fieldNames); + + for (int i = valueFields.size() - 1; i >= 0; i--) { + if (staticPartitionFields.contains(fieldNames.get(i))) { + valueFields.remove(i); + fieldTypes.remove(i); + } + } + + for (int i = 0; i < valueFields.size(); i++) { + if (partitionFields.contains(fieldNames.get(i))) { + valueFields.add(valueFields.remove(i)); + fieldTypes.add(fieldTypes.remove(i)); + } + } + + return valueFields; + } + + /** + * Quote a specific type of value, like string, timestamp + * before: 1, cast(tiezhu as string), cast(2001-01-09 01:05:01 as timestamp), cast(123 as int) + * after: 1, cast('tiezhu' as string), cast('2001-01-09 01:05:01' as timestamp), cast(123 as int) + * if cast value is null, then cast(null as type) + * + * @param valueCondition original value condition + * @return quoted condition + */ + private String valueConditionAddQuotation(String valueCondition) { + final String[] valueConditionCopy = {valueCondition}; + String[] temps = valueCondition.split(","); + Arrays.stream(temps).forEach( + item -> { + Matcher matcher = TYPE_PATTERN.matcher(item); + while (matcher.find()) { + String value = matcher.group(1); + String type = matcher.group(2); + + if (Arrays.asList(NEED_QUOTE_TYPE).contains(type)) { + if (!"null".equals(value)) { + valueConditionCopy[0] = valueConditionCopy[0].replace(value, "'" + value + "'"); + } + } + } + } + ); + return "(" + valueConditionCopy[0] + ")"; + } + + @Override + public synchronized void writeRecord(Tuple2 record) throws IOException { + try { + if (!record.f0) { + return; + } + + if (outRecords.getCount() % RECEIVE_DATA_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { + LOG.info("Receive data : {}", record); + } + + if (updateMode.equalsIgnoreCase(UPDATE_MODE)) { + rows.add(Row.copy(record.f1)); + } else { + Map valueMap = Maps.newHashMap(); + Row row = Row.copy(record.f1); + + for (int i = 0; i < row.getArity(); i++) { + valueMap.put(fieldNames.get(i), row.getField(i)); + } + + Tuple2 rowTuple2 = new Tuple2<>(); + if (storeType.equalsIgnoreCase(KUDU_TYPE) || !enablePartition) { + rowTuple2.f0 = NO_PARTITION; + } else { + rowTuple2.f0 = buildPartitionCondition(valueMap, partitionFields, staticPartitionFields); + } + + // 根据字段名对 row data 重组, 比如，原始 row data : (1, xxx, 20) -> (id, name, age) + // 但是由于 partition，写入的field 顺序变成了 (name, id, age)，则需要对 row data 重组变成 (xxx, 1, 20) + Row rowValue = new Row(fieldTypes.size()); + for (int i = 0; i < fieldTypes.size(); i++) { + rowValue.setField(i, valueMap.get(valueFieldNames.get(i))); + } + rowTuple2.f1 = valueConditionAddQuotation(buildValuesCondition(fieldTypes, rowValue)); + putRowIntoMap(rowDataMap, rowTuple2); + } + + batchCount++; + + if (batchCount >= batchSize) { + flush(); + } + + // Receive data + outRecords.inc(); + } catch (Exception e) { + throw new RuntimeException("Writing records to impala failed.", e); + } + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + // 将还未执行的SQL flush + if (batchCount > 0) { + try { + flush(); + } catch (Exception e) { + throw new RuntimeException("Writing records to impala failed.", e); + } + } + // cancel scheduled task + if (this.scheduledFuture != null) { + scheduledFuture.cancel(false); + this.scheduler.shutdown(); + } + // close connection + try { + if (connection != null && connection.isValid(DEFAULT_CONN_TIME_OUT)) { + connection.close(); + } + + if (statement != null && !statement.isClosed()) { + statement.close(); + } + + if (updateStatement != null && !updateStatement.isClosed()) { + updateStatement.close(); + } + } catch (SQLException e) { + throw new RemoteException("impala connection close failed!"); + } finally { + connection = null; + statement = null; + updateStatement = null; + } + closed = true; + } + + /** + * execute batch sql from row data map + * sql like 'insert into tableName(f1, f2, f3) ${partitionCondition} values(v1, v2, v3), (v4, v5, v6).... + * + * @param statement execute statement + * @param tempSql template sql + * @param storeType the store type of data + * @param enablePartition enable partition or not + * @param fieldNames field name list + * @param partitionFields partition fields + * @param rowDataMap row data map + * @throws SQLException throw sql exception + */ + private synchronized void executeBatchSql(Statement statement, + String tempSql, + String schema, + String tableName, + String storeType, + Boolean enablePartition, + List fieldNames, + String partitionFields, + Map> rowDataMap) throws SQLException { + StringBuilder valuesCondition = new StringBuilder(); + StringBuilder partitionCondition = new StringBuilder(); + String tableFieldsCondition = buildTableFieldsCondition(fieldNames, partitionFields); + ArrayList rowData; + String tableNameInfo = Objects.isNull(schema) ? + tableName : quoteIdentifier(schema) + "." + tableName; + tempSql = tempSql.replace("tableName", tableNameInfo); + + // kudu ${partitionCondition} is null + if (storeType.equalsIgnoreCase(KUDU_TYPE) || !enablePartition) { + rowData = rowDataMap.get(NO_PARTITION); + rowData.forEach(row -> valuesCondition.append(row).append(", ")); + String executeSql = tempSql.replace(VALUES_CONDITION, valuesCondition.toString()) + .replace(PARTITION_CONDITION, partitionCondition.toString()) + .replace(PARTITION_CONSTANT, "") + .replace(TABLE_FIELDS_CONDITION, tableFieldsCondition); + String substring = executeSql.substring(0, executeSql.length() - 2); + statement.execute(substring); + return; + } + + // partition sql + Set keySet = rowDataMap.keySet(); + String finalTempSql = tempSql; + keySet.forEach(key -> { + try { + String executeSql = String.copyValueOf(finalTempSql.toCharArray()); + ArrayList valuesConditionList = rowDataMap.get(key); + partitionCondition.append(key); + executeSql = executeSql.replace(PARTITION_CONDITION, partitionCondition.toString()) + .replace(TABLE_FIELDS_CONDITION, tableFieldsCondition) + .replace(VALUES_CONDITION, String.join(", ", valuesConditionList)); + statement.execute(executeSql); + partitionCondition.delete(0, partitionCondition.length()); + } catch (SQLException sqlException) { + throw new RuntimeException("execute impala partition SQL error! ", sqlException); + } + }); + } + + /** + * build partition condition with row data + * + * @param rowData row data + * @param partitionFields partition fields + * @param staticPartitionField static partition fields + * @return condition like '(ptOne, ptTwo=v2)' + */ + private String buildPartitionCondition(Map rowData, String partitionFields, List staticPartitionField) { + for (String key : staticPartitionField) { + StringBuilder sb = new StringBuilder(); + Object value = rowData.get(key); + sb.append(key).append("=").append(value); + partitionFields = partitionFields.replace("${" + key + "}", sb.toString()); + } + return "(" + partitionFields + ")"; + } + + /** + * build field condition according to field names + * replace ${tableFieldCondition} + * + * @param fieldNames the selected field names + * @param partitionFields the partition fields + * @return condition like '(id, name, age)' + */ + private String buildTableFieldsCondition(List fieldNames, String partitionFields) { + return "(" + fieldNames.stream() + .filter(f -> !partitionFields.contains(f)) + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")) + ")"; + } + + /** + * according to field types, build the values condition + * replace ${valuesCondition} + * + * @param fieldTypes field types + * @return condition like '(?, ?, cast(? as string))' and '?' will be replaced with row data + */ + private String buildValuesCondition(List fieldTypes, Row row) { + String valuesCondition = fieldTypes.stream().map( + f -> { + if (Arrays.asList(NEED_QUOTE_TYPE).contains(f.toLowerCase())) { + return String.format("cast(? as %s)", f.toLowerCase()); + } + return "?"; + }).collect(Collectors.joining(", ")); + for (int i = 0; i < row.getArity(); i++) { + valuesCondition = valuesCondition.replaceFirst("\\?", Objects.isNull(row.getField(i)) ? "null" : row.getField(i).toString()); + } + return valuesCondition; + } + + /** + * impala update mode SQL + * + * @return UPDATE tableName SET setCondition WHERE whereCondition + */ + private String buildUpdateSql(String schema, String tableName, List fieldNames, List primaryKeys) { + //跳过primary key字段 + String setClause = fieldNames.stream() + .filter(f -> !CollectionUtils.isNotEmpty(primaryKeys) || !primaryKeys.contains(f)) + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(", ")); + + String conditionClause = primaryKeys.stream() + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(" AND ")); + + return "UPDATE " + (Objects.isNull(schema) ? "" : quoteIdentifier(schema) + ".") + + quoteIdentifier(tableName) + " SET " + setClause + " WHERE " + conditionClause; + } + + private String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } + + public static Builder getImpalaBuilder() { + return new Builder(); + } + + public static class Builder { + private final ImpalaOutputFormat format = new ImpalaOutputFormat(); + + public Builder setDbUrl(String dbUrl) { + format.dbUrl = dbUrl; + return this; + } + + public Builder setUserName(String userName) { + format.userName = userName; + return this; + } + + public Builder setPassword(String password) { + format.password = password; + return this; + } + + public Builder setBatchSize(Integer batchSize) { + format.batchSize = batchSize; + return this; + } + + public Builder setBatchWaitInterval(Long batchWaitInterval) { + format.batchWaitInterval = batchWaitInterval; + return this; + } + + public Builder setTableName(String tableName) { + format.tableName = tableName; + return this; + } + + public Builder setPartitionFields(String partitionFields) { + format.partitionFields = Objects.isNull(partitionFields) ? + "" : partitionFields; + return this; + } + + public Builder setPrimaryKeys(List primaryKeys) { + format.primaryKeys = primaryKeys; + return this; + } + + public Builder setSchema(String schema) { + format.schema = schema; + return this; + } + + public Builder setEnablePartition(Boolean enablePartition) { + format.enablePartition = enablePartition; + return this; + } + + public Builder setUpdateMode(String updateMode) { + format.updateMode = updateMode; + return this; + } + + public Builder setFieldList(List fieldList) { + format.fieldNames = fieldList; + return this; + } + + public Builder setFieldTypeList(List fieldTypeList) { + format.fieldTypes = fieldTypeList; + return this; + } + + public Builder setStoreType(String storeType) { + format.storeType = storeType; + return this; + } + + public Builder setFieldExtraInfoList(List fieldExtraInfoList) { + format.fieldExtraInfoList = fieldExtraInfoList; + return this; + } + + public Builder setKeyTabPath(String keyTabPath) { + format.keytabPath = keyTabPath; + return this; + } + + public Builder setKrb5ConfPath(String krb5ConfPath) { + format.krb5confPath = krb5ConfPath; + return this; + } + + public Builder setPrincipal(String principal) { + format.principal = principal; + return this; + } + + public Builder setAuthMech(Integer authMech) { + format.authMech = authMech; + return this; + } + + private boolean canHandle(String url) { + return url.startsWith("jdbc:impala:"); + } + + public ImpalaOutputFormat build() { + if (!canHandle(format.dbUrl)) { + throw new IllegalArgumentException("impala dbUrl is illegal, check url: " + format.dbUrl); + } + + if (format.authMech == EAuthMech.Kerberos.getType()) { + checkNotNull(format.krb5confPath, + "When kerberos authentication is enabled, krb5confPath is required！"); + checkNotNull(format.principal, + "When kerberos authentication is enabled, principal is required！"); + checkNotNull(format.keytabPath, + "When kerberos authentication is enabled, keytabPath is required！"); + } + + if (format.authMech == EAuthMech.UserName.getType()) { + checkNotNull(format.userName, "userName is required!"); + } + + if (format.authMech == EAuthMech.NameANDPassword.getType()) { + checkNotNull(format.userName, "userName is required!"); + checkNotNull(format.password, "password is required!"); + } + + return format; + } + + } + +} diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index 34101d410..6d28f0173 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -20,64 +20,115 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.impala.table.ImpalaTableInfo; -import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; -import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.UserGroupInformation; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.DataStreamSink; +import org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction; +import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; +import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.types.Row; -import java.io.IOException; +import java.util.List; +import java.util.Objects; /** - * Date: 2019/11/11 + * Date: 2020/10/14 * Company: www.dtstack.com * - * @author xiuzhu + * @author tiezhu */ +public class ImpalaSink implements RetractStreamTableSink, IStreamSinkGener { -public class ImpalaSink extends AbstractRdbSink implements IStreamSinkGener { + private static final String DEFAULT_STORE_TYPE = "kudu"; - private ImpalaTableInfo impalaTableInfo; + protected String[] fieldNames; + TypeInformation[] fieldTypes; + protected String dbUrl; + protected String userName; + protected String password; + protected Integer authMech; + + protected String keytabPath; + protected String krb5confPath; + protected String principal; + + protected int batchSize = 100; + protected long batchWaitInterval = 60 * 1000L; + protected String tableName; + protected String registerTabName; + protected String storeType; + + protected List primaryKeys; + private int parallelism = 1; + protected String schema; + protected String updateMode; + protected Boolean enablePartition; + public List fieldList; + public List fieldTypeList; + public List fieldExtraInfoList; + protected String partitionFields; public ImpalaSink() { - super(null); + // do Nothing } @Override - public JDBCUpsertOutputFormat getOutputFormat() { - JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(getImpalaJdbcUrl()) - .setDialect(new ImpalaDialect(getFieldTypes(), primaryKeys)) - .setUsername(userName) - .setPassword(password) - .setTableName(tableName) - .build(); + public ImpalaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + ImpalaTableInfo impalaTableInfo = (ImpalaTableInfo) targetTableInfo; + this.dbUrl = getImpalaJdbcUrl(impalaTableInfo); + this.password = impalaTableInfo.getPassword(); + this.userName = impalaTableInfo.getUserName(); + this.authMech = impalaTableInfo.getAuthMech(); - return JDBCUpsertOutputFormat.builder() - .setOptions(jdbcOptions) - .setFieldNames(fieldNames) - .setFlushMaxSize(batchNum) - .setFlushIntervalMills(batchWaitInterval) - .setFieldTypes(sqlTypes) - .setKeyFields(primaryKeys) - .setPartitionFields(impalaTableInfo.getPartitionFields()) - .setAllReplace(allReplace) - .setUpdateMode(updateMode) - .build(); - } + this.principal = impalaTableInfo.getPrincipal(); + this.keytabPath = impalaTableInfo.getKeyTabFilePath(); + this.krb5confPath = impalaTableInfo.getKrb5FilePath(); + + this.updateMode = Objects.isNull(impalaTableInfo.getUpdateMode()) ? + "append" : impalaTableInfo.getUpdateMode(); + + this.batchSize = Objects.isNull(impalaTableInfo.getBatchSize()) ? + batchSize : impalaTableInfo.getBatchSize(); + this.batchWaitInterval = Objects.isNull(impalaTableInfo.getBatchWaitInterval()) ? + batchWaitInterval : impalaTableInfo.getBatchWaitInterval(); + this.parallelism = Objects.isNull(impalaTableInfo.getParallelism()) ? + parallelism : impalaTableInfo.getParallelism(); + this.registerTabName = impalaTableInfo.getTableName(); + this.fieldList = impalaTableInfo.getFieldList(); + this.fieldTypeList = impalaTableInfo.getFieldTypeList(); + this.fieldExtraInfoList = impalaTableInfo.getFieldExtraInfoList(); + this.tableName = impalaTableInfo.getTableName(); + this.schema = impalaTableInfo.getSchema(); + this.primaryKeys = impalaTableInfo.getPrimaryKeys(); + this.partitionFields = impalaTableInfo.getPartitionFields(); - public String getImpalaJdbcUrl() { + this.storeType = Objects.isNull(impalaTableInfo.getStoreType()) ? + DEFAULT_STORE_TYPE : impalaTableInfo.getStoreType(); + this.enablePartition = impalaTableInfo.isEnablePartition(); + + return this; + } + + /** + * build Impala Jdbc Url according to authMech + * + * @param impalaTableInfo impala table info + * @return jdbc url with auth mech info + */ + public String getImpalaJdbcUrl(ImpalaTableInfo impalaTableInfo) { Integer authMech = impalaTableInfo.getAuthMech(); - String newUrl = dbUrl; - StringBuffer urlBuffer = new StringBuffer(dbUrl); + String newUrl = impalaTableInfo.getUrl(); + StringBuilder urlBuffer = new StringBuilder(impalaTableInfo.getUrl()); if (authMech == EAuthMech.NoAuthentication.getType()) { return newUrl; } else if (authMech == EAuthMech.Kerberos.getType()) { - String keyTabFilePath = impalaTableInfo.getKeyTabFilePath(); - String krb5FilePath = impalaTableInfo.getKrb5FilePath(); - String principal = impalaTableInfo.getPrincipal(); String krbRealm = impalaTableInfo.getKrbRealm(); String krbHostFqdn = impalaTableInfo.getKrbHostFQDN(); String krbServiceName = impalaTableInfo.getKrbServiceName(); @@ -88,21 +139,10 @@ public String getImpalaJdbcUrl() { .concat("KrbServiceName=").concat(krbServiceName).concat(";") ); newUrl = urlBuffer.toString(); - - System.setProperty("java.security.krb5.conf", krb5FilePath); - Configuration configuration = new Configuration(); - configuration.set("hadoop.security.authentication", "Kerberos"); - UserGroupInformation.setConfiguration(configuration); - try { - UserGroupInformation.loginUserFromKeytab(principal, keyTabFilePath); - } catch (IOException e) { - throw new RuntimeException("loginUserFromKeytab error ..", e); - } - } else if (authMech == EAuthMech.UserName.getType()) { urlBuffer.append(";" .concat("AuthMech=3;") - .concat("UID=").concat(userName).concat(";") + .concat("UID=").concat(impalaTableInfo.getUserName()).concat(";") .concat("PWD=;") .concat("UseSasl=0") ); @@ -110,8 +150,8 @@ public String getImpalaJdbcUrl() { } else if (authMech == EAuthMech.NameANDPassword.getType()) { urlBuffer.append(";" .concat("AuthMech=3;") - .concat("UID=").concat(userName).concat(";") - .concat("PWD=").concat(password) + .concat("UID=").concat(impalaTableInfo.getUserName()).concat(";") + .concat("PWD=").concat(impalaTableInfo.getPassword()) ); newUrl = urlBuffer.toString(); } else { @@ -120,11 +160,69 @@ public String getImpalaJdbcUrl() { return newUrl; } + private ImpalaOutputFormat buildImpalaOutputFormat() { + + return ImpalaOutputFormat.getImpalaBuilder() + .setDbUrl(dbUrl) + .setPassword(password) + .setUserName(userName) + .setSchema(schema) + .setTableName(tableName) + .setUpdateMode(updateMode) + .setBatchSize(batchSize) + .setBatchWaitInterval(batchWaitInterval) + .setPrimaryKeys(primaryKeys) + .setPartitionFields(partitionFields) + .setFieldList(fieldList) + .setFieldTypeList(fieldTypeList) + .setFieldExtraInfoList(fieldExtraInfoList) + .setStoreType(storeType) + .setEnablePartition(enablePartition) + .setUpdateMode(updateMode) + .setAuthMech(authMech) + .setKeyTabPath(keytabPath) + .setKrb5ConfPath(krb5confPath) + .setPrincipal(principal) + .build(); + } + + @Override + public void emitDataStream(DataStream> dataStream) { + consumeDataStream(dataStream); + } + + public void consumeDataStream(DataStream> dataStream) { + ImpalaOutputFormat outputFormat = buildImpalaOutputFormat(); + RichSinkFunction> richSinkFunction = new OutputFormatSinkFunction(outputFormat); + dataStream.addSink(richSinkFunction) + .setParallelism(parallelism) + .name(tableName); + } + @Override - public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { - super.genStreamSink(targetTableInfo); - this.impalaTableInfo = (ImpalaTableInfo) targetTableInfo; + public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; return this; } + @Override + public TupleTypeInfo> getOutputType() { + return new TupleTypeInfo<>(org.apache.flink.table.api.Types.BOOLEAN(), getRecordType()); + } + + @Override + public TypeInformation getRecordType() { + return new RowTypeInfo(fieldTypes, fieldNames); + } + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java index 0d4857809..f1a475f61 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java @@ -18,109 +18,125 @@ package com.dtstack.flink.sql.sink.impala.table; -import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; import com.dtstack.flink.sql.util.MathUtil; -import java.math.BigDecimal; -import java.sql.Timestamp; import java.util.Arrays; import java.util.List; import java.util.Map; /** - * Reason: - * Date: 2019/11/11 + * Date: 2020/10/14 * Company: www.dtstack.com * - * @author xiuzhu + * @author tiezhu */ +public class ImpalaSinkParser extends AbstractTableParser { -public class ImpalaSinkParser extends RdbSinkParser { + private static final String PARALLELISM_KEY = "parallelism"; + + private static final String AUTH_MECH_KEY = "authMech"; + + private static final String KRB5FILEPATH_KEY = "krb5FilePath"; + + private static final String PRINCIPAL_KEY = "principal"; + + private static final String KEY_TAB_FILE_PATH_KEY = "keyTabFilePath"; + + private static final String KRB_REALM_KEY = "krbRealm"; + + private static final String KRB_HOST_FQDN_KEY = "krbHostFQDN"; + + private static final String KRB_SERVICE_NAME_KEY = "krbServiceName"; + + private static final String ENABLE_PARTITION_KEY = "enablePartition"; + + private static final String PARTITION_FIELDS_KEY = "partitionFields"; + + private static final String URL_KEY = "url"; + + private static final String TABLE_NAME_KEY = "tableName"; + + private static final String USER_NAME_KEY = "userName"; + + private static final String PASSWORD_KEY = "password"; + + private static final String BATCH_SIZE_KEY = "batchSize"; + + private static final String BATCH_WAIT_INTERVAL_KEY = "batchWaitInterval"; + + private static final String BUFFER_SIZE_KEY = "bufferSize"; + + private static final String FLUSH_INTERVAL_MS_KEY = "flushIntervalMs"; + + private static final String SCHEMA_KEY = "schema"; + + private static final String UPDATE_KEY = "updateMode"; + + private static final String KUDU_TYPE = "kudu"; + + private static final String STORE_TYPE_KEY = "storeType"; + + private static final String KRB_DEFAULT_REALM = "HADOOP.COM"; + + private static final String CURRENT_TYPE = "impala"; - private static final String CURR_TYPE = "impala"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { ImpalaTableInfo impalaTableInfo = new ImpalaTableInfo(); impalaTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, impalaTableInfo); - impalaTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(ImpalaTableInfo.PARALLELISM_KEY.toLowerCase()))); - impalaTableInfo.setUrl(MathUtil.getString(props.get(ImpalaTableInfo.URL_KEY.toLowerCase()))); - impalaTableInfo.setTableName(MathUtil.getString(props.get(ImpalaTableInfo.TABLE_NAME_KEY.toLowerCase()))); - impalaTableInfo.setBatchSize(MathUtil.getIntegerVal(props.get(ImpalaTableInfo.BATCH_SIZE_KEY.toLowerCase()))); - impalaTableInfo.setBatchWaitInterval(MathUtil.getLongVal(props.get(ImpalaTableInfo.BATCH_WAIT_INTERVAL_KEY.toLowerCase()))); - impalaTableInfo.setBufferSize(MathUtil.getString(props.get(ImpalaTableInfo.BUFFER_SIZE_KEY.toLowerCase()))); - impalaTableInfo.setFlushIntervalMs(MathUtil.getString(props.get(ImpalaTableInfo.FLUSH_INTERVALMS_KEY.toLowerCase()))); - impalaTableInfo.setSchema(MathUtil.getString(props.get(ImpalaTableInfo.SCHEMA_KEY.toLowerCase()))); - impalaTableInfo.setUpdateMode(MathUtil.getString(props.get(ImpalaTableInfo.UPDATE_KEY.toLowerCase()))); - - Integer authMech = MathUtil.getIntegerVal(props.get(ImpalaTableInfo.AUTHMECH_KEY.toLowerCase())); + impalaTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(PARALLELISM_KEY.toLowerCase()))); + impalaTableInfo.setUrl(MathUtil.getString(props.get(URL_KEY.toLowerCase()))); + impalaTableInfo.setTableName(MathUtil.getString(props.get(TABLE_NAME_KEY.toLowerCase()))); + impalaTableInfo.setBatchSize(MathUtil.getIntegerVal(props.get(BATCH_SIZE_KEY.toLowerCase()))); + impalaTableInfo.setBatchWaitInterval(MathUtil.getLongVal(props.get(BATCH_WAIT_INTERVAL_KEY.toLowerCase()))); + impalaTableInfo.setBufferSize(MathUtil.getString(props.get(BUFFER_SIZE_KEY.toLowerCase()))); + impalaTableInfo.setFlushIntervalMs(MathUtil.getString(props.get(FLUSH_INTERVAL_MS_KEY.toLowerCase()))); + impalaTableInfo.setSchema(MathUtil.getString(props.get(SCHEMA_KEY.toLowerCase()))); + impalaTableInfo.setUpdateMode(MathUtil.getString(props.get(UPDATE_KEY.toLowerCase()))); + + Integer authMech = MathUtil.getIntegerVal(props.get(AUTH_MECH_KEY.toLowerCase())); authMech = authMech == null ? 0 : authMech; impalaTableInfo.setAuthMech(authMech); - List authMechs = Arrays.asList(new Integer[]{0, 1, 2, 3}); + List authMechs = Arrays.asList(0, 1, 2, 3); if (!authMechs.contains(authMech)) { throw new IllegalArgumentException("The value of authMech is illegal, Please select 0, 1, 2, 3"); } else if (authMech == 1) { - impalaTableInfo.setPrincipal(MathUtil.getString(props.get(ImpalaTableInfo.PRINCIPAL_KEY.toLowerCase()))); - impalaTableInfo.setKeyTabFilePath(MathUtil.getString(props.get(ImpalaTableInfo.KEYTABFILEPATH_KEY.toLowerCase()))); - impalaTableInfo.setKrb5FilePath(MathUtil.getString(props.get(ImpalaTableInfo.KRB5FILEPATH_KEY.toLowerCase()))); - String krbRealm = MathUtil.getString(props.get(ImpalaTableInfo.KRBREALM_KEY.toLowerCase())); - krbRealm = krbRealm == null ? "HADOOP.COM" : krbRealm; + impalaTableInfo.setPrincipal(MathUtil.getString(props.get(PRINCIPAL_KEY.toLowerCase()))); + impalaTableInfo.setKeyTabFilePath(MathUtil.getString(props.get(KEY_TAB_FILE_PATH_KEY.toLowerCase()))); + impalaTableInfo.setKrb5FilePath(MathUtil.getString(props.get(KRB5FILEPATH_KEY.toLowerCase()))); + String krbRealm = MathUtil.getString(props.get(KRB_REALM_KEY.toLowerCase())); + krbRealm = krbRealm == null ? KRB_DEFAULT_REALM : krbRealm; impalaTableInfo.setKrbRealm(krbRealm); - impalaTableInfo.setKrbHostFQDN(MathUtil.getString(props.get(ImpalaTableInfo.KRBHOSTFQDN_KEY.toLowerCase()))); - impalaTableInfo.setKrbServiceName(MathUtil.getString(props.get(ImpalaTableInfo.KRBSERVICENAME_KEY.toLowerCase()))); + impalaTableInfo.setKrbHostFQDN(MathUtil.getString(props.get(KRB_HOST_FQDN_KEY.toLowerCase()))); + impalaTableInfo.setKrbServiceName(MathUtil.getString(props.get(KRB_SERVICE_NAME_KEY.toLowerCase()))); } else if (authMech == 2) { - impalaTableInfo.setUserName(MathUtil.getString(props.get(ImpalaTableInfo.USER_NAME_KEY.toLowerCase()))); + impalaTableInfo.setUserName(MathUtil.getString(props.get(USER_NAME_KEY.toLowerCase()))); } else if (authMech == 3) { - impalaTableInfo.setUserName(MathUtil.getString(props.get(ImpalaTableInfo.USER_NAME_KEY.toLowerCase()))); - impalaTableInfo.setPassword(MathUtil.getString(props.get(ImpalaTableInfo.PASSWORD_KEY.toLowerCase()))); + impalaTableInfo.setUserName(MathUtil.getString(props.get(USER_NAME_KEY.toLowerCase()))); + impalaTableInfo.setPassword(MathUtil.getString(props.get(PASSWORD_KEY.toLowerCase()))); } - String enablePartitionStr = (String) props.get(ImpalaTableInfo.ENABLEPARITION_KEY.toLowerCase()); + String storeType = MathUtil.getString(props.get(STORE_TYPE_KEY.toLowerCase())); + impalaTableInfo.setStoreType(storeType); + + String enablePartitionStr = (String) props.get(ENABLE_PARTITION_KEY.toLowerCase()); boolean enablePartition = MathUtil.getBoolean(enablePartitionStr == null ? "false" : enablePartitionStr); impalaTableInfo.setEnablePartition(enablePartition); - if (enablePartition) { - String partitionFields = MathUtil.getString(props.get(ImpalaTableInfo.PARTITIONFIELDS_KEY.toLowerCase())); + + if (!storeType.equalsIgnoreCase(KUDU_TYPE) && enablePartition) { + String partitionFields = MathUtil.getString(props.get(PARTITION_FIELDS_KEY.toLowerCase())); impalaTableInfo.setPartitionFields(partitionFields); } + impalaTableInfo.setType(CURRENT_TYPE); + impalaTableInfo.check(); return impalaTableInfo; } - - @Override - public Class dbTypeConvertToJavaType(String fieldType) { - switch (fieldType.toLowerCase()) { - case "boolean": - return Boolean.class; - case "char": - return Character.class; - case "double": - return Double.class; - case "float": - return Float.class; - case "tinyint": - return Byte.class; - case "smallint": - return Short.class; - case "int": - return Integer.class; - case "bigint": - return Long.class; - case "decimal": - return BigDecimal.class; - case "string": - case "varchar": - return String.class; - case "timestamp": - return Timestamp.class; - default: - break; - } - - throw new RuntimeException("不支持 " + fieldType + " 类型"); - } } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaTableInfo.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaTableInfo.java index 032d9ea18..bc0dbac23 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaTableInfo.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaTableInfo.java @@ -16,43 +16,46 @@ * limitations under the License. */ - package com.dtstack.flink.sql.sink.impala.table; -import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; +import com.dtstack.flink.sql.enums.EUpdateMode; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; +import java.util.Objects; + /** - * Date: 2019/11/13 + * Date: 2020/10/14 * Company: www.dtstack.com * - * @author xiuzhu + * @author tiezhu */ +public class ImpalaTableInfo extends AbstractTargetTableInfo { -public class ImpalaTableInfo extends RdbTableInfo { + private static final int MAX_BATCH_SIZE = 10000; - public static final String AUTHMECH_KEY = "authMech"; + private String url; - public static final String KRB5FILEPATH_KEY = "krb5FilePath"; + private String tableName; - public static final String PRINCIPAL_KEY = "principal"; + private String userName; - public static final String KEYTABFILEPATH_KEY = "keyTabFilePath"; + private String password; - public static final String KRBREALM_KEY = "krbRealm"; + private Integer batchSize; - public static final String KRBHOSTFQDN_KEY = "krbHostFQDN"; + private Long batchWaitInterval; - public static final String KRBSERVICENAME_KEY = "krbServiceName"; + private String bufferSize; - public static final String ENABLEPARITION_KEY = "enablePartition"; + private String flushIntervalMs; - public static final String PARTITIONFIELDS_KEY = "partitionFields"; + private String schema; - private static final String CURR_TYPE = "impala"; + private boolean allReplace; - private static final String PARTITION_FIELD_SPLIT_REGEX = ","; + private String updateMode; private Integer authMech; @@ -68,12 +71,90 @@ public class ImpalaTableInfo extends RdbTableInfo { private String krbServiceName; - private boolean enablePartition; + private boolean enablePartition = false; + + private String partitionFields; + + private String storeType; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public Integer getBatchSize() { + return batchSize; + } + + public void setBatchSize(Integer batchSize) { + this.batchSize = batchSize; + } + + public Long getBatchWaitInterval() { + return batchWaitInterval; + } + + public void setBatchWaitInterval(Long batchWaitInterval) { + this.batchWaitInterval = batchWaitInterval; + } + + public String getBufferSize() { + return bufferSize; + } + + public void setBufferSize(String bufferSize) { + this.bufferSize = bufferSize; + } + + public String getFlushIntervalMs() { + return flushIntervalMs; + } + + public void setFlushIntervalMs(String flushIntervalMs) { + this.flushIntervalMs = flushIntervalMs; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } - private String[] partitionFields; + public String getUpdateMode() { + return updateMode; + } - public ImpalaTableInfo() { - setType(CURR_TYPE); + public void setUpdateMode(String updateMode) { + this.updateMode = updateMode; } public Integer getAuthMech() { @@ -140,38 +221,49 @@ public void setEnablePartition(boolean enablePartition) { this.enablePartition = enablePartition; } - public String[] getPartitionFields() { + public String getPartitionFields() { return partitionFields; } public void setPartitionFields(String partitionFields) { - this.partitionFields = StringUtils.split(partitionFields, PARTITION_FIELD_SPLIT_REGEX); + this.partitionFields = partitionFields; + } + + public String getStoreType() { + return storeType; + } + + public void setStoreType(String storeType) { + this.storeType = storeType; + } + + @Override + public String getType() { + return super.getType().toLowerCase(); } @Override public boolean check() { - Preconditions.checkNotNull(this.getUrl(), "impala field of url is required"); - Preconditions.checkNotNull(this.getTableName(), "impala field of tableName is required"); - Preconditions.checkNotNull(this.getAuthMech(), "impala field of authMech is required"); - Integer authMech = getAuthMech(); - - if (authMech == 1) { - Preconditions.checkNotNull(this.getKrb5FilePath(), "impala field of krb5FilePath is required"); - Preconditions.checkNotNull(this.getPrincipal(), "impala field of principal is required"); - Preconditions.checkNotNull(this.getKeyTabFilePath(), "impala field of keyTabFilePath is required"); - Preconditions.checkNotNull(this.getKrbHostFQDN(), "impala field of krbHostFQDN is required"); - Preconditions.checkNotNull(this.getKrbServiceName(), "impala field of krbServiceName is required"); - } else if (authMech == 2) { - Preconditions.checkNotNull(this.getUserName(), "impala field of userName is required"); - }else if (authMech == 3) { - Preconditions.checkNotNull(this.getUserName(), "impala field of userName is required"); - Preconditions.checkNotNull(this.getPassword(), "impala field of password is required"); + Preconditions.checkNotNull(url, "impala field of URL is required"); + Preconditions.checkNotNull(tableName, "impala field of tableName is required"); + + if (Objects.nonNull(batchSize)) { + Preconditions.checkArgument(batchSize <= MAX_BATCH_SIZE, "batchSize must be less than " + MAX_BATCH_SIZE); } - if (isEnablePartition()) { - Preconditions.checkArgument(this.getPartitionFields().length > 0, "impala field of partitionFields is required"); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + Preconditions.checkArgument(Objects.nonNull(getPrimaryKeys()) && getPrimaryKeys().size() > 0, "updateMode mode primary is required"); } + if (Objects.nonNull(getPrimaryKeys())) { + getPrimaryKeys().forEach(pk -> { + Preconditions.checkArgument(getFieldList().contains(pk), "primary key " + pk + " not found in sink table field"); + }); + } + + + Preconditions.checkArgument(getFieldList().size() == getFieldExtraInfoList().size(), + "fields and fieldExtraInfoList attributes must be the same length"); return true; } } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index cf7f0dea7..0b30729f8 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -7,6 +7,7 @@ import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; +import com.dtstack.flink.sql.util.KrbUtils; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -18,6 +19,7 @@ import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.KuduClient; @@ -31,6 +33,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.security.PrivilegedAction; import java.sql.SQLException; import java.sql.Timestamp; import java.util.Arrays; @@ -218,24 +222,8 @@ private String buildKey(Map val, List equalFieldList) { private KuduScanner getConn(KuduSideTableInfo tableInfo) { try { if (client == null) { - String kuduMasters = tableInfo.getKuduMasters(); String tableName = tableInfo.getTableName(); - Integer workerCount = tableInfo.getWorkerCount(); - Integer defaultSocketReadTimeoutMs = tableInfo.getDefaultSocketReadTimeoutMs(); - Integer defaultOperationTimeoutMs = tableInfo.getDefaultOperationTimeoutMs(); - - Preconditions.checkNotNull(kuduMasters, "kuduMasters could not be null"); - - KuduClient.KuduClientBuilder kuduClientBuilder = new KuduClient.KuduClientBuilder(kuduMasters); - if (null != workerCount) { - kuduClientBuilder.workerCount(workerCount); - } - - if (null != defaultOperationTimeoutMs) { - kuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); - } - client = kuduClientBuilder.build(); - + client = getClient(tableInfo); if (!client.tableExists(tableName)) { throw new IllegalArgumentException("Table Open Failed , please check table exists"); } @@ -250,6 +238,36 @@ private KuduScanner getConn(KuduSideTableInfo tableInfo) { } } + private KuduClient getClient(KuduSideTableInfo tableInfo) throws IOException { + String kuduMasters = tableInfo.getKuduMasters(); + Integer workerCount = tableInfo.getWorkerCount(); + Integer defaultOperationTimeoutMs = tableInfo.getDefaultOperationTimeoutMs(); + + Preconditions.checkNotNull(kuduMasters, "kuduMasters could not be null"); + + KuduClient.KuduClientBuilder kuduClientBuilder = new KuduClient.KuduClientBuilder(kuduMasters); + + if (null != workerCount) { + kuduClientBuilder.workerCount(workerCount); + } + + if (null != defaultOperationTimeoutMs) { + kuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); + } + + if (tableInfo.isEnableKrb()) { + UserGroupInformation ugi = KrbUtils.getUgi(tableInfo.getPrincipal(), tableInfo.getKeytab(), tableInfo.getKrb5conf()); + return ugi.doAs(new PrivilegedAction() { + @Override + public KuduClient run() { + return kuduClientBuilder.build(); + } + }); + } else { + return kuduClientBuilder.build(); + } + } + /** * @param builder 创建AsyncKuduScanner对象 diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index edb49814f..43fda6e9f 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -10,6 +10,7 @@ import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; +import com.dtstack.flink.sql.util.KrbUtils; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.stumbleupon.async.Callback; @@ -23,6 +24,7 @@ import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.AsyncKuduClient; @@ -35,6 +37,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.security.PrivilegedAction; import java.sql.Timestamp; import java.util.*; @@ -80,25 +84,10 @@ public void open(Configuration parameters) throws Exception { * * @throws KuduException */ - private void connKuDu() throws KuduException { + private void connKuDu() throws IOException { if (null == table) { - String kuduMasters = kuduSideTableInfo.getKuduMasters(); String tableName = kuduSideTableInfo.getTableName(); - Integer workerCount = kuduSideTableInfo.getWorkerCount(); - Integer defaultSocketReadTimeoutMs = kuduSideTableInfo.getDefaultSocketReadTimeoutMs(); - Integer defaultOperationTimeoutMs = kuduSideTableInfo.getDefaultOperationTimeoutMs(); - - Preconditions.checkNotNull(kuduMasters, "kuduMasters could not be null"); - - AsyncKuduClient.AsyncKuduClientBuilder asyncKuduClientBuilder = new AsyncKuduClient.AsyncKuduClientBuilder(kuduMasters); - if (null != workerCount) { - asyncKuduClientBuilder.workerCount(workerCount); - } - - if (null != defaultOperationTimeoutMs) { - asyncKuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); - } - asyncClient = asyncKuduClientBuilder.build(); + asyncClient = getClient(); if (!asyncClient.syncClient().tableExists(tableName)) { throw new IllegalArgumentException("Table Open Failed , please check table exists"); } @@ -128,6 +117,40 @@ private void connKuDu() throws KuduException { scannerBuilder.setProjectedColumnNames(projectColumns); } + private AsyncKuduClient getClient() throws IOException { + String kuduMasters = kuduSideTableInfo.getKuduMasters(); + Integer workerCount = kuduSideTableInfo.getWorkerCount(); + Integer defaultOperationTimeoutMs = kuduSideTableInfo.getDefaultOperationTimeoutMs(); + + Preconditions.checkNotNull(kuduMasters, "kuduMasters could not be null"); + + AsyncKuduClient.AsyncKuduClientBuilder asyncKuduClientBuilder = new AsyncKuduClient.AsyncKuduClientBuilder(kuduMasters); + if (null != workerCount) { + asyncKuduClientBuilder.workerCount(workerCount); + } + + if (null != defaultOperationTimeoutMs) { + asyncKuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); + } + + if (kuduSideTableInfo.isEnableKrb()) { + UserGroupInformation ugi = KrbUtils.getUgi( + kuduSideTableInfo.getPrincipal(), + kuduSideTableInfo.getKeytab(), + kuduSideTableInfo.getKrb5conf() + ); + return ugi.doAs( + new PrivilegedAction() { + @Override + public AsyncKuduClient run() { + return asyncKuduClientBuilder.build(); + } + }); + } else { + return asyncKuduClientBuilder.build(); + } + } + @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { CRow inputCopy = new CRow(input.row(), input.change()); diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java index 62d215d87..f350dd6c3 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java @@ -1,5 +1,6 @@ package com.dtstack.flink.sql.side.kudu.table; +import com.dtstack.flink.sql.constrant.PluginParamConsts; import com.dtstack.flink.sql.table.AbstractSideTableParser; import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; @@ -71,6 +72,18 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map() { + @Override + public AsyncKuduClient run() { + return asyncKuduClientBuilder.build(); + } + }); + } else { + client = asyncKuduClientBuilder.build(); + } + LOG.info("connect kudu is successed!"); + KuduClient syncClient = client.syncClient(); if (syncClient.tableExists(tableName)) { table = syncClient.openTable(tableName); } @@ -215,6 +240,25 @@ public KuduOutputFormatBuilder setDefaultSocketReadTimeoutMs(Integer defaultSock return this; } + public KuduOutputFormatBuilder setPrincipal(String principal) { + kuduOutputFormat.principal = principal; + return this; + } + + public KuduOutputFormatBuilder setKeytab(String keytab) { + kuduOutputFormat.keytab = keytab; + return this; + } + + public KuduOutputFormatBuilder setKrb5conf(String krb5conf) { + kuduOutputFormat.krb5conf = krb5conf; + return this; + } + + public KuduOutputFormatBuilder setEnableKrb(boolean enableKrb) { + kuduOutputFormat.enableKrb = enableKrb; + return this; + } public KuduOutputFormat finish() { if (kuduOutputFormat.kuduMasters == null) { diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java index 4c7c3bea8..f8ac125be 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java @@ -38,6 +38,11 @@ public class KuduSink implements RetractStreamTableSink, Serializable, IStr private int parallelism = -1; + private String principal; + private String keytab; + private String krb5conf; + boolean enableKrb; + @Override public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KuduTableInfo kuduTableInfo = (KuduTableInfo) targetTableInfo; @@ -47,12 +52,20 @@ public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.defaultSocketReadTimeoutMs = kuduTableInfo.getDefaultSocketReadTimeoutMs(); this.workerCount = kuduTableInfo.getWorkerCount(); this.writeMode = kuduTableInfo.getWriteMode(); + this.principal = kuduTableInfo.getPrincipal(); + this.keytab = kuduTableInfo.getKeytab(); + this.krb5conf = kuduTableInfo.getKrb5conf(); + this.enableKrb = kuduTableInfo.isEnableKrb(); return this; } @Override public void emitDataStream(DataStream> dataStream) { + consumeDataStream(dataStream); + } + + public DataStreamSink> consumeDataStream(DataStream> dataStream) { KuduOutputFormat.KuduOutputFormatBuilder builder = KuduOutputFormat.buildKuduOutputFormat(); builder.setKuduMasters(this.kuduMasters) .setTableName(this.tableName) @@ -61,7 +74,11 @@ public void emitDataStream(DataStream> dataStream) { .setDefaultOperationTimeoutMs(this.defaultOperationTimeoutMs) .setDefaultSocketReadTimeoutMs(this.defaultSocketReadTimeoutMs) .setFieldNames(this.fieldNames) - .setFieldTypes(this.fieldTypes); + .setFieldTypes(this.fieldTypes) + .setPrincipal(this.principal) + .setKeytab(this.keytab) + .setKrb5conf(this.krb5conf) + .setEnableKrb(this.enableKrb); KuduOutputFormat kuduOutputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(kuduOutputFormat); DataStreamSink dataStreamSink = dataStream.addSink(richSinkFunction); @@ -69,6 +86,7 @@ public void emitDataStream(DataStream> dataStream) { if (parallelism > 0) { dataStreamSink.setParallelism(parallelism); } + return dataStreamSink; } @Override diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java index 20302d44f..80e9cc628 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java @@ -1,5 +1,6 @@ package com.dtstack.flink.sql.sink.kudu.table; +import com.dtstack.flink.sql.constrant.PluginParamConsts; import com.dtstack.flink.sql.sink.kudu.KuduOutputFormat; import com.dtstack.flink.sql.table.AbstractTableParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -39,6 +40,18 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map resultFuture){ + protected void preInvoke(CRow input, ResultFuture resultFuture) { } @@ -120,8 +123,8 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { AtomicLong networkLogCounter = new AtomicLong(0L); - while (!connectionStatus.get()){//network is unhealth - if(networkLogCounter.getAndIncrement() % 1000 == 0){ + while (!connectionStatus.get()) {//network is unhealth + if (networkLogCounter.getAndIncrement() % 1000 == 0) { LOG.info("network unhealth to block task"); } Thread.sleep(100); @@ -133,53 +136,84 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul private void connectWithRetry(Map inputParams, CRow input, ResultFuture resultFuture, SQLClient rdbSqlClient) { AtomicLong failCounter = new AtomicLong(0); AtomicBoolean finishFlag = new AtomicBoolean(false); - while(!finishFlag.get()){ - try{ + while (!finishFlag.get()) { + try { CountDownLatch latch = new CountDownLatch(1); - rdbSqlClient.getConnection(conn -> { - try { - if(conn.failed()){ - connectionStatus.set(false); - if(failCounter.getAndIncrement() % 1000 == 0){ - LOG.error("getConnection error", conn.cause()); - } - if(failCounter.get() >= sideInfo.getSideTableInfo().getConnectRetryMaxNum(100)){ - resultFuture.completeExceptionally(conn.cause()); - finishFlag.set(true); - } - return; - } - connectionStatus.set(true); - ScheduledFuture timerFuture = registerTimer(input, resultFuture); - cancelTimerWhenComplete(resultFuture, timerFuture); - handleQuery(conn.result(), inputParams, input, resultFuture); - finishFlag.set(true); - } catch (Exception e) { - dealFillDataError(input, resultFuture, e); - } finally { - latch.countDown(); - } - }); + asyncQueryData(inputParams, + input, resultFuture, + rdbSqlClient, + failCounter, + finishFlag, + latch); try { latch.await(); } catch (InterruptedException e) { LOG.error("", e); } - } catch (Exception e){ + } catch (Exception e) { //数据源队列溢出情况 connectionStatus.set(false); } - if(!finishFlag.get()){ + if (!finishFlag.get()) { try { Thread.sleep(3000); - } catch (Exception e){ + } catch (Exception e) { LOG.error("", e); } } } } + protected void asyncQueryData(Map inputParams, + CRow input, + ResultFuture resultFuture, + SQLClient rdbSqlClient, + AtomicLong failCounter, + AtomicBoolean finishFlag, + CountDownLatch latch) { + doAsyncQueryData(inputParams, + input, resultFuture, + rdbSqlClient, + failCounter, + finishFlag, + latch); + } + + final protected void doAsyncQueryData( + Map inputParams, + CRow input, + ResultFuture resultFuture, + SQLClient rdbSqlClient, + AtomicLong failCounter, + AtomicBoolean finishFlag, + CountDownLatch latch) { + rdbSqlClient.getConnection(conn -> { + try { + if (conn.failed()) { + connectionStatus.set(false); + if (failCounter.getAndIncrement() % 1000 == 0) { + LOG.error("getConnection error", conn.cause()); + } + if (failCounter.get() >= sideInfo.getSideTableInfo().getConnectRetryMaxNum(100)) { + resultFuture.completeExceptionally(conn.cause()); + finishFlag.set(true); + } + return; + } + connectionStatus.set(true); + preInvoke(input, resultFuture); + + handleQuery(conn.result(), inputParams, input, resultFuture); + finishFlag.set(true); + } catch (Exception e) { + dealFillDataError(input, resultFuture, e); + } finally { + latch.countDown(); + } + }); + } + private Object convertDataType(Object val) { if (val == null) { @@ -219,7 +253,7 @@ private Object convertDataType(Object val) { @Override public String buildCacheKey(Map inputParam) { - return StringUtils.join(inputParam.values(),"_"); + return StringUtils.join(inputParam.values(), "_"); } @Override @@ -231,7 +265,7 @@ public Row fillData(Row input, Object line) { boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 - obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); + obj = ((Timestamp) obj).getTime() + (long) LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); @@ -258,7 +292,7 @@ public void close() throws Exception { rdbSqlClient.close(); } - if(executor != null){ + if (executor != null) { executor.shutdown(); } @@ -268,7 +302,7 @@ public void setRdbSqlClient(SQLClient rdbSqlClient) { this.rdbSqlClient = rdbSqlClient; } - private void handleQuery(SQLConnection connection, Map inputParams, CRow input, ResultFuture resultFuture){ + private void handleQuery(SQLConnection connection, Map inputParams, CRow input, ResultFuture resultFuture) { String key = buildCacheKey(inputParams); JsonArray params = new JsonArray(Lists.newArrayList(inputParams.values())); connection.queryWithParams(sideInfo.getSqlCondition(), params, rs -> { @@ -312,9 +346,9 @@ private void handleQuery(SQLConnection connection, Map inputPara }); } - private Map formatInputParam(Map inputParam){ + private Map formatInputParam(Map inputParam) { Map result = Maps.newHashMap(); - inputParam.forEach((k,v) -> { + inputParam.forEach((k, v) -> { result.put(k, convertDataType(v)); }); return result; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java index b1a31de42..f06ecba74 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java @@ -228,4 +228,59 @@ public static int[] buildSqlTypes(List fieldTypeArray) { return tmpFieldsType; } + /** + * according to Java type, get the corresponding SQL type + * + * @param fieldTypeList the Java type + * @return the type number of the corresponding type + */ + public static int[] getSqlTypeFromFieldType(List fieldTypeList) { + int[] tmpFieldsType = new int[fieldTypeList.size()]; + for (int i = 0; i < fieldTypeList.size(); i++) { + String fieldType = fieldTypeList.get(i).toUpperCase(); + switch (fieldType) { + case "INT": + tmpFieldsType[i] = Types.INTEGER; + break; + case "BOOLEAN": + tmpFieldsType[i] = Types.BOOLEAN; + break; + case "BIGINT": + tmpFieldsType[i] = Types.BIGINT; + break; + case "SHORT": + tmpFieldsType[i] = Types.SMALLINT; + break; + case "STRING": + case "CHAR": + tmpFieldsType[i] = Types.CHAR; + break; + case "BYTE": + tmpFieldsType[i] = Types.BINARY; + break; + case "FLOAT": + tmpFieldsType[i] = Types.FLOAT; + break; + case "DOUBLE": + tmpFieldsType[i] = Types.DOUBLE; + break; + case "TIMESTAMP": + tmpFieldsType[i] = Types.TIMESTAMP; + break; + case "BIGDECIMAL": + tmpFieldsType[i] = Types.DECIMAL; + break; + case "DATE": + tmpFieldsType[i] = Types.DATE; + break; + case "TIME": + tmpFieldsType[i] = Types.TIME; + break; + default: + throw new RuntimeException("no support field type for sql. the input type:" + fieldType); + } + } + return tmpFieldsType; + } + } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index b87c2e042..c2491e586 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -29,19 +29,18 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.sql.ResultSet; +import java.security.PrivilegedAction; import java.sql.SQLException; -import java.sql.Statement; import java.util.List; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -77,7 +76,6 @@ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat tuple2) throws IOException { - if(!flushFlag.get()){ - throw new RuntimeException("connect exception,can not write record"); - } checkConnectionOpen(); try { if (outRecords.getCount() % RECEIVEDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { @@ -193,7 +190,6 @@ private void checkConnectionOpen() { public synchronized void flush() throws Exception { jdbcWriter.executeBatch(connection); batchCount = 0; - flushFlag.set(true); } /** From 92dd4327c67f78c9625a2629e077e8536955d0f3 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 27 Oct 2020 09:35:07 +0800 Subject: [PATCH 477/523] [fix] impala varchar data error; storeType npe; value condition bug --- .../sql/sink/impala/ImpalaOutputFormat.java | 17 +++++++++-------- .../sql/sink/impala/table/ImpalaSinkParser.java | 7 +++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index 19e904ef2..e880f58d4 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -76,7 +76,7 @@ public class ImpalaOutputFormat extends AbstractDtRichOutputFormat cast('value' as string) cast(value as timestamp) -> cast('value' as timestamp) private static final Pattern TYPE_PATTERN = Pattern.compile("cast\\((.*) as (.*)\\)"); //specific type which values need to be quoted - private static final String[] NEED_QUOTE_TYPE = {"string", "timestamp"}; + private static final String[] NEED_QUOTE_TYPE = {"string", "timestamp", "varchar"}; private static final Integer DEFAULT_CONN_TIME_OUT = 60; private static final int RECEIVE_DATA_PRINT_FREQUENCY = 1000; @@ -327,10 +327,7 @@ private void putRowIntoMap(Map> rowDataMap, Tuple2 rebuildFieldNameListAndTypeList(List fieldNames, - List staticPartitionFields, - List fieldTypes, - String partitionFields) { + private List rebuildFieldNameListAndTypeList(List fieldNames, List staticPartitionFields, List fieldTypes, String partitionFields) { if (partitionFields.isEmpty()) { return fieldNames; } @@ -364,8 +361,8 @@ private List rebuildFieldNameListAndTypeList(List fieldNames, * @return quoted condition */ private String valueConditionAddQuotation(String valueCondition) { - final String[] valueConditionCopy = {valueCondition}; String[] temps = valueCondition.split(","); + List replacedItem = new ArrayList<>(); Arrays.stream(temps).forEach( item -> { Matcher matcher = TYPE_PATTERN.matcher(item); @@ -375,13 +372,15 @@ private String valueConditionAddQuotation(String valueCondition) { if (Arrays.asList(NEED_QUOTE_TYPE).contains(type)) { if (!"null".equals(value)) { - valueConditionCopy[0] = valueConditionCopy[0].replace(value, "'" + value + "'"); + item = item.replace(value, "'" + value + "'"); } } } + replacedItem.add(item); } ); - return "(" + valueConditionCopy[0] + ")"; + + return "(" + String.join(", ", replacedItem) + ")"; } @Override @@ -746,6 +745,8 @@ public ImpalaOutputFormat build() { checkNotNull(format.password, "password is required!"); } + checkNotNull(format.storeType, "storeType is required!"); + return format; } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java index f1a475f61..003b5a6ab 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; /** * Date: 2020/10/14 @@ -76,6 +77,8 @@ public class ImpalaSinkParser extends AbstractTableParser { private static final String KUDU_TYPE = "kudu"; + private static final String DEFAULT_STORE_TYPE = "kudu"; + private static final String STORE_TYPE_KEY = "storeType"; private static final String KRB_DEFAULT_REALM = "HADOOP.COM"; @@ -123,13 +126,13 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Wed, 28 Oct 2020 14:24:24 +0800 Subject: [PATCH 478/523] [fix] varchar type bug --- .../flink/sql/sink/impala/ImpalaOutputFormat.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index e880f58d4..ed3370727 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -52,6 +52,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -370,9 +371,11 @@ private String valueConditionAddQuotation(String valueCondition) { String value = matcher.group(1); String type = matcher.group(2); - if (Arrays.asList(NEED_QUOTE_TYPE).contains(type)) { - if (!"null".equals(value)) { - item = item.replace(value, "'" + value + "'"); + for (String needQuoteType : NEED_QUOTE_TYPE) { + if (type.contains(needQuoteType)) { + if (!"null".equals(value)) { + item = item.replace(value, "'" + value + "'"); + } } } } @@ -580,8 +583,10 @@ private String buildTableFieldsCondition(List fieldNames, String partiti private String buildValuesCondition(List fieldTypes, Row row) { String valuesCondition = fieldTypes.stream().map( f -> { - if (Arrays.asList(NEED_QUOTE_TYPE).contains(f.toLowerCase())) { - return String.format("cast(? as %s)", f.toLowerCase()); + for(String item : NEED_QUOTE_TYPE) { + if (f.toLowerCase().contains(item)) { + return String.format("cast(? as %s)", f.toLowerCase()); + } } return "?"; }).collect(Collectors.joining(", ")); From c0b80769aa5b162b23e1cdb84280170cd22297ba Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 28 Oct 2020 15:08:06 +0800 Subject: [PATCH 479/523] [fix] fix krb bug --- .../java/com/dtstack/flink/sql/util/KrbUtils.java | 13 +++++++++++-- .../com/dtstack/flink/sql/util/KrbUtilsTest.java | 2 +- .../flink/sql/side/impala/ImpalaAllReqRow.java | 2 +- .../flink/sql/side/impala/ImpalaAsyncReqRow.java | 3 +-- .../flink/sql/sink/impala/ImpalaOutputFormat.java | 2 +- .../dtstack/flink/sql/side/kudu/KuduAllReqRow.java | 2 +- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 3 +-- .../flink/sql/sink/kudu/KuduOutputFormat.java | 2 +- 8 files changed, 18 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java index 2a83359c4..00f2f538d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java @@ -20,8 +20,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import sun.security.krb5.Config; +import sun.security.krb5.KrbException; import java.io.IOException; @@ -40,9 +43,15 @@ public class KrbUtils { // public static final String FALSE_STR = "false"; // public static final String SUBJECT_ONLY_KEY = "javax.security.auth.useSubjectCredsOnly"; - public static UserGroupInformation getUgi(String principal, String keytabPath, String krb5confPath) throws IOException { - LOG.info("Kerberos login with principal: {} and keytab: {}", principal, keytabPath); + public static UserGroupInformation loginAndReturnUgi(String principal, String keytabPath, String krb5confPath) throws IOException { LOG.info("Kerberos login with principal: {} and keytab: {}", principal, keytabPath); System.setProperty(KRB5_CONF_KEY, krb5confPath); + // 不刷新会读/etc/krb5.conf + try { + Config.refresh(); + KerberosName.resetDefaultRealm(); + } catch (KrbException e) { + LOG.warn("resetting default realm failed, current default realm will still be used.", e); + } // TODO 尚未探索出此选项的意义，以后研究明白方可打开 // System.setProperty(SUBJECT_ONLY_KEY, FALSE_STR); Configuration configuration = new Configuration(); diff --git a/core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java b/core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java index 6c7359e14..5d65158c7 100644 --- a/core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java +++ b/core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java @@ -35,7 +35,7 @@ public void testGetUgi() throws IOException { String keytabPath = ""; String krb5confPath = ""; try { - KrbUtils.getUgi(principal, keytabPath, krb5confPath); + KrbUtils.loginAndReturnUgi(principal, keytabPath, krb5confPath); } catch (IllegalArgumentException e) { Assert.assertEquals(e.getMessage(), "Can't get Kerberos realm"); } diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java index 179e52e56..85b1f1b1f 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java @@ -71,7 +71,7 @@ public Connection getConn(String dbUrl, String userName, String password) { String keyTabFilePath = impalaSideTableInfo.getKeyTabFilePath(); String krb5FilePath = impalaSideTableInfo.getKrb5FilePath(); String principal = impalaSideTableInfo.getPrincipal(); - UserGroupInformation ugi = KrbUtils.getUgi(principal, keyTabFilePath, krb5FilePath); + UserGroupInformation ugi = KrbUtils.loginAndReturnUgi(principal, keyTabFilePath, krb5FilePath); connection = ugi.doAs((PrivilegedAction) () -> { try { return DriverManager.getConnection(url); diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index ae8ac2be9..3e7da110a 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -33,7 +33,6 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,7 +70,7 @@ public void open(Configuration parameters) throws Exception { String keyTabFilePath = impalaSideTableInfo.getKeyTabFilePath(); String krb5FilePath = impalaSideTableInfo.getKrb5FilePath(); String principal = impalaSideTableInfo.getPrincipal(); - ugi = KrbUtils.getUgi(principal, keyTabFilePath, krb5FilePath); + ugi = KrbUtils.loginAndReturnUgi(principal, keyTabFilePath, krb5FilePath); } openJdbc(parameters); } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index ed3370727..a755ee2bf 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -207,7 +207,7 @@ private void initScheduledTask(Long batchWaitInterval) { private void openConnect() throws IOException { if (authMech == 1) { - UserGroupInformation ugi = KrbUtils.getUgi(principal, keytabPath, krb5confPath); + UserGroupInformation ugi = KrbUtils.loginAndReturnUgi(principal, keytabPath, krb5confPath); try { ugi.doAs((PrivilegedExceptionAction) () -> { openJdbc(); diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 0b30729f8..01ee1cd9b 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -256,7 +256,7 @@ private KuduClient getClient(KuduSideTableInfo tableInfo) throws IOException { } if (tableInfo.isEnableKrb()) { - UserGroupInformation ugi = KrbUtils.getUgi(tableInfo.getPrincipal(), tableInfo.getKeytab(), tableInfo.getKrb5conf()); + UserGroupInformation ugi = KrbUtils.loginAndReturnUgi(tableInfo.getPrincipal(), tableInfo.getKeytab(), tableInfo.getKrb5conf()); return ugi.doAs(new PrivilegedAction() { @Override public KuduClient run() { diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index 43fda6e9f..84b37ab67 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -15,7 +15,6 @@ import com.google.common.collect.Maps; import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; -import io.vertx.core.json.JsonArray; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; @@ -134,7 +133,7 @@ private AsyncKuduClient getClient() throws IOException { } if (kuduSideTableInfo.isEnableKrb()) { - UserGroupInformation ugi = KrbUtils.getUgi( + UserGroupInformation ugi = KrbUtils.loginAndReturnUgi( kuduSideTableInfo.getPrincipal(), kuduSideTableInfo.getKeytab(), kuduSideTableInfo.getKrb5conf() diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index 41047762b..ab6ac13a8 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -116,7 +116,7 @@ private void establishConnection() throws IOException { } if (enableKrb) { - UserGroupInformation ugi = KrbUtils.getUgi( + UserGroupInformation ugi = KrbUtils.loginAndReturnUgi( principal, keytab, krb5conf From 7d943052bd620d9fc4e4d6353f24f67b868d713d Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 2 Nov 2020 19:07:02 +0800 Subject: [PATCH 480/523] =?UTF-8?q?[fix]=20RDB=E5=85=A8=E9=87=8F=E7=BB=B4?= =?UTF-8?q?=E8=A1=A8=E4=BD=BF=E7=94=A8=E5=88=AB=E5=90=8D=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/rdb/all/AbstractRdbAllReqRow.java | 10 +++++++--- .../flink/sql/side/rdb/all/RdbAllSideInfo.java | 17 +++++++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 0530e57e2..0db947d42 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -213,13 +213,17 @@ private void queryAndFillData(Map>> tmpCache, C ResultSet resultSet = statement.executeQuery(sql); String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); - String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); + String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); + Map sideFieldNamesAndTypes = Maps.newHashMap(); + for (int i = 0; i < sideFieldNames.length; i++) { + sideFieldNamesAndTypes.put(sideFieldNames[i], sideFieldTypes[i]); + } + while (resultSet.next()) { Map oneRow = Maps.newHashMap(); for (String fieldName : sideFieldNames) { Object object = resultSet.getObject(fieldName.trim()); - int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); - object = SwitchUtil.getTarget(object, fields[fieldIndex]); + object = SwitchUtil.getTarget(object, sideFieldNamesAndTypes.get(fieldName)); oneRow.put(fieldName.trim(), object); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 82ec95ca0..cafc715d5 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -36,6 +36,8 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -59,7 +61,18 @@ public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List selectFields = Lists.newArrayList(); + Map physicalFields = rdbSideTableInfo.getPhysicalFields(); + physicalFields.keySet().forEach( + item -> { + if (Objects.isNull(physicalFields.get(item))) { + selectFields.add(quoteIdentifier(item)); + } else { + selectFields.add(quoteIdentifier(physicalFields.get(item)) + " AS " + quoteIdentifier(item)); + } + } + ); + sqlCondition = getSelectFromStatement(getTableName(rdbSideTableInfo), selectFields, sideTableInfo.getPredicateInfoes()); LOG.info("--------dimension sql query-------\n{}" + sqlCondition); } @@ -68,7 +81,7 @@ public String getAdditionalWhereClause() { } private String getSelectFromStatement(String tableName, List selectFields, List predicateInfoes) { - String fromClause = selectFields.stream().map(this::quoteIdentifier).collect(Collectors.joining(", ")); + String fromClause = String.join(", ", selectFields); String predicateClause = predicateInfoes.stream().map(this::buildFilterCondition).collect(Collectors.joining(" AND ")); String whereClause = buildWhereClause(predicateClause); return "SELECT " + fromClause + " FROM " + tableName + whereClause; From 845b55e7081f64a11974263948293aeebd5afe9e Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 3 Nov 2020 10:44:31 +0800 Subject: [PATCH 481/523] =?UTF-8?q?[fix]=20=E6=94=AF=E6=8C=81=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E6=88=B3=E8=BD=AC=E5=8C=96=E4=B8=BA=E5=AF=B9=E5=BA=94?= =?UTF-8?q?=E7=9A=84Timestamp=E3=80=81Time=E3=80=81Date=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DtNestRowDeserializationSchema.java | 52 +++++++++++++++++-- .../com/dtstack/flink/sql/util/DateUtil.java | 13 +++++ .../com/dtstack/flink/sql/util/MathUtil.java | 16 +++++- .../side/elasticsearch6/util/MathUtil.java | 38 +++++++++----- .../side/elasticsearch6/util/SwitchUtil.java | 3 +- .../flink/sql/side/rdb/util/SwitchUtil.java | 2 + 6 files changed, 104 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 13cc261ad..04679a039 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -42,6 +42,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; /** * source data parse to json format @@ -63,6 +64,15 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final List fieldExtraInfos; private final String charsetName; + private static final Pattern TIMESTAMP_PATTERN = Pattern.compile("^\\d+$"); + private static final Pattern TIMESTAMP_FORMAT_PATTERN = Pattern.compile("\\d+-\\d+-\\d+\\s+\\d+:\\d+:\\d+"); + private static final Pattern TIME_FORMAT_PATTERN = Pattern.compile("\\w+\\d+:\\d+:\\d+"); + private static final Pattern DATE_FORMAT_PATTERN = Pattern.compile("\\w+\\d+-\\d+-\\d+"); + + private static final String TIMESTAMP_TYPE = "timestamp"; + private static final String TIME_TYPE = "time"; + private static final String DATE_TYPE = "date"; + public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos, String charsetName) { @@ -141,14 +151,14 @@ private Object convert(JsonNode node, TypeInformation info) { return node.asText(); } } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { - return Date.valueOf(node.asText()); + return convertToTimestamp(node.asText(), DATE_TYPE); } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { // local zone - return Time.valueOf(node.asText()); + return convertToTimestamp(node.asText(), TIME_TYPE); } else if (info.getTypeClass().equals(Types.SQL_TIMESTAMP.getTypeClass())) { // local zone - return Timestamp.valueOf(node.asText()); - } else if (info instanceof RowTypeInfo) { + return convertToTimestamp(node.asText(), TIMESTAMP_TYPE); + } else if (info instanceof RowTypeInfo) { return convertRow(node, (RowTypeInfo) info); } else if (info instanceof ObjectArrayTypeInfo) { return convertObjectArray(node, ((ObjectArrayTypeInfo) info).getComponentInfo()); @@ -163,6 +173,38 @@ private Object convert(JsonNode node, TypeInformation info) { } } + /** + * 将 2020-09-07 14:49:10.0 和 1598446699685 两种格式都转化为 Timestamp、Time、Date + */ + private static Object convertToTimestamp(String timestamp, String type) { + if (TIMESTAMP_PATTERN.matcher(timestamp).find()) { + switch (type.toLowerCase()) { + case (TIMESTAMP_TYPE): + return new Timestamp(Long.parseLong(timestamp)); + case (DATE_TYPE): + return new Date(new Timestamp(Long.parseLong(timestamp)).getTime()); + case (TIME_TYPE): + return new Time(new Timestamp(Long.parseLong(timestamp)).getTime()); + default: + throw new RuntimeException(String.format("%s transform to %s error!", timestamp, type)); + } + } + + if (TIMESTAMP_FORMAT_PATTERN.matcher(timestamp).find() && TIMESTAMP_TYPE.equalsIgnoreCase(type)) { + return Timestamp.valueOf(timestamp); + } + + if (TIME_FORMAT_PATTERN.matcher(timestamp).find() && TIME_TYPE.equalsIgnoreCase(type)) { + return Time.valueOf(timestamp); + } + + if (DATE_FORMAT_PATTERN.matcher(timestamp).find() && DATE_TYPE.equalsIgnoreCase(type)) { + return Date.valueOf(timestamp); + } + + throw new IllegalArgumentException("Incorrect time format of timestamp, input: " + timestamp); + } + private Row convertTopRow() { Row row = new Row(fieldNames.length); try { @@ -173,7 +215,7 @@ private Row convertTopRow() { if (node == null) { if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { throw new IllegalStateException("Failed to find field with name '" - + fieldNames[i] + "'."); + + fieldNames[i] + "'."); } else { row.setField(i, null); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index b05de0f16..bb48b0678 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -59,6 +59,7 @@ public class DateUtil { private static final Pattern DATETIME = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); private static final Pattern DATE = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}$"); + private static final Pattern TIME = Pattern.compile("^\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); private static final int MILLIS_PER_SECOND = 1000; @@ -832,4 +833,16 @@ public static java.sql.Date getDateFromStr(String dateStr) { return null == date ? null : new java.sql.Date(date.getTime()); } + public static java.sql.Time getTimeFromStr(String dateStr) { + if (TIME.matcher(dateStr).matches()) { + dateStr = dateStr.substring(0,dateStr.length()-1); + Instant instant = LocalTime.parse(dateStr).atDate(LocalDate.now()).toInstant(ZoneOffset.UTC); + return new java.sql.Time(instant.toEpochMilli()); + } else if (DATETIME.matcher(dateStr).matches()) { + Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); + return new java.sql.Time(instant.toEpochMilli()); + } + Date date = stringToDate(dateStr); + return null == date ? null : new java.sql.Time(date.getTime()); + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index c5584b5d8..844b87033 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -22,7 +22,9 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Date; +import java.sql.Time; import java.sql.Timestamp; +import java.util.Objects; /** * Convert val to specified numeric type @@ -234,7 +236,19 @@ public static Date getDate(Object obj) { throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); } - + public static Time getTime(Object obj) { + if (Objects.isNull(obj)) { + return null; + } + if (obj instanceof String) { + return DateUtil.getTimeFromStr((String) obj); + } else if (obj instanceof Timestamp) { + return new Time(((Timestamp) obj).getTime()); + } else if (obj instanceof Time) { + return (Time) obj; + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Time."); + } public static Timestamp getTimestamp(Object obj) { if (obj == null) { diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java index d72530a56..faf256145 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java @@ -19,22 +19,23 @@ package com.dtstack.flink.sql.side.elasticsearch6.util; +import com.dtstack.flink.sql.util.DateUtil; + import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Date; +import java.sql.Time; import java.sql.Timestamp; -import java.text.ParseException; -import java.text.SimpleDateFormat; +import java.util.Objects; /** + * Convert val to specified numeric type * Date: 2017/4/21 * Company: www.dtstack.com - * * @author xuchao */ public class MathUtil { - public static Long getLongVal(Object obj) { if (obj == null) { return null; @@ -103,8 +104,6 @@ public static Float getFloatVal(Object obj) { return (Float) obj; } else if (obj instanceof BigDecimal) { return ((BigDecimal) obj).floatValue(); - } else if (obj instanceof Double) { - return ((Double) obj).floatValue(); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Float."); @@ -126,9 +125,13 @@ public static Double getDoubleVal(Object obj) { if (obj instanceof String) { return Double.valueOf((String) obj); } else if (obj instanceof Float) { + return ((Float) obj).doubleValue(); + } else if (obj instanceof Double) { return (Double) obj; } else if (obj instanceof BigDecimal) { return ((BigDecimal) obj).doubleValue(); + } else if (obj instanceof Integer) { + return ((Integer) obj).doubleValue(); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Double."); @@ -226,12 +229,7 @@ public static Date getDate(Object obj) { return null; } if (obj instanceof String) { - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - try { - return new Date(format.parse((String) obj).getTime()); - } catch (ParseException e) { - throw new RuntimeException("String convert to Date fail."); - } + return DateUtil.getDateFromStr((String) obj); } else if (obj instanceof Timestamp) { return new Date(((Timestamp) obj).getTime()); } else if (obj instanceof Date) { @@ -240,6 +238,20 @@ public static Date getDate(Object obj) { throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); } + public static Time getTime(Object obj) { + if (Objects.isNull(obj)) { + return null; + } + if (obj instanceof String) { + return DateUtil.getTimeFromStr((String) obj); + } else if (obj instanceof Timestamp) { + return new Time(((Timestamp) obj).getTime()); + } else if (obj instanceof Time) { + return (Time) obj; + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Time."); + } + public static Timestamp getTimestamp(Object obj) { if (obj == null) { return null; @@ -249,7 +261,7 @@ public static Timestamp getTimestamp(Object obj) { } else if (obj instanceof Date) { return new Timestamp(((Date) obj).getTime()); } else if (obj instanceof String) { - return new Timestamp(getDate(obj).getTime()); + return DateUtil.getTimestampFromStr(obj.toString()); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java index e0aaa123e..76165a4a7 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java @@ -39,7 +39,6 @@ public static Object getTarget(Object obj, String targetType) { case "int": return MathUtil.getIntegerVal(obj); - case "long": case "bigint": case "bigintunsigned": case "intunsigned": @@ -77,6 +76,8 @@ public static Object getTarget(Object obj, String targetType) { case "timestamp": case "datetime": return MathUtil.getTimestamp(obj); + case "time": + return MathUtil.getTime(obj); default: return obj; } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java index 49d163d4d..62390b47d 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java @@ -78,6 +78,8 @@ public static Object getTarget(Object obj, String targetType) { case "timestamp": case "datetime": return MathUtil.getTimestamp(obj); + case "time": + return MathUtil.getTime(obj); default: } return obj; From 7872c8072d7b5aac393e9b9e9dbffa37a1a34e3f Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 3 Nov 2020 10:44:31 +0800 Subject: [PATCH 482/523] =?UTF-8?q?[fix]=20=E6=94=AF=E6=8C=81=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E6=88=B3=E8=BD=AC=E5=8C=96=E4=B8=BA=E5=AF=B9=E5=BA=94?= =?UTF-8?q?=E7=9A=84Timestamp=E3=80=81Time=E3=80=81Date=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [fix] 支持时间戳转化为对应的Timestamp、Time、Date时间类型 [fix] 支持时间戳转化为对应的Timestamp、Time、Date时间类型 --- .../DtNestRowDeserializationSchema.java | 62 +++++++++++++++++-- .../com/dtstack/flink/sql/util/DateUtil.java | 13 ++++ .../com/dtstack/flink/sql/util/MathUtil.java | 16 ++++- .../side/elasticsearch6/util/MathUtil.java | 38 ++++++++---- .../side/elasticsearch6/util/SwitchUtil.java | 3 +- .../flink/sql/side/rdb/util/SwitchUtil.java | 2 + 6 files changed, 114 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 13cc261ad..8b2ca5f49 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -42,6 +42,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * source data parse to json format @@ -63,6 +65,15 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final List fieldExtraInfos; private final String charsetName; + private static final Pattern TIMESTAMP_PATTERN = Pattern.compile("^\\d+$"); + private static final Pattern TIMESTAMP_FORMAT_PATTERN = Pattern.compile("(\\d+-\\d+-\\d+\\s)(\\d+:\\d+:\\d+)"); + private static final Pattern TIME_FORMAT_PATTERN = Pattern.compile("\\w+\\d+:\\d+:\\d+"); + private static final Pattern DATE_FORMAT_PATTERN = Pattern.compile("\\w+\\d+-\\d+-\\d+"); + + private static final String TIMESTAMP_TYPE = "timestamp"; + private static final String TIME_TYPE = "time"; + private static final String DATE_TYPE = "date"; + public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos, String charsetName) { @@ -141,14 +152,14 @@ private Object convert(JsonNode node, TypeInformation info) { return node.asText(); } } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { - return Date.valueOf(node.asText()); + return convertToTimestamp(node.asText(), DATE_TYPE); } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { // local zone - return Time.valueOf(node.asText()); + return convertToTimestamp(node.asText(), TIME_TYPE); } else if (info.getTypeClass().equals(Types.SQL_TIMESTAMP.getTypeClass())) { // local zone - return Timestamp.valueOf(node.asText()); - } else if (info instanceof RowTypeInfo) { + return convertToTimestamp(node.asText(), TIMESTAMP_TYPE); + } else if (info instanceof RowTypeInfo) { return convertRow(node, (RowTypeInfo) info); } else if (info instanceof ObjectArrayTypeInfo) { return convertObjectArray(node, ((ObjectArrayTypeInfo) info).getComponentInfo()); @@ -163,6 +174,47 @@ private Object convert(JsonNode node, TypeInformation info) { } } + /** + * 将 2020-09-07 14:49:10.0 和 1598446699685 两种格式都转化为 Timestamp、Time、Date + */ + private static Object convertToTimestamp(String timestamp, String type) { + if (TIMESTAMP_PATTERN.matcher(timestamp).find()) { + switch (type.toLowerCase()) { + case (TIMESTAMP_TYPE): + return new Timestamp(Long.parseLong(timestamp)); + case (DATE_TYPE): + return new Date(new Timestamp(Long.parseLong(timestamp)).getTime()); + case (TIME_TYPE): + return new Time(new Timestamp(Long.parseLong(timestamp)).getTime()); + default: + throw new RuntimeException(String.format("%s transform to %s error!", timestamp, type)); + } + } + + Matcher matcher = TIMESTAMP_FORMAT_PATTERN.matcher(timestamp); + if (matcher.find()) { + switch (type.toLowerCase()) { + case TIME_TYPE: + return Time.valueOf(String.valueOf(matcher.group(2)).trim()); + case DATE_TYPE: + return Date.valueOf(String.valueOf(matcher.group(1)).trim()); + case TIMESTAMP_TYPE: + return Timestamp.valueOf(timestamp); + default: + } + } + + if (TIME_FORMAT_PATTERN.matcher(timestamp).find() && TIME_TYPE.equalsIgnoreCase(type)) { + return Time.valueOf(timestamp); + } + + if (DATE_FORMAT_PATTERN.matcher(timestamp).find() && DATE_TYPE.equalsIgnoreCase(type)) { + return Date.valueOf(timestamp); + } + + throw new IllegalArgumentException("Incorrect time format of timestamp, input: " + timestamp); + } + private Row convertTopRow() { Row row = new Row(fieldNames.length); try { @@ -173,7 +225,7 @@ private Row convertTopRow() { if (node == null) { if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { throw new IllegalStateException("Failed to find field with name '" - + fieldNames[i] + "'."); + + fieldNames[i] + "'."); } else { row.setField(i, null); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index b05de0f16..bb48b0678 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -59,6 +59,7 @@ public class DateUtil { private static final Pattern DATETIME = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); private static final Pattern DATE = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}$"); + private static final Pattern TIME = Pattern.compile("^\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); private static final int MILLIS_PER_SECOND = 1000; @@ -832,4 +833,16 @@ public static java.sql.Date getDateFromStr(String dateStr) { return null == date ? null : new java.sql.Date(date.getTime()); } + public static java.sql.Time getTimeFromStr(String dateStr) { + if (TIME.matcher(dateStr).matches()) { + dateStr = dateStr.substring(0,dateStr.length()-1); + Instant instant = LocalTime.parse(dateStr).atDate(LocalDate.now()).toInstant(ZoneOffset.UTC); + return new java.sql.Time(instant.toEpochMilli()); + } else if (DATETIME.matcher(dateStr).matches()) { + Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); + return new java.sql.Time(instant.toEpochMilli()); + } + Date date = stringToDate(dateStr); + return null == date ? null : new java.sql.Time(date.getTime()); + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index c5584b5d8..844b87033 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -22,7 +22,9 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Date; +import java.sql.Time; import java.sql.Timestamp; +import java.util.Objects; /** * Convert val to specified numeric type @@ -234,7 +236,19 @@ public static Date getDate(Object obj) { throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); } - + public static Time getTime(Object obj) { + if (Objects.isNull(obj)) { + return null; + } + if (obj instanceof String) { + return DateUtil.getTimeFromStr((String) obj); + } else if (obj instanceof Timestamp) { + return new Time(((Timestamp) obj).getTime()); + } else if (obj instanceof Time) { + return (Time) obj; + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Time."); + } public static Timestamp getTimestamp(Object obj) { if (obj == null) { diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java index d72530a56..faf256145 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java @@ -19,22 +19,23 @@ package com.dtstack.flink.sql.side.elasticsearch6.util; +import com.dtstack.flink.sql.util.DateUtil; + import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Date; +import java.sql.Time; import java.sql.Timestamp; -import java.text.ParseException; -import java.text.SimpleDateFormat; +import java.util.Objects; /** + * Convert val to specified numeric type * Date: 2017/4/21 * Company: www.dtstack.com - * * @author xuchao */ public class MathUtil { - public static Long getLongVal(Object obj) { if (obj == null) { return null; @@ -103,8 +104,6 @@ public static Float getFloatVal(Object obj) { return (Float) obj; } else if (obj instanceof BigDecimal) { return ((BigDecimal) obj).floatValue(); - } else if (obj instanceof Double) { - return ((Double) obj).floatValue(); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Float."); @@ -126,9 +125,13 @@ public static Double getDoubleVal(Object obj) { if (obj instanceof String) { return Double.valueOf((String) obj); } else if (obj instanceof Float) { + return ((Float) obj).doubleValue(); + } else if (obj instanceof Double) { return (Double) obj; } else if (obj instanceof BigDecimal) { return ((BigDecimal) obj).doubleValue(); + } else if (obj instanceof Integer) { + return ((Integer) obj).doubleValue(); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Double."); @@ -226,12 +229,7 @@ public static Date getDate(Object obj) { return null; } if (obj instanceof String) { - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - try { - return new Date(format.parse((String) obj).getTime()); - } catch (ParseException e) { - throw new RuntimeException("String convert to Date fail."); - } + return DateUtil.getDateFromStr((String) obj); } else if (obj instanceof Timestamp) { return new Date(((Timestamp) obj).getTime()); } else if (obj instanceof Date) { @@ -240,6 +238,20 @@ public static Date getDate(Object obj) { throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); } + public static Time getTime(Object obj) { + if (Objects.isNull(obj)) { + return null; + } + if (obj instanceof String) { + return DateUtil.getTimeFromStr((String) obj); + } else if (obj instanceof Timestamp) { + return new Time(((Timestamp) obj).getTime()); + } else if (obj instanceof Time) { + return (Time) obj; + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Time."); + } + public static Timestamp getTimestamp(Object obj) { if (obj == null) { return null; @@ -249,7 +261,7 @@ public static Timestamp getTimestamp(Object obj) { } else if (obj instanceof Date) { return new Timestamp(((Date) obj).getTime()); } else if (obj instanceof String) { - return new Timestamp(getDate(obj).getTime()); + return DateUtil.getTimestampFromStr(obj.toString()); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java index e0aaa123e..76165a4a7 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java @@ -39,7 +39,6 @@ public static Object getTarget(Object obj, String targetType) { case "int": return MathUtil.getIntegerVal(obj); - case "long": case "bigint": case "bigintunsigned": case "intunsigned": @@ -77,6 +76,8 @@ public static Object getTarget(Object obj, String targetType) { case "timestamp": case "datetime": return MathUtil.getTimestamp(obj); + case "time": + return MathUtil.getTime(obj); default: return obj; } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java index 49d163d4d..62390b47d 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java @@ -78,6 +78,8 @@ public static Object getTarget(Object obj, String targetType) { case "timestamp": case "datetime": return MathUtil.getTimestamp(obj); + case "time": + return MathUtil.getTime(obj); default: } return obj; From ab0cbbeeab68bf7e4c44ae5b75fead1d819b8f66 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 3 Nov 2020 17:15:55 +0800 Subject: [PATCH 483/523] =?UTF-8?q?[fix]=20=E4=BF=AE=E5=A4=8DHbase=20Times?= =?UTF-8?q?tamp=E7=B1=BB=E5=9E=8B=E6=95=B0=E6=8D=AE=E5=86=99=E5=85=A5?= =?UTF-8?q?=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hbase/rowkeydealer/AbstractRowKeyModeDealer.java | 3 --- .../flink/sql/sink/hbase/HbaseOutputFormat.java | 11 +++++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java index 90ee289bd..6b5d6a24f 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java @@ -93,9 +93,6 @@ protected Row fillData(Row input, Object sideInput){ Row row = new Row(outFieldInfoList.size()); for(Map.Entry entry : inFieldIndex.entrySet()){ Object obj = input.getField(entry.getValue()); - if(obj instanceof Timestamp){ - obj = ((Timestamp)obj).getTime(); - } row.setField(entry.getKey(), obj); } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 6c31bd8dd..42a796403 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -189,14 +189,13 @@ private Put getPutByRow(Row record) { Put put = new Put(rowKey.getBytes()); for (int i = 0; i < record.getArity(); ++i) { Object fieldVal = record.getField(i); - byte[] val = null; if (fieldVal != null) { - val = HbaseUtil.toByte(fieldVal); - } - byte[] cf = families[i].getBytes(); - byte[] qualifier = qualifiers[i].getBytes(); + byte[] val = fieldVal.toString().getBytes(); + byte[] cf = families[i].getBytes(); + byte[] qualifier = qualifiers[i].getBytes(); - put.addColumn(cf, qualifier, val); + put.addColumn(cf, qualifier, val); + } } return put; } From ac08db39c56acca87211d4e3d05a77be58b49dbd Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 4 Nov 2020 10:36:40 +0800 Subject: [PATCH 484/523] =?UTF-8?q?[fix]=20=E7=BB=B4=E8=A1=A8JOIN=E5=A4=9A?= =?UTF-8?q?=E4=B8=AA=E5=AD=97=E6=AE=B5=E6=97=A0=E6=95=B0=E6=8D=AE=E5=BC=82?= =?UTF-8?q?=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java | 2 +- .../com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 3b6a7f88c..b89eb5615 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -181,7 +181,7 @@ public void asyncInvoke(CRow row, ResultFuture resultFuture) throws Except } private Map parseInputParam(CRow input){ - Map inputParams = Maps.newHashMap(); + Map inputParams = Maps.newLinkedHashMap(); for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = input.row().getField(conValIndex); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index b3274b80b..e8ef2cd73 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -347,7 +347,7 @@ private void handleQuery(SQLConnection connection, Map inputPara } private Map formatInputParam(Map inputParam) { - Map result = Maps.newHashMap(); + Map result = Maps.newLinkedHashMap(); inputParam.forEach((k, v) -> { result.put(k, convertDataType(v)); }); From f553a3c6d05630ed6ae7b7c052edf33e184753bf Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 5 Nov 2020 16:12:11 +0800 Subject: [PATCH 485/523] =?UTF-8?q?[fix]=20kafka=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E4=BB=A5'|'=E4=BD=9C=E4=B8=BA=E5=88=86=E5=89=B2=E7=AC=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/format/DeserializationMetricWrapper.java | 3 ++- .../com/dtstack/flink/sql/parser/CreateTableParser.java | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java b/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java index 96ebe09ae..a4addc025 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java @@ -31,6 +31,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.charset.StandardCharsets; /** * add metric for source @@ -91,7 +92,7 @@ public void initMetric() { public Row deserialize(byte[] message) throws IOException { try { if (numInRecord.getCount() % dataPrintFrequency == 0) { - LOG.info("receive source data:" + new String(message, "UTF-8")); + LOG.info("receive source data:" + new String(message, StandardCharsets.UTF_8)); } numInRecord.inc(); numInBytes.inc(message.length); diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java index 91b1fd2ac..303295751 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java @@ -72,10 +72,10 @@ public void parseSql(String sql, SqlTree sqlTree) { private Map parseProp(String propsStr){ propsStr = propsStr.replaceAll("'\\s*,", "'|"); - String[] strs = propsStr.trim().split("\\|"); + List strings = DtStringUtil.splitIgnoreQuota(propsStr, '|'); Map propMap = Maps.newHashMap(); - for(int i=0; i ss = DtStringUtil.splitIgnoreQuota(strs[i], '='); + for (String str : strings) { + List ss = DtStringUtil.splitIgnoreQuota(str, '='); String key = ss.get(0).trim(); String value = extractValue(ss.get(1).trim()); propMap.put(key, value); From 3a1836c2c37e29e78597440f3f5e110755d8acbe Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 5 Nov 2020 16:44:37 +0800 Subject: [PATCH 486/523] =?UTF-8?q?[fix]=20=E4=BF=AE=E5=A4=8D=E5=BC=95?= =?UTF-8?q?=E5=8F=B7=E9=87=8C','=E8=A2=AB=E8=A7=A3=E6=9E=90=E5=BC=82?= =?UTF-8?q?=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/parser/CreateTableParser.java | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java index 303295751..925531240 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java @@ -16,8 +16,6 @@ * limitations under the License. */ - - package com.dtstack.flink.sql.parser; import com.dtstack.flink.sql.util.DtStringUtil; @@ -41,8 +39,6 @@ public class CreateTableParser implements IParser { private static final Pattern PATTERN = Pattern.compile(PATTERN_STR); - private static final Pattern PROP_PATTERN = Pattern.compile("^'\\s*(.+)\\s*'$"); - public static CreateTableParser newInstance(){ return new CreateTableParser(); } @@ -70,28 +66,19 @@ public void parseSql(String sql, SqlTree sqlTree) { } } - private Map parseProp(String propsStr){ - propsStr = propsStr.replaceAll("'\\s*,", "'|"); - List strings = DtStringUtil.splitIgnoreQuota(propsStr, '|'); + private Map parseProp(String propsStr){ + List strings = DtStringUtil.splitIgnoreQuota(propsStr.trim(), ','); Map propMap = Maps.newHashMap(); for (String str : strings) { List ss = DtStringUtil.splitIgnoreQuota(str, '='); String key = ss.get(0).trim(); - String value = extractValue(ss.get(1).trim()); + String value = ss.get(1).trim().replaceAll("'", "").trim(); propMap.put(key, value); } return propMap; } - private String extractValue(String value) { - Matcher matcher = PROP_PATTERN.matcher(value); - if (matcher.find()) { - return matcher.group(1); - } - throw new RuntimeException("[" + value + "] format is invalid"); - } - public static class SqlParserResult{ private String tableName; From e7610c229a1f36f0e73b406e35e9a6e83cc4091d Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 5 Nov 2020 20:02:10 +0800 Subject: [PATCH 487/523] =?UTF-8?q?[fix]=20add=20jar=20=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E5=8F=8A=E6=B3=A8=E9=87=8A=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/parser/SqlParser.java | 15 +++++++-------- .../com/dtstack/flink/sql/util/DtStringUtil.java | 11 +++-------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index badcf8d9c..f91aa63c4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -16,7 +16,7 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.parser; @@ -28,7 +28,6 @@ import com.google.common.collect.Lists; import com.google.common.base.Strings; -import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.regex.Matcher; @@ -54,7 +53,7 @@ public static void setLocalSqlPluginRoot(String localSqlPluginRoot){ LOCAL_SQL_PLUGIN_ROOT = localSqlPluginRoot; } - private static final Pattern ADD_FIlE_PATTERN = Pattern.compile("(?i).*add\\s+file\\s+.+"); + private static final Pattern ADD_FILE_AND_JAR_PATTERN = Pattern.compile("(?i).*add\\s+file\\s+.+|(?i).*add\\s+jar\\s+.+"); /** * flink support sql syntax @@ -79,7 +78,7 @@ public static SqlTree parseSql(String sql) throws Exception { .replace("\t", " ").trim(); List sqlArr = DtStringUtil.splitIgnoreQuota(sql, SQL_DELIMITER); - sqlArr = removeAddFileStmt(sqlArr); + sqlArr = removeAddFileAndJarStmt(sqlArr); SqlTree sqlTree = new SqlTree(); AbstractTableInfoParser tableInfoParser = new AbstractTableInfoParser(); for(String childSql : sqlArr){ @@ -162,12 +161,12 @@ public static SqlTree parseSql(String sql) throws Exception { } /** - * remove add file with statment etc. add file /etc/krb5.conf; + * remove add file and jar with statment etc. add file /etc/krb5.conf, add jar xxx.jar; */ - private static List removeAddFileStmt(List stmts) { - List cleanedStmts = new ArrayList<>(); + private static List removeAddFileAndJarStmt(List stmts) { + List cleanedStmts = Lists.newArrayList(); for (String stmt : stmts) { - Matcher matcher = ADD_FIlE_PATTERN.matcher(stmt); + Matcher matcher = ADD_FILE_AND_JAR_PATTERN.matcher(stmt); if(!matcher.matches()) { cleanedStmts.add(stmt); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index 3e2ebb82a..f47356c61 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -167,26 +167,20 @@ public static String replaceIgnoreQuota(String str, String oriStr, String replac public static String dealSqlComment(String sql) { boolean inQuotes = false; boolean inSingleQuotes = false; - int bracketLeftNum = 0; StringBuilder b = new StringBuilder(sql.length()); char[] chars = sql.toCharArray(); for (int index = 0; index < chars.length; index ++) { - if (index == chars.length) { - return b.toString(); - } StringBuilder tempSb = new StringBuilder(2); - if (index > 1) { + if (index >= 1) { tempSb.append(chars[index - 1]); tempSb.append(chars[index]); } - if (tempSb.toString().equals("--")) { + if ("--".equals(tempSb.toString())) { if (inQuotes) { b.append(chars[index]); } else if (inSingleQuotes) { b.append(chars[index]); - } else if (bracketLeftNum > 0) { - b.append(chars[index]); } else { b.deleteCharAt(b.length() - 1); while (chars[index] != '\n') { @@ -210,6 +204,7 @@ public static String dealSqlComment(String sql) { return b.toString(); } + public static String col2string(Object column, String type) { String rowData = column.toString(); ColumnType columnType = ColumnType.valueOf(type.toUpperCase()); From b3de9f1fa9c5cb213ca8c9e5f7b585bd33323eb2 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 6 Nov 2020 15:09:17 +0800 Subject: [PATCH 488/523] =?UTF-8?q?[opt]=20=E4=BC=98=E5=8C=96impala?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/impala/ImpalaOutputFormat.java | 89 ++++++++++--------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index a755ee2bf..b3271a36f 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -34,7 +34,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.rmi.RemoteException; import java.security.PrivilegedExceptionAction; import java.sql.Connection; import java.sql.DriverManager; @@ -52,7 +51,6 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -77,7 +75,7 @@ public class ImpalaOutputFormat extends AbstractDtRichOutputFormat cast('value' as string) cast(value as timestamp) -> cast('value' as timestamp) private static final Pattern TYPE_PATTERN = Pattern.compile("cast\\((.*) as (.*)\\)"); //specific type which values need to be quoted - private static final String[] NEED_QUOTE_TYPE = {"string", "timestamp", "varchar"}; + private static final String[] NEED_QUOTE_TYPE = {"timestamp", "varchar"}; private static final Integer DEFAULT_CONN_TIME_OUT = 60; private static final int RECEIVE_DATA_PRINT_FREQUENCY = 1000; @@ -162,7 +160,7 @@ public void open(int taskNumber, int numTasks) throws IOException { init(); initMetric(); } catch (Exception e) { - throw new RemoteException("impala output format open error!", e); + throw new RuntimeException("impala output format open error!", e); } } @@ -189,19 +187,23 @@ private void init() throws SQLException { } private void initScheduledTask(Long batchWaitInterval) { - if (batchWaitInterval != 0) { - this.scheduler = new ScheduledThreadPoolExecutor(1, - new DTThreadFactory("impala-upsert-output-format")); - this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(() -> { - synchronized (ImpalaOutputFormat.this) { - try { - flush(); - } catch (Exception e) { - LOG.error("Writing records to impala jdbc failed.", e); - throw new RuntimeException("Writing records to impala jdbc failed.", e); + try { + if (batchWaitInterval != 0) { + this.scheduler = new ScheduledThreadPoolExecutor(1, + new DTThreadFactory("impala-upsert-output-format")); + this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(() -> { + synchronized (ImpalaOutputFormat.this) { + try { + flush(); + } catch (Exception e) { + LOG.error("Writing records to impala jdbc failed.", e); + throw new RuntimeException("Writing records to impala jdbc failed.", e); + } } - } - }, batchWaitInterval, batchWaitInterval, TimeUnit.MILLISECONDS); + }, batchWaitInterval, batchWaitInterval, TimeUnit.MILLISECONDS); + } + } catch (Exception e) { + throw new RuntimeException(e); } } @@ -235,7 +237,7 @@ private void openJdbc() { } } - private synchronized void flush() throws SQLException { + private void flush() throws Exception { if (batchCount > 0) { if (updateMode.equalsIgnoreCase(UPDATE_MODE)) { executeUpdateBatch(); @@ -387,7 +389,7 @@ private String valueConditionAddQuotation(String valueCondition) { } @Override - public synchronized void writeRecord(Tuple2 record) throws IOException { + public void writeRecord(Tuple2 record) throws IOException { try { if (!record.f0) { return; @@ -433,7 +435,7 @@ public synchronized void writeRecord(Tuple2 record) throws IOExcep // Receive data outRecords.inc(); } catch (Exception e) { - throw new RuntimeException("Writing records to impala failed.", e); + throw new IOException("Writing records to impala failed.", e); } } @@ -469,7 +471,7 @@ public void close() throws IOException { updateStatement.close(); } } catch (SQLException e) { - throw new RemoteException("impala connection close failed!"); + throw new RuntimeException("impala connection close failed!", e); } finally { connection = null; statement = null; @@ -489,17 +491,16 @@ public void close() throws IOException { * @param fieldNames field name list * @param partitionFields partition fields * @param rowDataMap row data map - * @throws SQLException throw sql exception */ - private synchronized void executeBatchSql(Statement statement, - String tempSql, - String schema, - String tableName, - String storeType, - Boolean enablePartition, - List fieldNames, - String partitionFields, - Map> rowDataMap) throws SQLException { + private void executeBatchSql(Statement statement, + String tempSql, + String schema, + String tableName, + String storeType, + Boolean enablePartition, + List fieldNames, + String partitionFields, + Map> rowDataMap) { StringBuilder valuesCondition = new StringBuilder(); StringBuilder partitionCondition = new StringBuilder(); String tableFieldsCondition = buildTableFieldsCondition(fieldNames, partitionFields); @@ -510,21 +511,25 @@ private synchronized void executeBatchSql(Statement statement, // kudu ${partitionCondition} is null if (storeType.equalsIgnoreCase(KUDU_TYPE) || !enablePartition) { - rowData = rowDataMap.get(NO_PARTITION); - rowData.forEach(row -> valuesCondition.append(row).append(", ")); - String executeSql = tempSql.replace(VALUES_CONDITION, valuesCondition.toString()) - .replace(PARTITION_CONDITION, partitionCondition.toString()) - .replace(PARTITION_CONSTANT, "") - .replace(TABLE_FIELDS_CONDITION, tableFieldsCondition); - String substring = executeSql.substring(0, executeSql.length() - 2); - statement.execute(substring); + try { + rowData = rowDataMap.get(NO_PARTITION); + rowData.forEach(row -> valuesCondition.append(row).append(", ")); + String executeSql = tempSql.replace(VALUES_CONDITION, valuesCondition.toString()) + .replace(PARTITION_CONDITION, partitionCondition.toString()) + .replace(PARTITION_CONSTANT, "") + .replace(TABLE_FIELDS_CONDITION, tableFieldsCondition); + String substring = executeSql.substring(0, executeSql.length() - 2); + statement.execute(substring); + } catch (Exception e) { + throw new RuntimeException("execute impala SQL error!", e); + } return; } // partition sql Set keySet = rowDataMap.keySet(); String finalTempSql = tempSql; - keySet.forEach(key -> { + for (String key : keySet) { try { String executeSql = String.copyValueOf(finalTempSql.toCharArray()); ArrayList valuesConditionList = rowDataMap.get(key); @@ -535,9 +540,9 @@ private synchronized void executeBatchSql(Statement statement, statement.execute(executeSql); partitionCondition.delete(0, partitionCondition.length()); } catch (SQLException sqlException) { - throw new RuntimeException("execute impala partition SQL error! ", sqlException); + throw new RuntimeException("execute impala SQL error! ", sqlException); } - }); + } } /** @@ -583,7 +588,7 @@ private String buildTableFieldsCondition(List fieldNames, String partiti private String buildValuesCondition(List fieldTypes, Row row) { String valuesCondition = fieldTypes.stream().map( f -> { - for(String item : NEED_QUOTE_TYPE) { + for (String item : NEED_QUOTE_TYPE) { if (f.toLowerCase().contains(item)) { return String.format("cast(? as %s)", f.toLowerCase()); } From d733759d56cd04681b3aeb277187a14d346a029b Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 10 Nov 2020 15:58:34 +0800 Subject: [PATCH 489/523] =?UTF-8?q?[fix-32161]=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E4=BD=BF=E7=94=A8udf=E8=AF=AD=E6=B3=95=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E7=B1=BB=E5=86=B2=E7=AA=81=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/dtstack/flink/sql/GetPlan.java | 10 ++++++++++ .../dtstack/flink/sql/exec/ExecuteProcessHelper.java | 8 ++++++-- .../java/com/dtstack/flink/sql/exec/ParamsInfo.java | 12 ++++++++++-- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/GetPlan.java b/core/src/main/java/com/dtstack/flink/sql/GetPlan.java index ba6518b3d..409224577 100644 --- a/core/src/main/java/com/dtstack/flink/sql/GetPlan.java +++ b/core/src/main/java/com/dtstack/flink/sql/GetPlan.java @@ -24,6 +24,9 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import java.net.URL; +import java.net.URLClassLoader; + /** * local模式获取sql任务的执行计划 * Date: 2020/2/17 @@ -33,15 +36,22 @@ public class GetPlan { public static String getExecutionPlan(String[] args) { + ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); + ClassLoader envClassLoader = StreamExecutionEnvironment.class.getClassLoader(); + ClassLoader plannerClassLoader = new URLClassLoader(new URL[0], envClassLoader); try { long start = System.currentTimeMillis(); ParamsInfo paramsInfo = ExecuteProcessHelper.parseParams(args); + paramsInfo.setGetPlan(true); + Thread.currentThread().setContextClassLoader(plannerClassLoader); StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExecution(paramsInfo); String executionPlan = env.getExecutionPlan(); long end = System.currentTimeMillis(); return ApiResult.createSuccessResultJsonStr(executionPlan, end - start); } catch (Exception e) { return ApiResult.createErrorResultJsonStr(ExceptionUtils.getFullStackTrace(e)); + } finally { + Thread.currentThread().setContextClassLoader(currentClassLoader); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 5692acf1c..cdfacb733 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -159,7 +159,7 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf Map registerTableCache = Maps.newHashMap(); //register udf - ExecuteProcessHelper.registerUserDefinedFunction(sqlTree, paramsInfo.getJarUrlList(), tableEnv); + ExecuteProcessHelper.registerUserDefinedFunction(sqlTree, paramsInfo.getJarUrlList(), tableEnv, paramsInfo.isGetPlan()); //register table schema Set classPathSets = ExecuteProcessHelper.registerTable(sqlTree, env, tableEnv, paramsInfo.getLocalSqlPluginPath(), paramsInfo.getRemoteSqlPluginPath(), paramsInfo.getPluginLoadMode(), sideTableMap, registerTableCache); @@ -245,13 +245,17 @@ private static void sqlTranslation(String localSqlPluginPath, } } - public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrlList, TableEnvironment tableEnv) + public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrlList, TableEnvironment tableEnv, boolean isGetPlan) throws IllegalAccessException, InvocationTargetException { // udf和tableEnv须由同一个类加载器加载 ClassLoader levelClassLoader = tableEnv.getClass().getClassLoader(); + ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); URLClassLoader classLoader = null; List funcList = sqlTree.getFunctionList(); for (CreateFuncParser.SqlParserResult funcInfo : funcList) { + if (isGetPlan) { + classLoader = ClassLoaderManager.loadExtraJar(jarUrlList, (URLClassLoader) currentClassLoader); + } //classloader if (classLoader == null) { classLoader = ClassLoaderManager.loadExtraJar(jarUrlList, (URLClassLoader) levelClassLoader); diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java index 27cc7702d..9619c75c1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java @@ -21,6 +21,7 @@ import java.net.URL; import java.util.List; +import java.util.Objects; import java.util.Properties; /** @@ -39,6 +40,7 @@ public class ParamsInfo { private String pluginLoadMode; private String deployMode; private Properties confProp; + private boolean getPlan = false; public ParamsInfo(String sql, String name, List jarUrlList, String localSqlPluginPath, String remoteSqlPluginPath, String pluginLoadMode, String deployMode, Properties confProp) { @@ -52,6 +54,14 @@ public ParamsInfo(String sql, String name, List jarUrlList, String localSql this.confProp = confProp; } + public boolean isGetPlan() { + return getPlan; + } + + public void setGetPlan(boolean getPlan) { + this.getPlan = getPlan; + } + public String getSql() { return sql; } @@ -114,10 +124,8 @@ public static class Builder { private String remoteSqlPluginPath; private String pluginLoadMode; private String deployMode; - private String logLevel; private Properties confProp; - public ParamsInfo.Builder setSql(String sql) { this.sql = sql; return this; From ba0ee350efd4d6356bccae764a91e3657b5b291d Mon Sep 17 00:00:00 2001 From: wuren Date: Thu, 12 Nov 2020 20:46:18 +0800 Subject: [PATCH 490/523] [fix-32327][rdb] make rdb connect pool size smaller --- .../com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index ba37c4843..abd7dd50b 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -74,7 +74,7 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { public final static int DEFAULT_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; - public final static int MAX_DB_CONN_POOL_SIZE_LIMIT = 20; + public final static int MAX_DB_CONN_POOL_SIZE_LIMIT = 5; public final static int DEFAULT_IDLE_CONNECTION_TEST_PEROID = 60; From 210dc297329ea1847bfab08ca9f93321285c9ab2 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 13 Nov 2020 14:08:12 +0800 Subject: [PATCH 491/523] =?UTF-8?q?[fix-31988]\=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8Dstring=E7=B1=BB=E5=9E=8B=E8=BD=AC=E5=8C=96=E5=BC=82?= =?UTF-8?q?=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index b3271a36f..35b0c9a50 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -75,7 +75,7 @@ public class ImpalaOutputFormat extends AbstractDtRichOutputFormat cast('value' as string) cast(value as timestamp) -> cast('value' as timestamp) private static final Pattern TYPE_PATTERN = Pattern.compile("cast\\((.*) as (.*)\\)"); //specific type which values need to be quoted - private static final String[] NEED_QUOTE_TYPE = {"timestamp", "varchar"}; + private static final String[] NEED_QUOTE_TYPE = {"string", "timestamp", "varchar"}; private static final Integer DEFAULT_CONN_TIME_OUT = 60; private static final int RECEIVE_DATA_PRINT_FREQUENCY = 1000; @@ -759,7 +759,5 @@ public ImpalaOutputFormat build() { return format; } - } - } From 47c4a28b1750164f900430a2c6b017662156bc1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Tue, 17 Nov 2020 14:44:15 +0800 Subject: [PATCH 492/523] =?UTF-8?q?[fix-32324][rdb-all]=E4=BF=AE=E5=A4=8Dr?= =?UTF-8?q?db=E5=85=A8=E9=87=8F=E7=BB=B4=E8=A1=A8=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E5=8C=B9=E9=85=8D=E5=BC=82=E5=B8=B8=EF=BC=8C=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E6=83=85=E5=86=B5=EF=BC=9ADDL=E4=B8=AD=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E4=B8=BAid,=20name,=20age,=20=E5=8F=AASELECT=20id,=20age?= =?UTF-8?q?=E4=B8=A4=E4=B8=AA=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 0db947d42..7badb736f 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -213,10 +213,11 @@ private void queryAndFillData(Map>> tmpCache, C ResultSet resultSet = statement.executeQuery(sql); String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); + String[] fields = sideInfo.getSideTableInfo().getFields(); String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); Map sideFieldNamesAndTypes = Maps.newHashMap(); - for (int i = 0; i < sideFieldNames.length; i++) { - sideFieldNamesAndTypes.put(sideFieldNames[i], sideFieldTypes[i]); + for (int i = 0; i < fields.length; i++) { + sideFieldNamesAndTypes.put(fields[i], sideFieldTypes[i]); } while (resultSet.next()) { From d5557748f004967be787df48308eb8502db3d57b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Fri, 27 Nov 2020 10:08:45 +0800 Subject: [PATCH 493/523] [fix-32869][rdb]fix rdb task hangs after connect retry. --- .../com/dtstack/flink/sql/util/JDBCUtils.java | 80 +++++++++++++++++++ .../side/rdb/all/AbstractRdbAllReqRow.java | 46 +++++------ 2 files changed, 99 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java index faab22172..42d7b9356 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java @@ -19,9 +19,19 @@ package com.dtstack.flink.sql.util; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Objects; public class JDBCUtils { + private static final Logger LOG = LoggerFactory.getLogger(JDBCUtils.class); + private static final Object LOCK = new Object(); public static void forName(String clazz, ClassLoader classLoader) { @@ -44,4 +54,74 @@ public synchronized static void forName(String clazz) { throw new RuntimeException(e); } } + + /** + * 关闭连接资源 + * + * @param rs ResultSet + * @param stmt Statement + * @param conn Connection + * @param commit + */ + public static void closeConnectionResource(ResultSet rs, Statement stmt, Connection conn, boolean commit) { + if (Objects.nonNull(rs)) { + try { + rs.close(); + } catch (SQLException e) { + LOG.warn("Close resultSet error: {}", e.getMessage()); + } + } + + if (Objects.nonNull(stmt)) { + try { + stmt.close(); + } catch (SQLException e) { + LOG.warn("Close statement error:{}", e.getMessage()); + } + } + + if (Objects.nonNull(conn)) { + try { + if (commit) { + commit(conn); + } else { + rollBack(conn); + } + + conn.close(); + } catch (SQLException e) { + LOG.warn("Close connection error:{}", e.getMessage()); + } + } + } + + /** + * 手动提交事物 + * + * @param conn Connection + */ + public static void commit(Connection conn) { + try { + if (!conn.isClosed() && !conn.getAutoCommit()) { + conn.commit(); + } + } catch (SQLException e) { + LOG.warn("commit error:{}", e.getMessage()); + } + } + + /** + * 手动回滚事物 + * + * @param conn Connection + */ + public static void rollBack(Connection conn) { + try { + if (!conn.isClosed() && !conn.getAutoCommit()) { + conn.rollback(); + } + } catch (SQLException e) { + LOG.warn("rollBack error:{}", e.getMessage()); + } + } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 7badb736f..082784041 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.rdb.all; +import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.configuration.Configuration; import org.apache.flink.table.runtime.types.CRow; @@ -167,42 +168,32 @@ protected Object dealTimeAttributeType(Class entry, O boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(entry); if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 - obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); + obj = ((Timestamp) obj).getTime() + (long) LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } return obj; } private void loadData(Map>> tmpCache) throws SQLException { - RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - Connection connection = null; + queryAndFillData(tmpCache, getConnectionWithRetry((RdbSideTableInfo) sideInfo.getSideTableInfo())); + } - try { - for (int i = 0; i < CONN_RETRY_NUM; i++) { + private Connection getConnectionWithRetry(RdbSideTableInfo tableInfo) throws SQLException { + String connInfo = "url:" + tableInfo.getUrl() + "; userName:" + tableInfo.getUserName(); + String errorMsg = null; + for (int i = 0; i < CONN_RETRY_NUM; i++) { + try { + return getConn(tableInfo.getUrl(), tableInfo.getUserName(), tableInfo.getPassword()); + } catch (Exception e) { try { - connection = getConn(tableInfo.getUrl(), tableInfo.getUserName(), tableInfo.getPassword()); - break; - } catch (Exception e) { - if (i == CONN_RETRY_NUM - 1) { - throw new RuntimeException("", e); - } - try { - String connInfo = "url:" + tableInfo.getUrl() + ";userName:" + tableInfo.getUserName() + ",pwd:" + tableInfo.getPassword(); - LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); - Thread.sleep(5 * 1000); - } catch (InterruptedException e1) { - LOG.error("", e1); - } + LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); + errorMsg = e.getCause().toString(); + Thread.sleep(5 * 1000); + } catch (InterruptedException e1) { + LOG.error("", e1); } } - queryAndFillData(tmpCache, connection); - } catch (Exception e) { - LOG.error("", e); - throw new SQLException(e); - } finally { - if (connection != null) { - connection.close(); - } } + throw new SQLException("get conn fail. connInfo: " + connInfo + "\ncause by: " + errorMsg); } private void queryAndFillData(Map>> tmpCache, Connection connection) throws SQLException { @@ -213,8 +204,8 @@ private void queryAndFillData(Map>> tmpCache, C ResultSet resultSet = statement.executeQuery(sql); String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); - String[] fields = sideInfo.getSideTableInfo().getFields(); String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); + String[] fields = sideInfo.getSideTableInfo().getFields(); Map sideFieldNamesAndTypes = Maps.newHashMap(); for (int i = 0; i < fields.length; i++) { sideFieldNamesAndTypes.put(fields[i], sideFieldTypes[i]); @@ -236,6 +227,7 @@ private void queryAndFillData(Map>> tmpCache, C tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()) .add(oneRow); } + JDBCUtils.closeConnectionResource(resultSet, statement, connection, false); } public int getFetchSize() { From f22586705647a45a820de883bd1f5e5e2f9e3f05 Mon Sep 17 00:00:00 2001 From: chuixue Date: Tue, 1 Dec 2020 15:55:45 +0800 Subject: [PATCH 494/523] [fix-32989][rdb]rdb sink table can not delete --- .../flink/sql/sink/rdb/writer/AbstractUpsertWriter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java index bc4f763a5..9328257a5 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -122,7 +122,9 @@ public void addRecord(Tuple2 record) throws SQLException { // we don't need perform a deep copy, because jdbc field are immutable object. Tuple2 tuple2 = objectReuse ? new Tuple2<>(record.f0, Row.copy(record.f1)) : record; // add records to buffer - keyToRows.put(getPrimaryKey(tuple2.f1), tuple2); + if (tuple2.f0) { + keyToRows.put(getPrimaryKey(tuple2.f1), tuple2); + } } @Override From 81e6f8e0ed003e646fc4a36bcb8a6ec092a33203 Mon Sep 17 00:00:00 2001 From: xuchao Date: Thu, 3 Dec 2020 11:13:10 +0800 Subject: [PATCH 495/523] =?UTF-8?q?[fix-32973][core]=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E7=BB=B4=E8=A1=A8join=20=E5=90=8C=E5=90=8Dfield=E6=9B=BF?= =?UTF-8?q?=E6=8D=A2=E9=80=BB=E8=BE=91=E4=B8=8D=E4=B8=80=E8=87=B4=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E5=8F=96=E5=80=BC=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98?= =?UTF-8?q?=C2=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/BaseAsyncReqRow.java | 2 +- .../flink/sql/side/JoinNodeDealer.java | 17 +++++++------ .../dtstack/flink/sql/util/TableUtils.java | 25 ++++++------------- .../sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 4 files changed, 19 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index ae8c75f7e..5f85e434e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -181,7 +181,7 @@ public void asyncInvoke(CRow row, ResultFuture resultFuture) throws Except } private Map parseInputParam(CRow input){ - Map inputParams = Maps.newHashMap(); + Map inputParams = Maps.newLinkedHashMap(); for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = input.row().getField(conValIndex); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index 6afc12462..a65a53235 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -285,7 +285,7 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, SqlBasicCall buildAs = TableUtils.buildAsNodeByJoinInfo(joinInfo, null, null); if(rightIsSide){ - addSideInfoToExeQueue(queueInfo, joinInfo, joinNode, parentSelectList, parentGroupByList, parentWhere, tableRef); + addSideInfoToExeQueue(queueInfo, joinInfo, joinNode, parentSelectList, parentGroupByList, parentWhere, tableRef, fieldRef); } SqlNode newLeftNode = joinNode.getLeft(); @@ -298,7 +298,7 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, //替换leftNode 为新的查询 joinNode.setLeft(buildAs); - replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, parentSelectList, parentGroupByList, parentWhere); + replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, fieldRef, parentSelectList, parentGroupByList, parentWhere); } return joinInfo; @@ -321,7 +321,8 @@ public void addSideInfoToExeQueue(Queue queueInfo, SqlNodeList parentSelectList, SqlNodeList parentGroupByList, SqlNode parentWhere, - Map tableRef){ + Map tableRef, + Map fieldRef){ //只处理维表 if(!joinInfo.isRightIsSideTable()){ return; @@ -333,7 +334,7 @@ public void addSideInfoToExeQueue(Queue queueInfo, //替换左表为新的表名称 joinNode.setLeft(buildAs); - replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, parentSelectList, parentGroupByList, parentWhere); + replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, fieldRef, parentSelectList, parentGroupByList, parentWhere); } /** @@ -348,6 +349,7 @@ public void addSideInfoToExeQueue(Queue queueInfo, public void replaceSelectAndWhereField(SqlBasicCall buildAs, SqlNode leftJoinNode, Map tableRef, + Map fieldRef, SqlNodeList parentSelectList, SqlNodeList parentGroupByList, SqlNode parentWhere){ @@ -361,23 +363,22 @@ public void replaceSelectAndWhereField(SqlBasicCall buildAs, } //替换select field 中的对应字段 - HashBiMap fieldReplaceRef = HashBiMap.create(); for(SqlNode sqlNode : parentSelectList.getList()){ for(String tbTmp : fromTableNameSet) { - TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldReplaceRef); + TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldRef); } } //TODO 应该根据上面的查询字段的关联关系来替换 //替换where 中的条件相关 for(String tbTmp : fromTableNameSet){ - TableUtils.replaceWhereCondition(parentWhere, tbTmp, newLeftTableName, fieldReplaceRef); + TableUtils.replaceWhereCondition(parentWhere, tbTmp, newLeftTableName, fieldRef); } if(parentGroupByList != null){ for(SqlNode sqlNode : parentGroupByList.getList()){ for(String tbTmp : fromTableNameSet) { - TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldReplaceRef); + TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldRef); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 2aeb06cf8..30ef8b4b3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -301,7 +301,7 @@ public static void getFromTableInfo(SqlNode fromTable, Set tableNameSet) public static void replaceSelectFieldTable(SqlNode selectNode, String oldTbName, String newTbName, - HashBiMap fieldReplaceRef) { + Map fieldReplaceRef) { if (selectNode.getKind() == AS) { SqlNode leftNode = ((SqlBasicCall) selectNode).getOperands()[0]; replaceSelectFieldTable(leftNode, oldTbName, newTbName, fieldReplaceRef); @@ -395,22 +395,13 @@ public static void replaceSelectFieldTable(SqlNode selectNode, private static void replaceOneSelectField(SqlIdentifier sqlIdentifier, String newTbName, String oldTbName, - HashBiMap fieldReplaceRef){ + Map fieldReplaceRef){ SqlIdentifier newField = sqlIdentifier.setName(0, newTbName); String fieldName = sqlIdentifier.names.get(1); - String fieldKey = oldTbName + "_" + fieldName; - - if(!fieldReplaceRef.containsKey(fieldKey)){ - if(fieldReplaceRef.inverse().get(fieldName) != null){ - //换一个名字 - String mappingFieldName = ParseUtils.dealDuplicateFieldName(fieldReplaceRef, fieldName); - newField = newField.setName(1, mappingFieldName); - fieldReplaceRef.put(fieldKey, mappingFieldName); - } else { - fieldReplaceRef.put(fieldKey, fieldName); - } - }else { - newField = newField.setName(1, fieldReplaceRef.get(fieldKey)); + String fieldKey = oldTbName + "." + fieldName; + if(fieldReplaceRef.get(fieldKey) != null){ + String newFieldName = fieldReplaceRef.get(fieldKey).split("\\.")[1]; + newField = newField.setName(1, newFieldName); } sqlIdentifier.assignNamesFrom(newField); @@ -511,7 +502,7 @@ public static String getTargetRefField(Map refFieldMap, String c return preFieldName; } - public static void replaceWhereCondition(SqlNode parentWhere, String oldTbName, String newTbName, HashBiMap fieldReplaceRef){ + public static void replaceWhereCondition(SqlNode parentWhere, String oldTbName, String newTbName, Map fieldReplaceRef){ if(parentWhere == null){ return; @@ -527,7 +518,7 @@ public static void replaceWhereCondition(SqlNode parentWhere, String oldTbName, } } - private static void replaceConditionNode(SqlNode selectNode, String oldTbName, String newTbName, HashBiMap fieldReplaceRef) { + private static void replaceConditionNode(SqlNode selectNode, String oldTbName, String newTbName, Map fieldReplaceRef) { if(selectNode.getKind() == IDENTIFIER){ SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index abd7dd50b..2bcbd1f28 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -313,7 +313,7 @@ private void handleQuery(SQLConnection connection, Map inputPara } private Map formatInputParam(Map inputParam){ - Map result = Maps.newHashMap(); + Map result = Maps.newLinkedHashMap(); inputParam.forEach((k,v) -> { result.put(k, convertDataType(v)); }); From a4d1b82af371b66f3e38ec36cb1f67986f729e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Tue, 15 Dec 2020 18:58:35 +0800 Subject: [PATCH 496/523] [fix-33268][core] fix hbase rowKey not available due to remove all single quota. rowKey like 'stu'+'_'+sid+'_'+md5(sid). --- .../java/com/dtstack/flink/sql/parser/CreateTableParser.java | 3 ++- .../main/java/com/dtstack/flink/sql/util/DtStringUtil.java | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java index 925531240..1486e8bfa 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java @@ -20,6 +20,7 @@ import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Map; @@ -72,7 +73,7 @@ private Map parseProp(String propsStr){ for (String str : strings) { List ss = DtStringUtil.splitIgnoreQuota(str, '='); String key = ss.get(0).trim(); - String value = ss.get(1).trim().replaceAll("'", "").trim(); + String value = DtStringUtil.removeStartAndEndQuota(ss.get(1).trim()); propMap.put(key, value); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index f47356c61..5af657512 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -408,4 +408,9 @@ public static String getStartQuote() { public static String getEndQuote() { return "\""; } + + public static String removeStartAndEndQuota(String str) { + String removeStart = StringUtils.removeStart(str, "'"); + return StringUtils.removeEnd(removeStart, "'"); + } } From 834e82da46577c75867de73db0f935c178af121f Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 29 Dec 2020 10:13:36 +0800 Subject: [PATCH 497/523] [fix-33847][core][kafka] add array and map types support in the kafka sink. --- .../flink/sql/exec/ExecuteProcessHelper.java | 4 +- .../flink/sql/function/FunctionManager.java | 9 ---- .../com/dtstack/flink/sql/util/ClassUtil.java | 6 +++ .../dtstack/flink/sql/util/DataTypeUtils.java | 45 ++++++++++++------- .../sql/sink/kafka/AbstractKafkaSink.java | 16 ++++++- 5 files changed, 50 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 5692acf1c..b7af4822f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -315,9 +315,7 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment } else if (tableInfo instanceof AbstractTargetTableInfo) { TableSink tableSink = StreamSinkFactory.getTableSink((AbstractTargetTableInfo) tableInfo, localSqlPluginPath); - TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); - tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); - + tableEnv.registerTableSink(tableInfo.getName(), tableSink); URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPathSets.add(sinkTablePathUrl); } else if (tableInfo instanceof AbstractSideTableInfo) { diff --git a/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java b/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java index 360cc82ab..5409ac36f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java @@ -122,13 +122,4 @@ public static void registerAggregateUDF(String classPath, String funcName, Table } } - - public static TypeInformation[] transformTypes(Class[] fieldTypes) { - TypeInformation[] types = new TypeInformation[fieldTypes.length]; - for (int i = 0; i < fieldTypes.length; i++) { - types[i] = TypeInformation.of(fieldTypes[i]); - } - - return types; - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index 21551364f..aa293616f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -25,6 +25,8 @@ import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; +import java.util.HashMap; +import java.util.Map; /** * Reason: TODO ADD REASON(可选) @@ -41,6 +43,10 @@ public static Class stringConvertClass(String str) { if (lowerStr.startsWith("array")) { return Array.newInstance(Integer.class, 0).getClass(); } + if (lowerStr.startsWith("map")) { + Map m = new HashMap(); + return m.getClass(); + } switch (lowerStr) { case "boolean": diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index 72a8b4b66..bb15b9c45 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -20,6 +20,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -30,6 +31,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.apache.commons.lang3.StringUtils.split; + /** * @program: flink.sql * @author: wuren @@ -39,28 +42,13 @@ public class DataTypeUtils { private final static Pattern COMPOSITE_TYPE_PATTERN = Pattern.compile("(.+?)<(.+)>"); private final static String ARRAY = "ARRAY"; + private final static String MAP = "MAP"; private final static String ROW = "ROW"; private final static char FIELD_DELIMITER = ','; private final static char TYPE_DELIMITER = ' '; private DataTypeUtils() {} - /** - * 现在只支持ARRAY类型后续可以加入 MAP等类型 - * @param compositeTypeString - * @return - */ - public static TypeInformation convertToCompositeType(String compositeTypeString) { - Matcher matcher = matchCompositeType(compositeTypeString); - final String errorMsg = "type " + compositeTypeString + "is not support!"; - Preconditions.checkState(matcher.find(), errorMsg); - - String normalizedType = normalizeType(matcher.group(1)); - Preconditions.checkState(ARRAY.equals(normalizedType), errorMsg); - - return convertToArray(compositeTypeString); - } - /** * 目前ARRAY里只支持ROW和其他基本类型 * @param arrayTypeString @@ -86,6 +74,30 @@ public static TypeInformation convertToArray(String arrayTypeString) { return Types.OBJECT_ARRAY(elementType); } + /** + * 目前Map里只支持基本类型 + * @param mapTypeString + * @return + */ + public static TypeInformation convertToMap(String mapTypeString) { + Matcher matcher = matchCompositeType(mapTypeString); + final String errorMsg = mapTypeString + "convert to map type error!"; + Preconditions.checkState(matcher.find(), errorMsg); + + String normalizedType = normalizeType(matcher.group(1)); + Preconditions.checkState(MAP.equals(normalizedType), errorMsg); + + String kvTypeString = matcher.group(2); + String[] kvTypeStringList = StringUtils.split(kvTypeString, ","); + final String mapTypeErrorMsg = "There can only be key and value two types in map declaration."; + Preconditions.checkState(kvTypeStringList.length == 2, mapTypeErrorMsg); + String keyTypeString = normalizeType(kvTypeStringList[0]); + String valueTypeString = normalizeType(kvTypeStringList[1]); + TypeInformation keyType = convertToAtomicType(keyTypeString); + TypeInformation valueType = convertToAtomicType(valueTypeString); + return Types.MAP(keyType, valueType); + } + /** * 目前ROW里只支持基本类型 * @param rowTypeString @@ -104,6 +116,7 @@ public static RowTypeInfo convertToRow(String rowTypeString) { return new RowTypeInfo(info.f0, info.f1); } + private static Tuple2 genFieldInfo(Iterable fieldInfoStrs) { ArrayList types = Lists.newArrayList(); ArrayList fieldNames = Lists.newArrayList(); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index 55d1fa7f8..27699c4f3 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; +import com.dtstack.flink.sql.util.DataTypeUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -38,6 +39,7 @@ import org.apache.flink.util.Preconditions; import org.apache.kafka.clients.consumer.ConsumerConfig; +import java.util.HashMap; import java.util.Optional; import java.util.Properties; import java.util.stream.IntStream; @@ -77,11 +79,21 @@ protected Properties getKafkaProperties(KafkaSinkTableInfo KafkaSinkTableInfo) { } return props; } - + // TODO Source有相同的方法日后可以合并 protected TypeInformation[] getTypeInformations(KafkaSinkTableInfo kafka11SinkTableInfo) { + String[] fieldTypes = kafka11SinkTableInfo.getFieldTypes(); Class[] fieldClasses = kafka11SinkTableInfo.getFieldClasses(); TypeInformation[] types = IntStream.range(0, fieldClasses.length) - .mapToObj(i -> TypeInformation.of(fieldClasses[i])) + .mapToObj( + i -> { + if (fieldClasses[i].isArray()) { + return DataTypeUtils.convertToArray(fieldTypes[i]); + } + if (fieldClasses[i] == new HashMap().getClass()) { + return DataTypeUtils.convertToMap(fieldTypes[i]); + } + return TypeInformation.of(fieldClasses[i]); + }) .toArray(TypeInformation[]::new); return types; } From 8b839324ef2b597f6527ad6f3d595381313b08ae Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 29 Dec 2020 13:47:55 +0800 Subject: [PATCH 498/523] [fix-33847][doc] add kafka sink map type doc. --- docs/plugin/kafkaSink.md | 26 ++++++++++++++++++++++++++ docs/pluginsInfo.md | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/plugin/kafkaSink.md b/docs/plugin/kafkaSink.md index 86b01f3ee..b3a1614bc 100644 --- a/docs/plugin/kafkaSink.md +++ b/docs/plugin/kafkaSink.md @@ -221,3 +221,29 @@ into from MyTable a ``` +## MAP类型示例 +目前Kafka Sink支持Map类型 +```sql +CREATE TABLE ods( + id INT, + name STRING +) WITH ( + ... +); + +CREATE TABLE dwd ( + id INT, + dids MAP> +) WITH ( + type ='kafka', + bootstrapServers ='localhost:9092', + offsetReset ='latest', + groupId='wuren_foo', + topic ='luna_foo', + parallelism ='1' +); + +INSERT INTO dwd + SELECT ods.id, MAP['foo', 1, 'bar', 2] AS dids + FROM ods; +``` diff --git a/docs/pluginsInfo.md b/docs/pluginsInfo.md index 88fda90c9..e9d23cdbd 100644 --- a/docs/pluginsInfo.md +++ b/docs/pluginsInfo.md @@ -1,9 +1,9 @@ ### 1 插件列表 #### 1.1 源表插件 * [kafka 源表插件](plugin/kafkaSource.md) -* [kafka 结果表插件](plugin/kafkaSink.md) #### 1.2 结果表插件 +* [kafka 结果表插件](plugin/kafkaSink.md) * [elasticsearch 结果表插件](plugin/elasticsearchSink.md) * [hbase 结果表插件](plugin/hbaseSink.md) * [mysql 结果表插件](plugin/mysqlSink.md) From cf00b23bf2fb7dfe2e27127cacb9a3f96d70fad3 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 29 Dec 2020 21:41:27 +0800 Subject: [PATCH 499/523] [fix-33847][core] fix other sink except kafka table schema not found problem. --- .../com/dtstack/flink/sql/exec/ExecuteProcessHelper.java | 9 ++++++++- .../com/dtstack/flink/sql/function/FunctionManager.java | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index b7af4822f..df5c6619b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -315,7 +315,14 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment } else if (tableInfo instanceof AbstractTargetTableInfo) { TableSink tableSink = StreamSinkFactory.getTableSink((AbstractTargetTableInfo) tableInfo, localSqlPluginPath); - tableEnv.registerTableSink(tableInfo.getName(), tableSink); + // TODO Kafka Sink直接注册，其他的Sink要修复才可以。 + if (tableInfo.getType().startsWith("kafka")) { + tableEnv.registerTableSink(tableInfo.getName(), tableSink); + } else { + TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); + tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); + } + URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPathSets.add(sinkTablePathUrl); } else if (tableInfo instanceof AbstractSideTableInfo) { diff --git a/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java b/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java index 5409ac36f..549276dc2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java @@ -122,4 +122,12 @@ public static void registerAggregateUDF(String classPath, String funcName, Table } } + public static TypeInformation[] transformTypes(Class[] fieldTypes) { + TypeInformation[] types = new TypeInformation[fieldTypes.length]; + for (int i = 0; i < fieldTypes.length; i++) { + types[i] = TypeInformation.of(fieldTypes[i]); + } + return types; + } + } From d58b2c73a8157eb5b7562ee5e66e497e8b05ae38 Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Thu, 31 Dec 2020 10:19:42 +0800 Subject: [PATCH 500/523] =?UTF-8?q?[hotfix-34172][core]=20fix=20flinkPlann?= =?UTF-8?q?er=20=E5=AF=BC=E8=87=B4=E7=9A=84metaspace=E5=86=85=E5=AD=98?= =?UTF-8?q?=E6=B3=84=E6=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/parser/FlinkPlanner.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java b/core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java index 7c76ec2cd..3f48d5049 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java @@ -36,12 +36,8 @@ private FlinkPlanner() { } public static FlinkPlannerImpl createFlinkPlanner(FrameworkConfig frameworkConfig, RelOptPlanner relOptPlanner, FlinkTypeFactory typeFactory) { - if (flinkPlanner == null) { - synchronized (FlinkPlanner.class) { - if (flinkPlanner == null) { - flinkPlanner = new FlinkPlannerImpl(frameworkConfig, relOptPlanner, typeFactory); - } - } + synchronized (FlinkPlanner.class) { + flinkPlanner = new FlinkPlannerImpl(frameworkConfig, relOptPlanner, typeFactory); } return flinkPlanner; } From 7de4d0432cba027dc28058576dd7a186b11334a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Thu, 31 Dec 2020 11:14:36 +0800 Subject: [PATCH 501/523] =?UTF-8?q?[fix-34156][rdb]=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E5=85=A8=E9=87=8F=E7=BB=B4=E8=A1=A8=E4=B8=BB=E9=94=AE=E5=AD=98?= =?UTF-8?q?=E5=9C=A8null=E5=80=BC=E6=97=B6=E5=87=BA=E7=8E=B0=E7=A9=BA?= =?UTF-8?q?=E6=8C=87=E9=92=88=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 0530e57e2..1f8cc2a7f 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -225,7 +225,7 @@ private void queryAndFillData(Map>> tmpCache, C String cacheKey = sideInfo.getEqualFieldList().stream() .map(oneRow::get) - .map(Object::toString) + .map(String::valueOf) .collect(Collectors.joining("_")); tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()) From 50dc15b64492bf9a70e5acb4f029a7fb25c6f007 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Thu, 31 Dec 2020 11:14:36 +0800 Subject: [PATCH 502/523] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=85=A8=E9=87=8F?= =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E4=B8=BB=E9=94=AE=E5=AD=98=E5=9C=A8null?= =?UTF-8?q?=E5=80=BC=E6=97=B6=E5=87=BA=E7=8E=B0=E7=A9=BA=E6=8C=87=E9=92=88?= =?UTF-8?q?=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 082784041..853060755 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -221,7 +221,7 @@ private void queryAndFillData(Map>> tmpCache, C String cacheKey = sideInfo.getEqualFieldList().stream() .map(oneRow::get) - .map(Object::toString) + .map(String::valueOf) .collect(Collectors.joining("_")); tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()) From 1b7936107963d1ebb65a6c1913f0c262c90505b4 Mon Sep 17 00:00:00 2001 From: HiLany Date: Mon, 4 Jan 2021 16:05:02 +0800 Subject: [PATCH 503/523] [fix-33981][elasticsearch6][sink]quota RequestLogger.class source code, adjust log level to debug when httpResponse.getHeaders has warning. --- .../elasticsearch/client/RequestLogger.java | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/org/elasticsearch/client/RequestLogger.java diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/org/elasticsearch/client/RequestLogger.java b/elasticsearch6/elasticsearch6-sink/src/main/java/org/elasticsearch/client/RequestLogger.java new file mode 100644 index 000000000..39d79e03a --- /dev/null +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/org/elasticsearch/client/RequestLogger.java @@ -0,0 +1,180 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.HttpEntityEnclosingRequest; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.RequestLine; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.entity.BufferedHttpEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.util.EntityUtils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +/** + * Quota this source code in 2021-01-04. + * Quota Reason: http://redmine.prod.dtstack.cn/issues/33981 + * Modify Content: adjust log level to debug when httpResponse.getHeaders has warning. + * + * Helper class that exposes static methods to unify the way requests are logged. + * Includes trace logging to log complete requests and responses in curl format. + * Useful for debugging, manually sending logged requests via curl and checking their responses. + * Trace logging is a feature that all the language clients provide. + */ +final class RequestLogger { + + private static final Log tracer = LogFactory.getLog("tracer"); + + private RequestLogger() { + } + + /** + * Logs a request that yielded a response + */ + static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpResponse httpResponse) { + if (logger.isDebugEnabled()) { + logger.debug("request [" + request.getMethod() + " " + host + getUri(request.getRequestLine()) + + "] returned [" + httpResponse.getStatusLine() + "]"); + } + // adjust log level to debug when httpResponse.getHeaders has warning. + if (logger.isDebugEnabled()) { + Header[] warnings = httpResponse.getHeaders("Warning"); + if (warnings != null && warnings.length > 0) { + logger.debug(buildWarningMessage(request, host, warnings)); + } + } + if (tracer.isTraceEnabled()) { + String requestLine; + try { + requestLine = buildTraceRequest(request, host); + } catch(IOException e) { + requestLine = ""; + tracer.trace("error while reading request for trace purposes", e); + } + String responseLine; + try { + responseLine = buildTraceResponse(httpResponse); + } catch(IOException e) { + responseLine = ""; + tracer.trace("error while reading response for trace purposes", e); + } + tracer.trace(requestLine + '\n' + responseLine); + } + } + + /** + * Logs a request that failed + */ + static void logFailedRequest(Log logger, HttpUriRequest request, Node node, Exception e) { + if (logger.isDebugEnabled()) { + logger.debug("request [" + request.getMethod() + " " + node.getHost() + getUri(request.getRequestLine()) + "] failed", e); + } + if (tracer.isTraceEnabled()) { + String traceRequest; + try { + traceRequest = buildTraceRequest(request, node.getHost()); + } catch (IOException e1) { + tracer.trace("error while reading request for trace purposes", e); + traceRequest = ""; + } + tracer.trace(traceRequest); + } + } + + static String buildWarningMessage(HttpUriRequest request, HttpHost host, Header[] warnings) { + StringBuilder message = new StringBuilder("request [").append(request.getMethod()).append(" ").append(host) + .append(getUri(request.getRequestLine())).append("] returned ").append(warnings.length).append(" warnings: "); + for (int i = 0; i < warnings.length; i++) { + if (i > 0) { + message.append(","); + } + message.append("[").append(warnings[i].getValue()).append("]"); + } + return message.toString(); + } + + /** + * Creates curl output for given request + */ + static String buildTraceRequest(HttpUriRequest request, HttpHost host) throws IOException { + String requestLine = "curl -iX " + request.getMethod() + " '" + host + getUri(request.getRequestLine()) + "'"; + if (request instanceof HttpEntityEnclosingRequest) { + HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request; + if (enclosingRequest.getEntity() != null) { + requestLine += " -d '"; + HttpEntity entity = enclosingRequest.getEntity(); + if (entity.isRepeatable() == false) { + entity = new BufferedHttpEntity(enclosingRequest.getEntity()); + enclosingRequest.setEntity(entity); + } + requestLine += EntityUtils.toString(entity, StandardCharsets.UTF_8) + "'"; + } + } + return requestLine; + } + + /** + * Creates curl output for given response + */ + static String buildTraceResponse(HttpResponse httpResponse) throws IOException { + StringBuilder responseLine = new StringBuilder(); + responseLine.append("# ").append(httpResponse.getStatusLine()); + for (Header header : httpResponse.getAllHeaders()) { + responseLine.append("\n# ").append(header.getName()).append(": ").append(header.getValue()); + } + responseLine.append("\n#"); + HttpEntity entity = httpResponse.getEntity(); + if (entity != null) { + if (entity.isRepeatable() == false) { + entity = new BufferedHttpEntity(entity); + } + httpResponse.setEntity(entity); + ContentType contentType = ContentType.get(entity); + Charset charset = StandardCharsets.UTF_8; + if (contentType != null && contentType.getCharset() != null) { + charset = contentType.getCharset(); + } + try (BufferedReader reader = new BufferedReader(new InputStreamReader(entity.getContent(), charset))) { + String line; + while( (line = reader.readLine()) != null) { + responseLine.append("\n# ").append(line); + } + } + } + return responseLine.toString(); + } + + private static String getUri(RequestLine requestLine) { + if (requestLine.getUri().charAt(0) != '/') { + return "/" + requestLine.getUri(); + } + return requestLine.getUri(); + } +} From ec96b760638b8b248ef9eed0e1981ef57a485ecd Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 5 Jan 2021 18:41:04 +0800 Subject: [PATCH 504/523] [fix-34344][kafka] remove partition key blank char. --- .../flink/sql/sink/kafka/AbstractKafkaSink.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index 27699c4f3..89be2840b 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -39,6 +39,7 @@ import org.apache.flink.util.Preconditions; import org.apache.kafka.clients.consumer.ConsumerConfig; +import java.util.Arrays; import java.util.HashMap; import java.util.Optional; import java.util.Properties; @@ -128,8 +129,13 @@ public CRowTypeInfo getRowTypeInfo() { } protected String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo) { - if (StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())) { - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + String keysStr = kafkaSinkTableInfo.getPartitionKeys(); + if (StringUtils.isNotBlank(keysStr)) { + String[] keys = keysStr.split(","); + String[] cleanedKeys = Arrays.stream(keys) + .map(x -> x.trim()) + .toArray(String[]::new); + return cleanedKeys; } return null; } From 410c5e0e7c39e25930f1db2858250dbfb912326a Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 5 Jan 2021 19:31:54 +0800 Subject: [PATCH 505/523] [fix-34344][kafka] use apache common split --- .../com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index 89be2840b..0a27cb074 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -131,7 +131,7 @@ public CRowTypeInfo getRowTypeInfo() { protected String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo) { String keysStr = kafkaSinkTableInfo.getPartitionKeys(); if (StringUtils.isNotBlank(keysStr)) { - String[] keys = keysStr.split(","); + String[] keys = StringUtils.split(keysStr, ","); String[] cleanedKeys = Arrays.stream(keys) .map(x -> x.trim()) .toArray(String[]::new); From 69e9b1fb5ee4a8039002341394b45f9fd3bc1c08 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 5 Jan 2021 21:01:04 +0800 Subject: [PATCH 506/523] [fix-34193][root][pom] delete plugins dir in maven clean phase --- pom.xml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pom.xml b/pom.xml index d10954d3e..f2e85ea33 100644 --- a/pom.xml +++ b/pom.xml @@ -137,6 +137,25 @@ + + maven-antrun-plugin + 1.8 + + + clean-plugins + + clean + + run + + + + + + + + + From 708d9235e40e978da7234a058f705e2cd22e697a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Wed, 6 Jan 2021 16:08:05 +0800 Subject: [PATCH 507/523] =?UTF-8?q?[opt-34257][kudu]=20=E4=BC=98=E5=8C=96k?= =?UTF-8?q?udu=E6=8F=92=E4=BB=B6=EF=BC=8C=E6=94=B9=E7=94=A8KuduClient=20AP?= =?UTF-8?q?I=E4=BB=A3=E6=9B=BFAsyncKuduClient?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/kudu/KuduOutputFormat.java | 272 +++++++++--------- .../dtstack/flink/sql/sink/kudu/KuduSink.java | 21 +- 2 files changed, 140 insertions(+), 153 deletions(-) diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index ab6ac13a8..e8b508f1f 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -25,10 +25,9 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.kudu.client.AsyncKuduClient; -import org.apache.kudu.client.AsyncKuduSession; import org.apache.kudu.client.KuduClient; import org.apache.kudu.client.KuduException; +import org.apache.kudu.client.KuduSession; import org.apache.kudu.client.KuduTable; import org.apache.kudu.client.Operation; import org.apache.kudu.client.PartialRow; @@ -40,40 +39,29 @@ import java.security.PrivilegedAction; import java.sql.Timestamp; import java.util.Date; +import java.util.Objects; /** - * @author gituser - * @modify xiuzhu + * @author gituser + * @modify xiuzhu */ public class KuduOutputFormat extends AbstractDtRichOutputFormat { private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger(KuduOutputFormat.class); - - public enum WriteMode { - // insert - INSERT, - // update - UPDATE, - // update or insert - UPSERT - } - + protected String[] fieldNames; + TypeInformation[] fieldTypes; + boolean enableKrb; private String kuduMasters; - private String tableName; - private WriteMode writeMode; - - protected String[] fieldNames; - - TypeInformation[] fieldTypes; - - private AsyncKuduClient client; + private KuduClient client; private KuduTable table; + private volatile KuduSession session; + private Integer workerCount; private Integer defaultOperationTimeoutMs; @@ -86,11 +74,14 @@ public enum WriteMode { private String principal; private String keytab; private String krb5conf; - boolean enableKrb; private KuduOutputFormat() { } + public static KuduOutputFormatBuilder buildKuduOutputFormat() { + return new KuduOutputFormatBuilder(); + } + @Override public void configure(Configuration parameters) { @@ -103,16 +94,16 @@ public void open(int taskNumber, int numTasks) throws IOException { } private void establishConnection() throws IOException { - AsyncKuduClient.AsyncKuduClientBuilder asyncKuduClientBuilder = new AsyncKuduClient.AsyncKuduClientBuilder(kuduMasters); + KuduClient.KuduClientBuilder kuduClientBuilder = new KuduClient.KuduClientBuilder(kuduMasters); if (null != workerCount) { - asyncKuduClientBuilder.workerCount(workerCount); + kuduClientBuilder.workerCount(workerCount); } if (null != defaultSocketReadTimeoutMs) { - asyncKuduClientBuilder.workerCount(defaultSocketReadTimeoutMs); + kuduClientBuilder.defaultSocketReadTimeoutMs(defaultSocketReadTimeoutMs); } if (null != defaultOperationTimeoutMs) { - asyncKuduClientBuilder.workerCount(defaultOperationTimeoutMs); + kuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); } if (enableKrb) { @@ -122,20 +113,21 @@ private void establishConnection() throws IOException { krb5conf ); client = ugi.doAs( - new PrivilegedAction() { - @Override - public AsyncKuduClient run() { - return asyncKuduClientBuilder.build(); - } - }); + (PrivilegedAction) kuduClientBuilder::build); } else { - client = asyncKuduClientBuilder.build(); + client = kuduClientBuilder.build(); + } + + if (client.tableExists(tableName)) { + table = client.openTable(tableName); } - LOG.info("connect kudu is successed!"); - KuduClient syncClient = client.syncClient(); - if (syncClient.tableExists(tableName)) { - table = syncClient.openTable(tableName); + if (Objects.isNull(table)) { + throw new IllegalArgumentException( + String.format("Table [%s] Open Failed , please check table exists", tableName)); } + LOG.info("connect kudu is succeed!"); + + session = client.newSession(); } @Override @@ -147,26 +139,22 @@ public void writeRecord(Tuple2 record) throws IOException { } Row row = tupleTrans.getField(1); if (row.getArity() != fieldNames.length) { - if(outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0) { + if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0) { LOG.error("record insert failed ..{}", row.toString()); LOG.error("cause by row.getArity() != fieldNames.length"); } outDirtyRecords.inc(); return; } - Operation operation = toOperation(writeMode, row); - AsyncKuduSession session = client.newSession(); try { if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0) { LOG.info("Receive data : {}", row); } - - session.apply(operation); - session.close(); + session.apply(toOperation(writeMode, row)); outRecords.inc(); } catch (KuduException e) { - if(outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0){ + if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0) { LOG.error("record insert failed, total dirty record:{} current row:{}", outDirtyRecords.getCount(), row.toString()); LOG.error("", e); } @@ -176,107 +164,24 @@ public void writeRecord(Tuple2 record) throws IOException { @Override public void close() { - if (null != client) { + if (Objects.nonNull(session) && !session.isClosed()) { try { - client.close(); + session.close(); } catch (Exception e) { - throw new IllegalArgumentException("[closeKudu]:" + e.getMessage()); + throw new IllegalArgumentException("[closeKuduSession]: " + e.getMessage()); } } - } - - public static KuduOutputFormatBuilder buildKuduOutputFormat() { - return new KuduOutputFormatBuilder(); - } - - public static class KuduOutputFormatBuilder { - private final KuduOutputFormat kuduOutputFormat; - - protected KuduOutputFormatBuilder() { - this.kuduOutputFormat = new KuduOutputFormat(); - } - - public KuduOutputFormatBuilder setKuduMasters(String kuduMasters) { - kuduOutputFormat.kuduMasters = kuduMasters; - return this; - } - - public KuduOutputFormatBuilder setTableName(String tableName) { - kuduOutputFormat.tableName = tableName; - return this; - } - - - public KuduOutputFormatBuilder setFieldNames(String[] fieldNames) { - kuduOutputFormat.fieldNames = fieldNames; - return this; - } - - public KuduOutputFormatBuilder setFieldTypes(TypeInformation[] fieldTypes) { - kuduOutputFormat.fieldTypes = fieldTypes; - return this; - } - - public KuduOutputFormatBuilder setWriteMode(WriteMode writeMode) { - if (null == writeMode) { - kuduOutputFormat.writeMode = WriteMode.UPSERT; - } - kuduOutputFormat.writeMode = writeMode; - return this; - } - - public KuduOutputFormatBuilder setWorkerCount(Integer workerCount) { - kuduOutputFormat.workerCount = workerCount; - return this; - } - - public KuduOutputFormatBuilder setDefaultOperationTimeoutMs(Integer defaultOperationTimeoutMs) { - kuduOutputFormat.defaultOperationTimeoutMs = defaultOperationTimeoutMs; - return this; - } - - public KuduOutputFormatBuilder setDefaultSocketReadTimeoutMs(Integer defaultSocketReadTimeoutMs) { - kuduOutputFormat.defaultSocketReadTimeoutMs = defaultSocketReadTimeoutMs; - return this; - } - - public KuduOutputFormatBuilder setPrincipal(String principal) { - kuduOutputFormat.principal = principal; - return this; - } - - public KuduOutputFormatBuilder setKeytab(String keytab) { - kuduOutputFormat.keytab = keytab; - return this; - } - - public KuduOutputFormatBuilder setKrb5conf(String krb5conf) { - kuduOutputFormat.krb5conf = krb5conf; - return this; - } - - public KuduOutputFormatBuilder setEnableKrb(boolean enableKrb) { - kuduOutputFormat.enableKrb = enableKrb; - return this; - } - public KuduOutputFormat finish() { - if (kuduOutputFormat.kuduMasters == null) { - throw new IllegalArgumentException("No kuduMasters supplied."); - } - - if (kuduOutputFormat.tableName == null) { - throw new IllegalArgumentException("No tablename supplied."); + if (null != client) { + try { + client.shutdown(); + } catch (Exception e) { + throw new IllegalArgumentException("[closeKuduClient]:" + e.getMessage()); } - - return kuduOutputFormat; } } private Operation toOperation(WriteMode writeMode, Row row) { - if (null == table) { - throw new IllegalArgumentException("Table Open Failed , please check table exists"); - } Operation operation = toOperation(writeMode); PartialRow partialRow = operation.getRow(); @@ -355,11 +260,102 @@ private Operation toOperation(WriteMode writeMode) { return table.newInsert(); case UPDATE: return table.newUpdate(); - case UPSERT: - return table.newUpsert(); default: return table.newUpsert(); } } + public enum WriteMode { + // insert + INSERT, + // update + UPDATE, + // update or insert + UPSERT + } + + public static class KuduOutputFormatBuilder { + private final KuduOutputFormat kuduOutputFormat; + + protected KuduOutputFormatBuilder() { + this.kuduOutputFormat = new KuduOutputFormat(); + } + + public KuduOutputFormatBuilder setKuduMasters(String kuduMasters) { + kuduOutputFormat.kuduMasters = kuduMasters; + return this; + } + + public KuduOutputFormatBuilder setTableName(String tableName) { + kuduOutputFormat.tableName = tableName; + return this; + } + + + public KuduOutputFormatBuilder setFieldNames(String[] fieldNames) { + kuduOutputFormat.fieldNames = fieldNames; + return this; + } + + public KuduOutputFormatBuilder setFieldTypes(TypeInformation[] fieldTypes) { + kuduOutputFormat.fieldTypes = fieldTypes; + return this; + } + + public KuduOutputFormatBuilder setWriteMode(WriteMode writeMode) { + if (null == writeMode) { + kuduOutputFormat.writeMode = WriteMode.UPSERT; + } + kuduOutputFormat.writeMode = writeMode; + return this; + } + + public KuduOutputFormatBuilder setWorkerCount(Integer workerCount) { + kuduOutputFormat.workerCount = workerCount; + return this; + } + + public KuduOutputFormatBuilder setDefaultOperationTimeoutMs(Integer defaultOperationTimeoutMs) { + kuduOutputFormat.defaultOperationTimeoutMs = defaultOperationTimeoutMs; + return this; + } + + public KuduOutputFormatBuilder setDefaultSocketReadTimeoutMs(Integer defaultSocketReadTimeoutMs) { + kuduOutputFormat.defaultSocketReadTimeoutMs = defaultSocketReadTimeoutMs; + return this; + } + + public KuduOutputFormatBuilder setPrincipal(String principal) { + kuduOutputFormat.principal = principal; + return this; + } + + public KuduOutputFormatBuilder setKeytab(String keytab) { + kuduOutputFormat.keytab = keytab; + return this; + } + + public KuduOutputFormatBuilder setKrb5conf(String krb5conf) { + kuduOutputFormat.krb5conf = krb5conf; + return this; + } + + public KuduOutputFormatBuilder setEnableKrb(boolean enableKrb) { + kuduOutputFormat.enableKrb = enableKrb; + return this; + } + + public KuduOutputFormat finish() { + if (kuduOutputFormat.kuduMasters == null) { + throw new IllegalArgumentException("No kuduMasters supplied."); + } + + if (kuduOutputFormat.tableName == null) { + throw new IllegalArgumentException("No tablename supplied."); + } + + return kuduOutputFormat; + } + } + } \ No newline at end of file diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java index f8ac125be..cd87e3fbf 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java @@ -20,28 +20,19 @@ public class KuduSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { + protected String[] fieldNames; + TypeInformation[] fieldTypes; + boolean enableKrb; private String kuduMasters; - private String tableName; - private KuduOutputFormat.WriteMode writeMode; - - protected String[] fieldNames; - - TypeInformation[] fieldTypes; - private Integer workerCount; - private Integer defaultOperationTimeoutMs; - private Integer defaultSocketReadTimeoutMs; - private int parallelism = -1; - private String principal; private String keytab; private String krb5conf; - boolean enableKrb; @Override public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { @@ -67,7 +58,7 @@ public void emitDataStream(DataStream> dataStream) { public DataStreamSink> consumeDataStream(DataStream> dataStream) { KuduOutputFormat.KuduOutputFormatBuilder builder = KuduOutputFormat.buildKuduOutputFormat(); - builder.setKuduMasters(this.kuduMasters) + KuduOutputFormat kuduOutputFormat = builder.setKuduMasters(this.kuduMasters) .setTableName(this.tableName) .setWriteMode(writeMode) .setWorkerCount(this.workerCount) @@ -78,8 +69,8 @@ public DataStreamSink> consumeDataStream(DataStream Date: Wed, 6 Jan 2021 18:51:14 +0800 Subject: [PATCH 508/523] [hotfix-33981][elasticsearch6][sink]remove logback dependence and exclude jar from 'flink-connector-elasticsearch6_2.11' --- elasticsearch6/elasticsearch6-sink/pom.xml | 24 ++++++---------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/elasticsearch6/elasticsearch6-sink/pom.xml b/elasticsearch6/elasticsearch6-sink/pom.xml index cb5fb753c..c1e755b2d 100644 --- a/elasticsearch6/elasticsearch6-sink/pom.xml +++ b/elasticsearch6/elasticsearch6-sink/pom.xml @@ -30,28 +30,16 @@ ${elasticsearch.version} - - ch.qos.logback - logback-core - 1.1.7 - - - - ch.qos.logback - logback-classic - 1.1.7 - - - - org.apache.logging.log4j - log4j-to-slf4j - 2.7 - - org.apache.flink flink-connector-elasticsearch6_2.11 ${flink.version} + + + log4j-to-slf4j + org.apache.logging.log4j + + From 081b0c2a1fcb8e817217312e60649667c59663bc Mon Sep 17 00:00:00 2001 From: wuren Date: Wed, 6 Jan 2021 19:45:55 +0800 Subject: [PATCH 509/523] [fix-34400][root][pom] optimize pom for cleaning output dir --- pom.xml | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/pom.xml b/pom.xml index f2e85ea33..1b797690d 100644 --- a/pom.xml +++ b/pom.xml @@ -138,23 +138,19 @@ - maven-antrun-plugin - 1.8 - - - clean-plugins - - clean - - run - - - - - - - - + org.apache.maven.plugins + maven-clean-plugin + 3.1.0 + + + + ${basedir}/plugins + + + ${basedir}/sqlplugins + + + From 4388684ea2c16bf4effd15c214a5ded2a6cc6de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Thu, 7 Jan 2021 17:59:01 +0800 Subject: [PATCH 510/523] =?UTF-8?q?[fix-34473][core]=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E5=85=A8=E5=B1=80=E5=B9=B6=E8=A1=8C=E5=BA=A6?= =?UTF-8?q?=E5=90=8E=EF=BC=8C=E5=85=A8=E9=87=8F=E7=BB=B4=E8=A1=A8=E5=8D=95?= =?UTF-8?q?=E7=8B=AC=E8=AE=BE=E7=BD=AE=E5=B9=B6=E8=A1=8C=E5=BA=A6=E4=B8=8D?= =?UTF-8?q?=E7=94=9F=E6=95=88=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/operator/SideWithAllCacheOperator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java index 6b6f9fe1b..f825180e4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java @@ -60,6 +60,6 @@ private static BaseAllReqRow loadFlatMap(String sideType, String sqlRootDir, Row public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { BaseAllReqRow allReqRow = loadFlatMap(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); - return inputStream.flatMap(allReqRow); + return inputStream.flatMap(allReqRow).setParallelism(sideTableInfo.getParallelism()); } } From b720842135014e93e871326ccfece24afd2d386b Mon Sep 17 00:00:00 2001 From: mowen Date: Fri, 8 Jan 2021 11:27:07 +0800 Subject: [PATCH 511/523] [hotfix-34455][core]verify whether join's columns exists in table. --- .../dtstack/flink/sql/side/SideSqlExec.java | 83 +++++++++++++++++-- 1 file changed, 76 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index bb8faf8ee..6dad92061 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -60,12 +60,7 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.Set; +import java.util.*; import static org.apache.calcite.sql.SqlKind.*; @@ -351,7 +346,7 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, FieldReplaceInfo fieldReplaceInfo = parseAsQuery((SqlBasicCall) pollSqlNode, tableCache); if(fieldReplaceInfo == null){ - return; + return; } //as 的源表 @@ -361,6 +356,77 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, } + /** + * check whether all table fields exist in join condition. + * @param conditionNode + * @param joinScope + */ + public void checkConditionFieldsInTable(SqlNode conditionNode, JoinScope joinScope, AbstractSideTableInfo sideTableInfo) { + List sqlNodeList = Lists.newArrayList(); + ParseUtils.parseAnd(conditionNode, sqlNodeList); + for (SqlNode sqlNode : sqlNodeList) { + if (!SqlKind.COMPARISON.contains(sqlNode.getKind())) { + throw new RuntimeException("It is not comparison operator."); + } + + SqlNode leftNode = ((SqlBasicCall) sqlNode).getOperands()[0]; + SqlNode rightNode = ((SqlBasicCall) sqlNode).getOperands()[1]; + + if (leftNode.getKind() == SqlKind.IDENTIFIER) { + checkFieldInTable((SqlIdentifier) leftNode, joinScope, conditionNode, sideTableInfo); + } + + if (rightNode.getKind() == SqlKind.IDENTIFIER) { + checkFieldInTable((SqlIdentifier) rightNode, joinScope, conditionNode, sideTableInfo); + } + + } + } + + /** + * check whether table exists and whether field is in table. + * @param sqlNode + * @param joinScope + * @param conditionNode + */ + private void checkFieldInTable(SqlIdentifier sqlNode, JoinScope joinScope, SqlNode conditionNode, AbstractSideTableInfo sideTableInfo) { + String tableName = sqlNode.getComponent(0).getSimple(); + String fieldName = sqlNode.getComponent(1).getSimple(); + JoinScope.ScopeChild scopeChild = joinScope.getScope(tableName); + String tableErrorMsg = "Table [%s] is not exist. Error condition is [%s]. If you find [%s] is exist. Please check AS statement."; + Preconditions.checkState( + scopeChild != null, + tableErrorMsg, + tableName, + conditionNode.toString(), + tableName + ); + + String[] fieldNames = scopeChild.getRowTypeInfo().getFieldNames(); + ArrayList allFieldNames = new ArrayList( + Arrays.asList(fieldNames) + ); + // HBase、Redis这种NoSQL Primary Key不在字段列表中，所以要加进去。 + if (sideTableInfo != null) { + List pks = sideTableInfo.getPrimaryKeys(); + if (pks != null) { + pks.stream() + .filter(pk -> !allFieldNames.contains(pk)) + .forEach(pk -> allFieldNames.add(pk)); + } + } + + boolean hasField = allFieldNames.contains(fieldName); + String fieldErrorMsg = "Table [%s] has not [%s] field. Error join condition is [%s]. If you find it is exist. Please check AS statement."; + Preconditions.checkState( + hasField, + fieldErrorMsg, + tableName, + fieldName, + conditionNode.toString() + ); + } + private void joinFun(Object pollObj, Map localTableCache, Map sideTableMap, @@ -395,6 +461,9 @@ private void joinFun(Object pollObj, HashBasedTable mappingTable = ((JoinInfo) pollObj).getTableFieldRef(); + // verify whether join's columns exists in table. + checkConditionFieldsInTable(joinInfo.getCondition(), joinScope, sideTableInfo); + //获取两个表的所有字段 List sideJoinFieldInfo = ParserJoinField.getRowTypeInfo(joinInfo.getSelectNode(), joinScope, true); //通过join的查询字段信息过滤出需要的字段信息 From 8009e35325b6ecb42704e98a50f1fe200e4007a8 Mon Sep 17 00:00:00 2001 From: mowen Date: Mon, 18 Jan 2021 15:24:19 +0800 Subject: [PATCH 512/523] [hotfix-34801][core]fix proplem when replace alias and fieldName --- .../com/dtstack/flink/sql/util/TableUtils.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 30ef8b4b3..9d11014bf 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -528,11 +528,25 @@ private static void replaceConditionNode(SqlNode selectNode, String oldTbName, S String tableName = sqlIdentifier.names.asList().get(0); String tableField = sqlIdentifier.names.asList().get(1); - String fieldKey = tableName + "_" + tableField; + String fieldKey = tableName + "." + tableField; if(tableName.equalsIgnoreCase(oldTbName)){ - String newFieldName = fieldReplaceRef.get(fieldKey) == null ? tableField : fieldReplaceRef.get(fieldKey); + /* + * ****Before replace:***** + * fieldKey: b.department + * fieldReplaceRef : b.department -> a_b_0.department0 + * oldFieldRef: a_b_0.department0 + * oldTbName: b + * oldFieldName: department + * ****After replace:***** + * newTbName: a_b_0 + * newFieldName: department0 + */ + String oldFieldRef = fieldReplaceRef.get(fieldKey); + String newFieldName = (oldFieldRef != null && !StringUtils.substringAfter(oldFieldRef, ".").isEmpty()) ? + StringUtils.substringAfter(oldFieldRef, ".") : tableField; + SqlIdentifier newField = ((SqlIdentifier)selectNode).setName(0, newTbName); newField = newField.setName(1, newFieldName); ((SqlIdentifier)selectNode).assignNamesFrom(newField); From c36e131dc912ecedb4f099f0ba54e0671b81bd7a Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Tue, 19 Jan 2021 14:03:55 +0800 Subject: [PATCH 513/523] =?UTF-8?q?[hotfix-34751][core]=20udf=20=E6=B3=A8?= =?UTF-8?q?=E5=86=8C=E4=BD=BF=E7=94=A8=E5=BD=93=E5=89=8D=E7=BA=BF=E7=A8=8B?= =?UTF-8?q?classloader?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/classloader/ClassLoaderManager.java | 14 +++++++++++++- .../flink/sql/exec/ExecuteProcessHelper.java | 10 +--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java index 2e62e11ab..bd805007e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java @@ -20,10 +20,12 @@ import com.dtstack.flink.sql.util.PluginUtil; import com.dtstack.flink.sql.util.ReflectionUtils; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.FileInputStream; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; @@ -73,7 +75,17 @@ private static DtClassLoader retrieveClassLoad(String pluginJarPath) { private static DtClassLoader retrieveClassLoad(List jarUrls) { jarUrls.sort(Comparator.comparing(URL::toString)); - String jarUrlkey = StringUtils.join(jarUrls, "_"); + + List jarMd5s = new ArrayList<>(jarUrls.size()); + for (URL jarUrl : jarUrls) { + try (FileInputStream inputStream = new FileInputStream(jarUrl.getPath())){ + String jarMd5 = DigestUtils.md5Hex(inputStream); + jarMd5s.add(jarMd5); + } catch (Exception e) { + throw new RuntimeException("Exceptions appears when read file:" + e); + } + } + String jarUrlkey = StringUtils.join(jarMd5s, "_"); return pluginClassLoader.computeIfAbsent(jarUrlkey, k -> { try { URL[] urls = jarUrls.toArray(new URL[jarUrls.size()]); diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index d3a6f78be..97c40f93b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -248,18 +248,10 @@ private static void sqlTranslation(String localSqlPluginPath, public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrlList, TableEnvironment tableEnv, boolean isGetPlan) throws IllegalAccessException, InvocationTargetException { // udf和tableEnv须由同一个类加载器加载 - ClassLoader levelClassLoader = tableEnv.getClass().getClassLoader(); ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); - URLClassLoader classLoader = null; + URLClassLoader classLoader = ClassLoaderManager.loadExtraJar(jarUrlList, (URLClassLoader) currentClassLoader); List funcList = sqlTree.getFunctionList(); for (CreateFuncParser.SqlParserResult funcInfo : funcList) { - if (isGetPlan) { - classLoader = ClassLoaderManager.loadExtraJar(jarUrlList, (URLClassLoader) currentClassLoader); - } - //classloader - if (classLoader == null) { - classLoader = ClassLoaderManager.loadExtraJar(jarUrlList, (URLClassLoader) levelClassLoader); - } FunctionManager.registerUDF(funcInfo.getType(), funcInfo.getClassName(), funcInfo.getName(), tableEnv, classLoader); } } From 9e89588b9a081c053eb24412f04041a3434dad23 Mon Sep 17 00:00:00 2001 From: wuren Date: Thu, 4 Feb 2021 17:17:59 +0800 Subject: [PATCH 514/523] [fix-35290][core] fix watermark metrics throw a warning log --- .../flink/sql/watermarker/AbstractCustomerWaterMarker.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java index d75d26a61..9991c7021 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java @@ -92,7 +92,6 @@ public IterationRuntimeContext getIterationRuntimeContext() { public void setRuntimeContext(RuntimeContext t) { this.runtimeContext = t; eventDelayGauge = new EventDelayGauge(); - t.getMetricGroup().getAllVariables().put("", fromSourceTag); t.getMetricGroup().gauge(MetricConstant.DT_EVENT_DELAY_GAUGE, eventDelayGauge); } From d0b8a1d7afd9d81e407962423bcc13803020cc1f Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 4 Feb 2021 20:51:44 +0800 Subject: [PATCH 515/523] [feat-34267][redis] redis sink and side support ipv6. --- redis5/pom.xml | 13 ++ redis5/redis5-side/pom.xml | 14 --- .../flink/sql/side/redis/RedisAllReqRow.java | 114 ++++++++++-------- .../sql/side/redis/RedisAsyncReqRow.java | 31 +++-- redis5/redis5-side/redis-side-core/pom.xml | 11 +- .../flink/sql/side/redis/enums/RedisType.java | 3 +- .../sql/side/redis/table/RedisSideReqRow.java | 4 +- redis5/redis5-sink/pom.xml | 14 --- .../sql/sink/redis/RedisOutputFormat.java | 64 ++++++---- 9 files changed, 137 insertions(+), 131 deletions(-) diff --git a/redis5/pom.xml b/redis5/pom.xml index 894f786ef..d8e63bef1 100644 --- a/redis5/pom.xml +++ b/redis5/pom.xml @@ -16,5 +16,18 @@ redis5-side + + + com.dtstack.flink + sql.core + 1.0-SNAPSHOT + provided + + + redis.clients + jedis + 2.9.0 + + \ No newline at end of file diff --git a/redis5/redis5-side/pom.xml b/redis5/redis5-side/pom.xml index c6623b9d2..10462574c 100644 --- a/redis5/redis5-side/pom.xml +++ b/redis5/redis5-side/pom.xml @@ -11,20 +11,6 @@ 4.0.0 sql.side.redis redis-side - - - com.dtstack.flink - sql.core - 1.0-SNAPSHOT - provided - - - redis.clients - jedis - 2.8.0 - - - redis-side-core diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 8e6a08791..2e3c0887b 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -26,6 +26,7 @@ import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; import com.esotericsoftware.minlog.Log; +import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -35,7 +36,6 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.HostAndPort; @@ -48,14 +48,18 @@ import java.io.Closeable; import java.io.IOException; import java.sql.SQLException; - +import java.util.Arrays; import java.util.Calendar; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + /** * @author yanxi */ @@ -63,6 +67,8 @@ public class RedisAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 7578879189085344807L; + private static final Pattern HOST_PORT_PATTERN = Pattern.compile("(?(.*)):(?\\d+)*"); + private static final Logger LOG = LoggerFactory.getLogger(RedisAllReqRow.class); private static final int CONN_RETRY_NUM = 3; @@ -73,9 +79,9 @@ public class RedisAllReqRow extends BaseAllReqRow { private RedisSideTableInfo tableInfo; - private AtomicReference>> cacheRef = new AtomicReference<>(); + private final AtomicReference>> cacheRef = new AtomicReference<>(); - private RedisSideReqRow redisSideReqRow; + private final RedisSideReqRow redisSideReqRow; public RedisAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new RedisAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); @@ -115,8 +121,8 @@ public void flatMap(CRow input, Collector out) throws Exception { for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = input.row().getField(conValIndex); - if(equalObj == null){ - if(sideInfo.getJoinType() == JoinType.LEFT){ + if (equalObj == null) { + if (sideInfo.getJoinType() == JoinType.LEFT) { Row data = fillData(input.row(), null); out.collect(new CRow(data, input.change())); } @@ -125,13 +131,13 @@ public void flatMap(CRow input, Collector out) throws Exception { inputParams.put(sideInfo.getEqualFieldList().get(i), equalObj.toString()); } String key = buildCacheKey(inputParams); - if(StringUtils.isBlank(key)){ + if (StringUtils.isBlank(key)) { return; } Map cacheMap = cacheRef.get().get(key); - if(MapUtils.isEmpty(cacheMap)){ - if(sideInfo.getJoinType() != JoinType.LEFT){ + if (MapUtils.isEmpty(cacheMap)) { + if (sideInfo.getJoinType() != JoinType.LEFT) { return; } Row data = fillData(input.row(), null); @@ -146,8 +152,8 @@ public void flatMap(CRow input, Collector out) throws Exception { private String buildCacheKey(Map refData) { StringBuilder keyBuilder = new StringBuilder(tableInfo.getTableName()); List primaryKeys = tableInfo.getPrimaryKeys(); - for(String primaryKey : primaryKeys){ - if(!refData.containsKey(primaryKey)){ + for (String primaryKey : primaryKeys) { + if (!refData.containsKey(primaryKey)) { return null; } keyBuilder.append("_").append(refData.get(primaryKey)); @@ -160,7 +166,7 @@ private void loadData(Map> tmpCache) throws SQLExcep JedisCommands jedis = null; try { StringBuilder keyPattern = new StringBuilder(tableInfo.getTableName()); - for (String key : tableInfo.getPrimaryKeys()) { + for (int i = 0; i < tableInfo.getPrimaryKeys().size(); i++) { keyPattern.append("_").append("*"); } jedis = getJedisWithRetry(CONN_RETRY_NUM); @@ -195,41 +201,54 @@ private JedisCommands getJedis(RedisSideTableInfo tableInfo) { String url = tableInfo.getUrl(); String password = tableInfo.getPassword(); String database = tableInfo.getDatabase() == null ? "0" : tableInfo.getDatabase(); + String masterName = tableInfo.getMasterName(); int timeout = tableInfo.getTimeout(); - if (timeout == 0){ - timeout = 1000; + if (timeout == 0) { + timeout = 10000; } String[] nodes = url.split(","); - String[] firstIpPort = nodes[0].split(":"); - String firstIp = firstIpPort[0]; - String firstPort = firstIpPort[1]; - Set addresses = new HashSet<>(); - Set ipPorts = new HashSet<>(); - for (String ipPort : nodes) { - ipPorts.add(ipPort); - String[] ipPortPair = ipPort.split(":"); - addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.valueOf(ipPortPair[1].trim()))); - } - if (timeout == 0){ - timeout = 1000; - } JedisCommands jedis = null; GenericObjectPoolConfig poolConfig = setPoolConfig(tableInfo.getMaxTotal(), tableInfo.getMaxIdle(), tableInfo.getMinIdle()); - switch (RedisType.parse(tableInfo.getRedisType())){ - //单机 + switch (RedisType.parse(tableInfo.getRedisType())) { case STANDALONE: - pool = new JedisPool(poolConfig, firstIp, Integer.parseInt(firstPort), timeout, password, Integer.parseInt(database)); - jedis = pool.getResource(); + String firstIp = null; + String firstPort = null; + Matcher standalone = HOST_PORT_PATTERN.matcher(nodes[0]); + if (standalone.find()) { + firstIp = standalone.group("host").trim(); + firstPort = standalone.group("port").trim(); + } + if (Objects.nonNull(firstIp)) { + pool = new JedisPool(poolConfig, firstIp, Integer.parseInt(firstPort), timeout, password, Integer.parseInt(database)); + jedis = pool.getResource(); + } else { + throw new IllegalArgumentException( + String.format("redis url error. current url [%s]", nodes[0])); + } break; - //哨兵 case SENTINEL: - jedisSentinelPool = new JedisSentinelPool(tableInfo.getMasterName(), ipPorts, poolConfig, timeout, password, Integer.parseInt(database)); + Set ipPorts = new HashSet<>(Arrays.asList(nodes)); + jedisSentinelPool = new JedisSentinelPool(masterName, ipPorts, poolConfig, timeout, password, Integer.parseInt(database)); jedis = jedisSentinelPool.getResource(); break; - //集群 case CLUSTER: - jedis = new JedisCluster(addresses, timeout, timeout,1, poolConfig); + Set addresses = new HashSet<>(); + // 对ipv6 支持 + for (String node : nodes) { + Matcher matcher = HOST_PORT_PATTERN.matcher(node); + if (matcher.find()) { + String host = matcher.group("host").trim(); + String portStr = matcher.group("port").trim(); + if (org.apache.commons.lang3.StringUtils.isNotBlank(host) && org.apache.commons.lang3.StringUtils.isNotBlank(portStr)) { + // 转化为int格式的端口 + int port = Integer.parseInt(portStr); + addresses.add(new HostAndPort(host, port)); + } + } + } + jedis = new JedisCluster(addresses, timeout, timeout, 10, password, poolConfig); + break; default: break; } @@ -257,35 +276,32 @@ private JedisCommands getJedisWithRetry(int retryNum) { return null; } - private Set getRedisKeys(RedisType redisType, JedisCommands jedis, String keyPattern){ - if(!redisType.equals(RedisType.CLUSTER)){ + private Set getRedisKeys(RedisType redisType, JedisCommands jedis, String keyPattern) { + if (!redisType.equals(RedisType.CLUSTER)) { return ((Jedis) jedis).keys(keyPattern); } Set keys = new TreeSet<>(); - Map clusterNodes = ((JedisCluster)jedis).getClusterNodes(); - for(String k : clusterNodes.keySet()){ + Map clusterNodes = ((JedisCluster) jedis).getClusterNodes(); + for (String k : clusterNodes.keySet()) { JedisPool jp = clusterNodes.get(k); - Jedis connection = jp.getResource(); - try { + try (Jedis connection = jp.getResource()) { keys.addAll(connection.keys(keyPattern)); - } catch (Exception e){ - LOG.error("Getting keys error: {}", e); - } finally { - connection.close(); + } catch (Exception e) { + LOG.error("Getting keys error.", e); } } return keys; } - private GenericObjectPoolConfig setPoolConfig(String maxTotal, String maxIdle, String minIdle){ + private GenericObjectPoolConfig setPoolConfig(String maxTotal, String maxIdle, String minIdle) { GenericObjectPoolConfig config = new GenericObjectPoolConfig(); - if (maxTotal != null){ + if (maxTotal != null) { config.setMaxTotal(Integer.parseInt(maxTotal)); } - if (maxIdle != null){ + if (maxIdle != null) { config.setMaxIdle(Integer.parseInt(maxIdle)); } - if (minIdle != null){ + if (minIdle != null) { config.setMinIdle(Integer.parseInt(minIdle)); } return config; diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 954469049..534b3d291 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -73,7 +73,7 @@ public class RedisAsyncReqRow extends BaseAsyncReqRow { private RedisSideTableInfo redisSideTableInfo; - private RedisSideReqRow redisSideReqRow; + private final RedisSideReqRow redisSideReqRow; public RedisAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new RedisAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); @@ -98,7 +98,7 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ case STANDALONE: RedisURI redisURI = RedisURI.create("redis://" + url); redisURI.setPassword(password); - redisURI.setDatabase(Integer.valueOf(database)); + redisURI.setDatabase(Integer.parseInt(database)); redisClient = RedisClient.create(redisURI); connection = redisClient.connect(); async = connection.async(); @@ -106,7 +106,7 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ case SENTINEL: RedisURI redisSentinelURI = RedisURI.create("redis-sentinel://" + url); redisSentinelURI.setPassword(password); - redisSentinelURI.setDatabase(Integer.valueOf(database)); + redisSentinelURI.setDatabase(Integer.parseInt(database)); redisSentinelURI.setSentinelMasterId(redisSideTableInfo.getMasterName()); redisClient = RedisClient.create(redisSentinelURI); connection = redisClient.connect(); @@ -135,21 +135,18 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul return; } RedisFuture> future = ((RedisHashAsyncCommands) async).hgetall(key); - future.thenAccept(new Consumer>() { - @Override - public void accept(Map values) { - if (MapUtils.isNotEmpty(values)) { - try { - Row row = fillData(input.row(), values); - dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.SingleLine, values)); - resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); - } catch (Exception e) { - dealFillDataError(input, resultFuture, e); - } - } else { - dealMissKey(input, resultFuture); - dealCacheData(key,CacheMissVal.getMissKeyObj()); + future.thenAccept(values -> { + if (MapUtils.isNotEmpty(values)) { + try { + Row row = fillData(input.row(), values); + dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.SingleLine, values)); + resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); + } catch (Exception e) { + dealFillDataError(input, resultFuture, e); } + } else { + dealMissKey(input, resultFuture); + dealCacheData(key,CacheMissVal.getMissKeyObj()); } }); } diff --git a/redis5/redis5-side/redis-side-core/pom.xml b/redis5/redis5-side/redis-side-core/pom.xml index 7fb45a291..88e0a00dd 100644 --- a/redis5/redis5-side/redis-side-core/pom.xml +++ b/redis5/redis5-side/redis-side-core/pom.xml @@ -12,15 +12,6 @@ sql.side.redis.core - - - com.dtstack.flink - sql.core - 1.0-SNAPSHOT - provided - - - jar - + jar \ No newline at end of file diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/enums/RedisType.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/enums/RedisType.java index 048c32b08..614447a5d 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/enums/RedisType.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/enums/RedisType.java @@ -28,6 +28,7 @@ public static RedisType parse(int redisType){ return type; } } - throw new RuntimeException("unsupport redis type["+ redisType + "]"); + throw new IllegalArgumentException( + "unsupported redis type["+ redisType + "]"); } } diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java index 634cc66fa..af1a1f2e4 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java @@ -43,7 +43,7 @@ public class RedisSideReqRow implements ISideReqRow, Serializable { private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); - private BaseSideInfo sideInfo; + private final BaseSideInfo sideInfo; public RedisSideReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; @@ -102,7 +102,7 @@ public void setRowField(Row row, Integer index, BaseSideInfo sideInfo, String va row.setField(index, Float.valueOf(value)); break; case "java.math.BigDecimal": - row.setField(index, BigDecimal.valueOf(Long.valueOf(value))); + row.setField(index, BigDecimal.valueOf(Long.parseLong(value))); break; case "java.sql.Timestamp": row.setField(index, Timestamp.valueOf(value)); diff --git a/redis5/redis5-sink/pom.xml b/redis5/redis5-sink/pom.xml index be6a445ed..4fe35fb09 100644 --- a/redis5/redis5-sink/pom.xml +++ b/redis5/redis5-sink/pom.xml @@ -16,20 +16,6 @@ redis-sink http://maven.apache.org - - - com.dtstack.flink - sql.core - 1.0-SNAPSHOT - provided - - - redis.clients - jedis - 2.9.0 - - - diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index ab97cf60e..d86f90685 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -38,13 +38,17 @@ import java.io.Closeable; import java.io.IOException; import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * @author yanxi */ -public class RedisOutputFormat extends AbstractDtRichOutputFormat { +public class RedisOutputFormat extends AbstractDtRichOutputFormat> { private static final Logger LOG = LoggerFactory.getLogger(RedisOutputFormat.class); + private static final Pattern HOST_PORT_PATTERN = Pattern.compile("(?(.*)):(?\\d+)*"); + private String url; private String database = "0"; @@ -77,8 +81,6 @@ public class RedisOutputFormat extends AbstractDtRichOutputFormat { private JedisSentinelPool jedisSentinelPool; - private GenericObjectPoolConfig poolConfig; - private RedisOutputFormat(){ } @Override @@ -107,44 +109,60 @@ private GenericObjectPoolConfig setPoolConfig(String maxTotal, String maxIdle, S } private void establishConnection() { - poolConfig = setPoolConfig(maxTotal, maxIdle, minIdle); + GenericObjectPoolConfig poolConfig = setPoolConfig(maxTotal, maxIdle, minIdle); String[] nodes = StringUtils.split(url, ","); - String[] firstIpPort = StringUtils.split(nodes[0], ":"); - String firstIp = firstIpPort[0]; - String firstPort = firstIpPort[1]; - Set addresses = new HashSet<>(); - Set ipPorts = new HashSet<>(); - for (String ipPort : nodes) { - ipPorts.add(ipPort); - String[] ipPortPair = StringUtils.split(ipPort, ":"); - addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.valueOf(ipPortPair[1].trim()))); - } switch (RedisType.parse(redisType)){ case STANDALONE: - pool = new JedisPool(poolConfig, firstIp, Integer.parseInt(firstPort), timeout, password, Integer.parseInt(database)); - jedis = pool.getResource(); + String firstIp = null; + String firstPort = null; + Matcher standalone = HOST_PORT_PATTERN.matcher(nodes[0]); + if (standalone.find()) { + firstIp = standalone.group("host").trim(); + firstPort = standalone.group("port").trim(); + } + if (Objects.nonNull(firstIp)) { + pool = new JedisPool(poolConfig, firstIp, Integer.parseInt(firstPort), timeout, password, Integer.parseInt(database)); + jedis = pool.getResource(); + } else { + throw new IllegalArgumentException( + String.format("redis url error. current url [%s]", nodes[0])); + } break; case SENTINEL: + Set ipPorts = new HashSet<>(Arrays.asList(nodes)); jedisSentinelPool = new JedisSentinelPool(masterName, ipPorts, poolConfig, timeout, password, Integer.parseInt(database)); jedis = jedisSentinelPool.getResource(); break; case CLUSTER: + Set addresses = new HashSet<>(); + // 对ipv6 支持 + for (String node : nodes) { + Matcher matcher = HOST_PORT_PATTERN.matcher(node); + if (matcher.find()) { + String host = matcher.group("host").trim(); + String portStr = matcher.group("port").trim(); + if (StringUtils.isNotBlank(host) && StringUtils.isNotBlank(portStr)) { + // 转化为int格式的端口 + int port = Integer.parseInt(portStr); + addresses.add(new HostAndPort(host, port)); + } + } + } jedis = new JedisCluster(addresses, timeout, timeout, 10, password, poolConfig); break; default: - throw new RuntimeException("unsupport redis type[ " + redisType + "]"); + throw new IllegalArgumentException("unsupported redis type[ " + redisType + "]"); } } @Override - public void writeRecord(Tuple2 record) throws IOException { - Tuple2 tupleTrans = record; - Boolean retract = tupleTrans.getField(0); + public void writeRecord(Tuple2 record) throws IOException { + Boolean retract = record.getField(0); if (!retract) { return; } - Row row = tupleTrans.getField(1); + Row row = record.getField(1); if (row.getArity() != fieldNames.length) { return; } @@ -153,9 +171,7 @@ public void writeRecord(Tuple2 record) throws IOException { refData.put(fieldNames[i], row.getField(i)); } String redisKey = buildCacheKey(refData); - refData.entrySet().forEach(e ->{ - jedis.hset(redisKey, e.getKey(), String.valueOf(e.getValue())); - }); + refData.forEach((key, value) -> jedis.hset(redisKey, key, String.valueOf(value))); if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0){ LOG.info(record.toString()); From 28fc9264413ee876ea4c6747ddf2f2bff9b9a611 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 4 Feb 2021 21:23:10 +0800 Subject: [PATCH 516/523] [feat-34267][redis] fix multi-primary-key with whitespace throw null when buildCache key. multi-primary-key like 'PRIMARY KEY(id, name, ....)' --- .../flink/sql/table/AbstractTableParser.java | 26 +++++++++++-------- .../sql/sink/redis/table/RedisSinkParser.java | 2 +- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index f0898cfd6..3df278d5e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -26,10 +26,12 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Reason: @@ -44,16 +46,16 @@ public abstract class AbstractTableParser { private static final String NEST_JSON_FIELD_KEY = "nestFieldKey"; private static final String CHAR_TYPE_NO_LENGTH = "CHAR"; - private static Pattern primaryKeyPattern = Pattern.compile("(?i)PRIMARY\\s+KEY\\s*\\((.*)\\)"); - private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+?)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); - private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); - private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); - private static Pattern typePattern = Pattern.compile("(\\S+)\\s+(\\w+.*)"); + private static final Pattern primaryKeyPattern = Pattern.compile("(?i)(^\\s*)PRIMARY\\s+KEY\\s*\\((.*)\\)"); + private static final Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+?)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); + private static final Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); + private static final Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); + private static final Pattern typePattern = Pattern.compile("(\\S+)\\s+(\\w+.*)"); - private Map patternMap = Maps.newHashMap(); + private final Map patternMap = Maps.newHashMap(); - private Map handlerMap = Maps.newHashMap(); + private final Map handlerMap = Maps.newHashMap(); public AbstractTableParser() { addParserHandler(PRIMARY_KEY, primaryKeyPattern, this::dealPrimaryKey); @@ -112,7 +114,7 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { if (matcher.find()) { fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); - fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); + fieldExtraInfo.setLength(Integer.parseInt(matcher.group(1))); } else { fieldClass = dbTypeConvertToJavaType(fieldType); } @@ -140,9 +142,11 @@ private Tuple2 extractType(String fieldRow, String tableName) { } public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo) { - String primaryFields = matcher.group(1).trim(); - String[] splitArray = primaryFields.split(","); - List primaryKeys = Lists.newArrayList(splitArray); + String primaryFields = matcher.group(2).trim(); + List primaryKeys = Arrays + .stream(primaryFields.split(",")) + .map(String::trim) + .collect(Collectors.toList()); tableInfo.setPrimaryKeys(primaryKeys); } diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 8961f7da9..44d127884 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -51,7 +51,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map primaryKeysList = Lists.newArrayList(); + List primaryKeysList; primaryKeysList = Arrays.asList(StringUtils.split(primaryKeysStr, ",")); redisTableInfo.setPrimaryKeys(primaryKeysList); } From 3b598af38c5b3ecb9d021c597d554b8ca4e4e093 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Sun, 7 Feb 2021 19:57:13 +0800 Subject: [PATCH 517/523] [feat-35355][redis] fix redis async side plugin connect error on sentinel mode. --- .../sql/side/redis/RedisAsyncReqRow.java | 68 ++++++++++++++----- .../sql/side/redis/table/RedisSideReqRow.java | 3 + 2 files changed, 53 insertions(+), 18 deletions(-) diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 954469049..3dd39e759 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -18,20 +18,9 @@ package com.dtstack.flink.sql.side.redis; +import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.BaseAsyncReqRow; -import io.lettuce.core.KeyValue; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.async.RedisAsyncCommands; -import io.lettuce.core.api.async.RedisStringAsyncCommands; -import io.lettuce.core.cluster.api.async.RedisAdvancedClusterAsyncCommands; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; - -import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; @@ -41,6 +30,7 @@ import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; import io.lettuce.core.RedisClient; import io.lettuce.core.RedisFuture; +import io.lettuce.core.RedisURI; import io.lettuce.core.api.StatefulRedisConnection; import io.lettuce.core.api.async.RedisHashAsyncCommands; import io.lettuce.core.api.async.RedisKeyAsyncCommands; @@ -48,12 +38,19 @@ import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; -import com.google.common.collect.Maps; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Consumer; +import java.util.regex.Matcher; + /** * @author yanxi */ @@ -104,11 +101,34 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ async = connection.async(); break; case SENTINEL: - RedisURI redisSentinelURI = RedisURI.create("redis-sentinel://" + url); - redisSentinelURI.setPassword(password); - redisSentinelURI.setDatabase(Integer.valueOf(database)); - redisSentinelURI.setSentinelMasterId(redisSideTableInfo.getMasterName()); - redisClient = RedisClient.create(redisSentinelURI); + String[] urlSplit = StringUtils.split(url, ","); + RedisURI.Builder builder = null; + for (String item : urlSplit) { + Matcher mather = RedisSideReqRow.HOST_PORT_PATTERN.matcher(item); + if (mather.find()) { + builder = buildSentinelUri( + mather.group("host"), + mather.group("port"), + builder + ); + } else { + throw new IllegalArgumentException( + String.format("Illegal format with redis url [%s]", item) + ); + } + } + + if (Objects.nonNull(builder)) { + builder + .withPassword(tableInfo.getPassword()) + .withDatabase(Integer.parseInt(tableInfo.getDatabase())) + .withSentinelMasterId(tableInfo.getMasterName()); + } else { + throw new NullPointerException("build redis uri error!"); + } + + RedisURI uri = builder.build(); + redisClient = RedisClient.create(uri); connection = redisClient.connect(); async = connection.async(); break; @@ -123,6 +143,18 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ } } + private RedisURI.Builder buildSentinelUri( + String host, + String port, + RedisURI.Builder builder) { + if (Objects.nonNull(builder)) { + builder.withSentinel(host, Integer.parseInt(port)); + } else { + builder = RedisURI.Builder.sentinel(host, Integer.parseInt(port)); + } + return builder; + } + @Override public Row fillData(Row input, Object sideInput) { return redisSideReqRow.fillData(input, sideInput); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java index 634cc66fa..a51be6543 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java @@ -29,6 +29,7 @@ import java.sql.Timestamp; import java.util.Map; import java.util.TimeZone; +import java.util.regex.Pattern; /** * redis fill row data @@ -41,6 +42,8 @@ public class RedisSideReqRow implements ISideReqRow, Serializable { private static final long serialVersionUID = 3751171828444748982L; + public static final Pattern HOST_PORT_PATTERN = Pattern.compile("(?(.*)):(?\\d+)*"); + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); private BaseSideInfo sideInfo; From c06b190bf6a261c009d7f9ca13cbc41bd73938e2 Mon Sep 17 00:00:00 2001 From: mowen Date: Fri, 5 Mar 2021 11:18:57 +0800 Subject: [PATCH 518/523] [hotfix-35517][elasticsearch5&6][sink]log framework has been replaced from logback to log4j. --- elasticsearch5/elasticsearch5-sink/pom.xml | 19 +++++++------------ elasticsearch6/elasticsearch6-sink/pom.xml | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/elasticsearch5/elasticsearch5-sink/pom.xml b/elasticsearch5/elasticsearch5-sink/pom.xml index 515410c6e..0ad18338a 100644 --- a/elasticsearch5/elasticsearch5-sink/pom.xml +++ b/elasticsearch5/elasticsearch5-sink/pom.xml @@ -19,24 +19,19 @@ 5.3.3 - - ch.qos.logback - logback-core - 1.1.7 - - - - ch.qos.logback - logback-classic - 1.1.7 - - org.elasticsearch.client x-pack-transport 5.3.3 + + + org.slf4j + slf4j-log4j12 + 1.6.1 + + org.apache.logging.log4j log4j-to-slf4j diff --git a/elasticsearch6/elasticsearch6-sink/pom.xml b/elasticsearch6/elasticsearch6-sink/pom.xml index c1e755b2d..8caa7eeb8 100644 --- a/elasticsearch6/elasticsearch6-sink/pom.xml +++ b/elasticsearch6/elasticsearch6-sink/pom.xml @@ -30,6 +30,25 @@ ${elasticsearch.version} + + log4j + log4j + 1.2.17 + + + + slf4j-api + org.slf4j + 1.7.15 + + + + + org.slf4j + slf4j-log4j12 + 1.6.1 + + org.apache.flink flink-connector-elasticsearch6_2.11 From 79692cc7f1e0b1cd283374073b9d2f4a424b72a9 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 4 Mar 2021 19:56:12 +0800 Subject: [PATCH 519/523] =?UTF-8?q?[hotfix-35512][impala]=201.=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E6=89=B9=E9=87=8F=E5=86=99=E5=85=A5=E6=97=B6=E5=87=BA?= =?UTF-8?q?=E7=8E=B0=E5=BC=82=E5=B8=B8=EF=BC=8C=E5=B0=86=E6=89=B9=E9=87=8F?= =?UTF-8?q?sql=E8=BD=AC=E4=B8=BA=E5=8D=95=E6=9D=A1sql=E6=89=A7=E8=A1=8C=20?= =?UTF-8?q?2.fix=20inserting=20the=20value=20like=20'[a,b]'=20or=20'{a,=20?= =?UTF-8?q?b}'=20or=20'=20'=20error=20with=20impala=20sink.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/exception/ExceptionTrace.java | 39 +++ .../dtstack/flink/sql/util/DtStringUtil.java | 22 +- .../sql/sink/impala/ImpalaOutputFormat.java | 263 +++++++++--------- 3 files changed, 183 insertions(+), 141 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/exception/ExceptionTrace.java diff --git a/core/src/main/java/com/dtstack/flink/sql/exception/ExceptionTrace.java b/core/src/main/java/com/dtstack/flink/sql/exception/ExceptionTrace.java new file mode 100644 index 000000000..07db131d9 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/exception/ExceptionTrace.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.exception; + +import java.util.Objects; + +/** + * @author tiezhu + * @date 2021/3/8 星期一 + * Company dtstack + */ +public class ExceptionTrace { + // 追溯当前异常的最原始异常信息 + public static String traceOriginalCause(Throwable e) { + String errorMsg; + if (Objects.nonNull(e.getCause())) { + errorMsg = traceOriginalCause(e.getCause()); + } else { + errorMsg = e.getMessage(); + } + return errorMsg; + } +} diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index 5af657512..dbf6d9e7b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -21,15 +21,17 @@ package com.dtstack.flink.sql.util; import com.dtstack.flink.sql.enums.ColumnType; -import org.apache.commons.lang3.StringUtils; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; import com.google.common.collect.Maps; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.sql.Timestamp; +import org.apache.commons.lang3.StringUtils; + import java.math.BigDecimal; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -204,7 +206,6 @@ public static String dealSqlComment(String sql) { return b.toString(); } - public static String col2string(Object column, String type) { String rowData = column.toString(); ColumnType columnType = ColumnType.valueOf(type.toUpperCase()); @@ -374,8 +375,7 @@ public static String getTableFullPath(String schema, String tableName) { return addQuoteForStr(tableName); } - String schemaAndTabName = addQuoteForStr(schema) + "." + addQuoteForStr(tableName); - return schemaAndTabName; + return addQuoteForStr(schema) + "." + addQuoteForStr(tableName); } /** @@ -413,4 +413,14 @@ public static String removeStartAndEndQuota(String str) { String removeStart = StringUtils.removeStart(str, "'"); return StringUtils.removeEnd(removeStart, "'"); } + + /** + * 判断当前对象是null 还是空 + * + * @param obj 需要判断的对象 + * @return 返回true 如果对象是空或者为null + */ + public static boolean isEmptyOrNull(Object obj) { + return Objects.isNull(obj) || obj.toString().isEmpty(); + } } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index 35b0c9a50..82122e00f 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -18,10 +18,12 @@ package com.dtstack.flink.sql.sink.impala; +import com.dtstack.flink.sql.exception.ExceptionTrace; import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.sink.rdb.JDBCTypeConvertUtils; import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.JDBCUtils; import com.dtstack.flink.sql.util.KrbUtils; import com.google.common.collect.Maps; @@ -41,7 +43,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -72,8 +73,6 @@ public class ImpalaOutputFormat extends AbstractDtRichOutputFormat cast('value' as string) cast(value as timestamp) -> cast('value' as timestamp) - private static final Pattern TYPE_PATTERN = Pattern.compile("cast\\((.*) as (.*)\\)"); //specific type which values need to be quoted private static final String[] NEED_QUOTE_TYPE = {"string", "timestamp", "varchar"}; @@ -178,27 +177,20 @@ private void init() throws SQLException { if (!storeType.equalsIgnoreCase(KUDU_TYPE)) { throw new IllegalArgumentException("update mode not support for non-kudu table!"); } - updateStatement = connection.prepareStatement(buildUpdateSql(schema, tableName, fieldNames, primaryKeys)); - return; + } else { + valueFieldNames = rebuildFieldNameListAndTypeList(fieldNames, staticPartitionFields, fieldTypes, partitionFields); } - - valueFieldNames = rebuildFieldNameListAndTypeList(fieldNames, staticPartitionFields, fieldTypes, partitionFields); } private void initScheduledTask(Long batchWaitInterval) { try { if (batchWaitInterval != 0) { this.scheduler = new ScheduledThreadPoolExecutor(1, - new DTThreadFactory("impala-upsert-output-format")); + new DTThreadFactory("impala-upsert-output-format")); this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(() -> { synchronized (ImpalaOutputFormat.this) { - try { - flush(); - } catch (Exception e) { - LOG.error("Writing records to impala jdbc failed.", e); - throw new RuntimeException("Writing records to impala jdbc failed.", e); - } + flush(); } }, batchWaitInterval, batchWaitInterval, TimeUnit.MILLISECONDS); } @@ -237,16 +229,16 @@ private void openJdbc() { } } - private void flush() throws Exception { - if (batchCount > 0) { - if (updateMode.equalsIgnoreCase(UPDATE_MODE)) { - executeUpdateBatch(); - } - if (!rowDataMap.isEmpty()) { - String templateSql = + private synchronized void flush() { + try { + if (batchCount > 0) { + if (updateMode.equalsIgnoreCase(UPDATE_MODE)) { + executeUpdateBatch(); + } + if (!rowDataMap.isEmpty()) { + String templateSql = "INSERT INTO tableName ${tableFieldsCondition} PARTITION ${partitionCondition} VALUES ${valuesCondition}"; - executeBatchSql( - statement, + executeBatchSql( templateSql, schema, tableName, @@ -255,12 +247,15 @@ private void flush() throws Exception { valueFieldNames, partitionFields, rowDataMap - ); - rowDataMap.clear(); + ); + rowDataMap.clear(); + } } + batchCount = 0; + } catch (Exception e) { + LOG.error("Writing records to impala jdbc failed.", e); + throw new RuntimeException("Writing records to impala jdbc failed.", e); } - batchCount = 0; - } /** @@ -273,10 +268,10 @@ private void executeUpdateBatch() throws SQLException { rows.forEach(row -> { try { JDBCTypeConvertUtils.setRecordToStatement( - updateStatement, - JDBCTypeConvertUtils.getSqlTypeFromFieldType(fieldTypes), - row, - primaryKeys.stream().mapToInt(fieldNames::indexOf).toArray() + updateStatement, + JDBCTypeConvertUtils.getSqlTypeFromFieldType(fieldTypes), + row, + primaryKeys.stream().mapToInt(fieldNames::indexOf).toArray() ); updateStatement.addBatch(); } catch (Exception e) { @@ -288,8 +283,8 @@ private void executeUpdateBatch() throws SQLException { rows.clear(); } catch (Exception e) { LOG.debug("impala jdbc execute batch error ", e); - connection.rollback(); - connection.commit(); + JDBCUtils.rollBack(connection); + JDBCUtils.commit(connection); updateStatement.clearBatch(); executeUpdate(connection); } @@ -300,14 +295,10 @@ public void executeUpdate(Connection connection) { try { setRecordToStatement(updateStatement, JDBCTypeConvertUtils.getSqlTypeFromFieldType(fieldTypes), row); updateStatement.executeUpdate(); - connection.commit(); + JDBCUtils.commit(connection); } catch (Exception e) { - try { - connection.rollback(); - connection.commit(); - } catch (SQLException e1) { - throw new RuntimeException(e1); - } + JDBCUtils.rollBack(connection); + JDBCUtils.commit(connection); if (metricOutputFormat.outDirtyRecords.getCount() % DIRTY_DATA_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed ,this row is {}", row.toString()); LOG.error("", e); @@ -331,7 +322,7 @@ private void putRowIntoMap(Map> rowDataMap, Tuple2 rebuildFieldNameListAndTypeList(List fieldNames, List staticPartitionFields, List fieldTypes, String partitionFields) { - if (partitionFields.isEmpty()) { + if (partitionFields == null || partitionFields.isEmpty()) { return fieldNames; } @@ -354,40 +345,6 @@ private List rebuildFieldNameListAndTypeList(List fieldNames, Li return valueFields; } - /** - * Quote a specific type of value, like string, timestamp - * before: 1, cast(tiezhu as string), cast(2001-01-09 01:05:01 as timestamp), cast(123 as int) - * after: 1, cast('tiezhu' as string), cast('2001-01-09 01:05:01' as timestamp), cast(123 as int) - * if cast value is null, then cast(null as type) - * - * @param valueCondition original value condition - * @return quoted condition - */ - private String valueConditionAddQuotation(String valueCondition) { - String[] temps = valueCondition.split(","); - List replacedItem = new ArrayList<>(); - Arrays.stream(temps).forEach( - item -> { - Matcher matcher = TYPE_PATTERN.matcher(item); - while (matcher.find()) { - String value = matcher.group(1); - String type = matcher.group(2); - - for (String needQuoteType : NEED_QUOTE_TYPE) { - if (type.contains(needQuoteType)) { - if (!"null".equals(value)) { - item = item.replace(value, "'" + value + "'"); - } - } - } - } - replacedItem.add(item); - } - ); - - return "(" + String.join(", ", replacedItem) + ")"; - } - @Override public void writeRecord(Tuple2 record) throws IOException { try { @@ -422,7 +379,7 @@ public void writeRecord(Tuple2 record) throws IOException { for (int i = 0; i < fieldTypes.size(); i++) { rowValue.setField(i, valueMap.get(valueFieldNames.get(i))); } - rowTuple2.f1 = valueConditionAddQuotation(buildValuesCondition(fieldTypes, rowValue)); + rowTuple2.f1 = buildValuesCondition(fieldTypes, rowValue); putRowIntoMap(rowDataMap, rowTuple2); } @@ -446,11 +403,7 @@ public void close() throws IOException { } // 将还未执行的SQL flush if (batchCount > 0) { - try { - flush(); - } catch (Exception e) { - throw new RuntimeException("Writing records to impala failed.", e); - } + flush(); } // cancel scheduled task if (this.scheduledFuture != null) { @@ -484,7 +437,6 @@ public void close() throws IOException { * execute batch sql from row data map * sql like 'insert into tableName(f1, f2, f3) ${partitionCondition} values(v1, v2, v3), (v4, v5, v6).... * - * @param statement execute statement * @param tempSql template sql * @param storeType the store type of data * @param enablePartition enable partition or not @@ -492,8 +444,7 @@ public void close() throws IOException { * @param partitionFields partition fields * @param rowDataMap row data map */ - private void executeBatchSql(Statement statement, - String tempSql, + private void executeBatchSql(String tempSql, String schema, String tableName, String storeType, @@ -501,46 +452,83 @@ private void executeBatchSql(Statement statement, List fieldNames, String partitionFields, Map> rowDataMap) { - StringBuilder valuesCondition = new StringBuilder(); StringBuilder partitionCondition = new StringBuilder(); String tableFieldsCondition = buildTableFieldsCondition(fieldNames, partitionFields); - ArrayList rowData; + ArrayList rowData = new ArrayList<>(); String tableNameInfo = Objects.isNull(schema) ? - tableName : quoteIdentifier(schema) + "." + tableName; + tableName : quoteIdentifier(schema) + "." + tableName; tempSql = tempSql.replace("tableName", tableNameInfo); + boolean isPartitioned = storeType.equalsIgnoreCase(KUDU_TYPE) || !enablePartition; - // kudu ${partitionCondition} is null - if (storeType.equalsIgnoreCase(KUDU_TYPE) || !enablePartition) { - try { - rowData = rowDataMap.get(NO_PARTITION); - rowData.forEach(row -> valuesCondition.append(row).append(", ")); - String executeSql = tempSql.replace(VALUES_CONDITION, valuesCondition.toString()) + try { + // kudu ${partitionCondition} is null + if (isPartitioned) { + tempSql = tempSql + .replace(PARTITION_CONDITION, partitionCondition.toString()) + .replace(PARTITION_CONSTANT, "") + .replace(TABLE_FIELDS_CONDITION, tableFieldsCondition); + rowData.addAll(rowDataMap.get(NO_PARTITION)); + String executeSql = tempSql.replace(VALUES_CONDITION, String.join(", ", rowData)); + statement.execute(executeSql); + rowData.clear(); + } else { + // partition sql + Set keySet = rowDataMap.keySet(); + for (String key : keySet) { + rowData.addAll(rowDataMap.get(key)); + partitionCondition.append(key); + tempSql = tempSql .replace(PARTITION_CONDITION, partitionCondition.toString()) - .replace(PARTITION_CONSTANT, "") .replace(TABLE_FIELDS_CONDITION, tableFieldsCondition); - String substring = executeSql.substring(0, executeSql.length() - 2); - statement.execute(substring); - } catch (Exception e) { - throw new RuntimeException("execute impala SQL error!", e); + String executeSql = tempSql + .replace(VALUES_CONDITION, String.join(", ", rowData)); + statement.execute(executeSql); + partitionCondition.delete(0, partitionCondition.length()); + } } - return; + } catch (Exception e) { + if (e instanceof SQLException) { + dealBatchSqlError(rowData, connection, statement, tempSql); + } else { + throw new RuntimeException("Insert into impala error!", e); + } + } finally { + rowData.clear(); } + } - // partition sql - Set keySet = rowDataMap.keySet(); - String finalTempSql = tempSql; - for (String key : keySet) { + /** + * 当批量写入失败时，把批量的sql拆解为单条sql提交，对于单条写入的sql记做脏数据 + * + * @param rowData 批量的values + * @param connection 当前数据库connect + * @param statement 当前statement + * @param templateSql 模版sql，例如insert into tableName(f1, f2, f3) [partition] values $valueCondition + */ + private void dealBatchSqlError(List rowData, + Connection connection, + Statement statement, + String templateSql) { + String errorMsg = "Insert into impala error. \nCause: [%s]\nRow: [%s]"; + JDBCUtils.rollBack(connection); + JDBCUtils.commit(connection); + for (String rowDatum : rowData) { + String executeSql = templateSql.replace(VALUES_CONDITION, rowDatum); try { - String executeSql = String.copyValueOf(finalTempSql.toCharArray()); - ArrayList valuesConditionList = rowDataMap.get(key); - partitionCondition.append(key); - executeSql = executeSql.replace(PARTITION_CONDITION, partitionCondition.toString()) - .replace(TABLE_FIELDS_CONDITION, tableFieldsCondition) - .replace(VALUES_CONDITION, String.join(", ", valuesConditionList)); statement.execute(executeSql); - partitionCondition.delete(0, partitionCondition.length()); - } catch (SQLException sqlException) { - throw new RuntimeException("execute impala SQL error! ", sqlException); + JDBCUtils.commit(connection); + } catch (SQLException e) { + JDBCUtils.rollBack(connection); + JDBCUtils.commit(connection); + if (metricOutputFormat.outDirtyRecords.getCount() % DIRTY_DATA_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { + LOG.error( + String.format( + errorMsg, + ExceptionTrace.traceOriginalCause(e), + rowDatum) + ); + } + metricOutputFormat.outDirtyRecords.inc(); } } } @@ -573,9 +561,9 @@ private String buildPartitionCondition(Map rowData, String parti */ private String buildTableFieldsCondition(List fieldNames, String partitionFields) { return "(" + fieldNames.stream() - .filter(f -> !partitionFields.contains(f)) - .map(this::quoteIdentifier) - .collect(Collectors.joining(", ")) + ")"; + .filter(f -> !partitionFields.contains(f)) + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")) + ")"; } /** @@ -583,22 +571,27 @@ private String buildTableFieldsCondition(List fieldNames, String partiti * replace ${valuesCondition} * * @param fieldTypes field types - * @return condition like '(?, ?, cast(? as string))' and '?' will be replaced with row data + * @return condition like '(?, ?, cast('?' as string))' and '?' will be replaced with row data */ private String buildValuesCondition(List fieldTypes, Row row) { String valuesCondition = fieldTypes.stream().map( - f -> { - for (String item : NEED_QUOTE_TYPE) { - if (f.toLowerCase().contains(item)) { - return String.format("cast(? as %s)", f.toLowerCase()); - } + f -> { + for (String item : NEED_QUOTE_TYPE) { + if (f.toLowerCase().contains(item)) { + return String.format("cast('?' as %s)", f.toLowerCase()); } - return "?"; - }).collect(Collectors.joining(", ")); + } + return "?"; + }).collect(Collectors.joining(", ")); for (int i = 0; i < row.getArity(); i++) { - valuesCondition = valuesCondition.replaceFirst("\\?", Objects.isNull(row.getField(i)) ? "null" : row.getField(i).toString()); + Object rowField = row.getField(i); + if (DtStringUtil.isEmptyOrNull(rowField)) { + valuesCondition = valuesCondition.replaceFirst("'\\?'", "null"); + } else { + valuesCondition = valuesCondition.replaceFirst("\\?", rowField.toString()); + } } - return valuesCondition; + return "(" + valuesCondition + ")"; } /** @@ -609,16 +602,16 @@ private String buildValuesCondition(List fieldTypes, Row row) { private String buildUpdateSql(String schema, String tableName, List fieldNames, List primaryKeys) { //跳过primary key字段 String setClause = fieldNames.stream() - .filter(f -> !CollectionUtils.isNotEmpty(primaryKeys) || !primaryKeys.contains(f)) - .map(f -> quoteIdentifier(f) + "=?") - .collect(Collectors.joining(", ")); + .filter(f -> !CollectionUtils.isNotEmpty(primaryKeys) || !primaryKeys.contains(f)) + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(", ")); String conditionClause = primaryKeys.stream() - .map(f -> quoteIdentifier(f) + "=?") - .collect(Collectors.joining(" AND ")); + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(" AND ")); return "UPDATE " + (Objects.isNull(schema) ? "" : quoteIdentifier(schema) + ".") - + quoteIdentifier(tableName) + " SET " + setClause + " WHERE " + conditionClause; + + quoteIdentifier(tableName) + " SET " + setClause + " WHERE " + conditionClause; } private String quoteIdentifier(String identifier) { @@ -664,7 +657,7 @@ public Builder setTableName(String tableName) { public Builder setPartitionFields(String partitionFields) { format.partitionFields = Objects.isNull(partitionFields) ? - "" : partitionFields; + "" : partitionFields; return this; } @@ -739,11 +732,11 @@ public ImpalaOutputFormat build() { if (format.authMech == EAuthMech.Kerberos.getType()) { checkNotNull(format.krb5confPath, - "When kerberos authentication is enabled, krb5confPath is required！"); + "When kerberos authentication is enabled, krb5confPath is required！"); checkNotNull(format.principal, - "When kerberos authentication is enabled, principal is required！"); + "When kerberos authentication is enabled, principal is required！"); checkNotNull(format.keytabPath, - "When kerberos authentication is enabled, keytabPath is required！"); + "When kerberos authentication is enabled, keytabPath is required！"); } if (format.authMech == EAuthMech.UserName.getType()) { From 758c4db4b62fe2c5630058496989a99fdbfd9a51 Mon Sep 17 00:00:00 2001 From: mowen Date: Wed, 10 Mar 2021 14:52:25 +0800 Subject: [PATCH 520/523] [hotfix-34789][elasticsearch5&6][sink]support timestamp datatype. --- .../java/com/dtstack/flink/sql/enums/ColumnType.java | 4 ++++ .../java/com/dtstack/flink/sql/util/DtStringUtil.java | 3 +++ .../sql/sink/elasticsearch/CustomerSinkFunc.java | 11 +++++++++++ .../sql/sink/elasticsearch/CustomerSinkFunc.java | 6 ++++++ 4 files changed, 24 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java index 7f3f0019c..e93c92f51 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java @@ -102,6 +102,10 @@ public enum ColumnType { * timestamp */ TIMESTAMP, + /** + * time eg: 23:59:59 + */ + TIME, /** * decimal */ diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index dbf6d9e7b..bfa0c7ca5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -243,6 +243,9 @@ public static String col2string(Object column, String type) { case DATE: result = DateUtil.dateToString((java.util.Date)column); break; + case TIME: + result = DateUtil.getTimeFromStr(String.valueOf(column)); + break; case TIMESTAMP: result = DateUtil.timestampToString((java.util.Date)column); break; diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index 23213e721..9c68ffea7 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.sink.elasticsearch; +import com.dtstack.flink.sql.util.DateUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.api.java.tuple.Tuple2; @@ -32,6 +33,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.sql.Date; +import java.sql.Timestamp; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -107,6 +110,14 @@ private IndexRequest createIndexRequest(Row element) { Map dataMap = EsUtil.rowToJsonMap(element,fieldNames,fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); for(int i=0; i Date: Mon, 22 Mar 2021 10:00:53 +0800 Subject: [PATCH 521/523] =?UTF-8?q?[hotfix-35753][impala]=20fix=20String.r?= =?UTF-8?q?eplaceFirst=E4=B8=AD=E7=9A=84=E5=9D=91=EF=BC=8CreplaceFirst("\\?= =?UTF-8?q?=3F",=20value)=EF=BC=8C=E5=A6=82=E6=9E=9Cvalue=E4=B8=AD$?= =?UTF-8?q?=E5=90=8E=E4=B8=8D=E6=98=AF=E6=95=B0=E5=AD=97=EF=BC=8C=E9=82=A3?= =?UTF-8?q?=E4=B9=88=E4=BC=9A=E5=87=BA=E7=8E=B0"illegal=20group=20referenc?= =?UTF-8?q?e=EF=BC=9Agroup=20index=20is=20missing"=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index 82122e00f..8723254bf 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -588,7 +588,7 @@ private String buildValuesCondition(List fieldTypes, Row row) { if (DtStringUtil.isEmptyOrNull(rowField)) { valuesCondition = valuesCondition.replaceFirst("'\\?'", "null"); } else { - valuesCondition = valuesCondition.replaceFirst("\\?", rowField.toString()); + valuesCondition = valuesCondition.replaceFirst("\\?", Matcher.quoteReplacement(rowField.toString())); } } return "(" + valuesCondition + ")"; From 1297b4fa75c206c32ded2c8fa99df8d5ca03b7bb Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Tue, 13 Apr 2021 17:13:08 +0800 Subject: [PATCH 522/523] =?UTF-8?q?[hotfix-36198][redis]=20=E8=A7=A3?= =?UTF-8?q?=E5=86=B3redis=E5=BC=82=E6=AD=A5=E7=BB=B4=E8=A1=A8cluster?= =?UTF-8?q?=E6=A8=A1=E5=BC=8F=E8=BF=9E=E6=8E=A5=E5=BC=82=E5=B8=B8=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hbase/hbase-side/hbase-all-side/pom.xml | 4 +++ hbase/pom.xml | 4 +++ .../sql/side/redis/RedisAsyncReqRow.java | 27 ++++++++++++++++--- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/hbase/hbase-side/hbase-all-side/pom.xml b/hbase/hbase-side/hbase-all-side/pom.xml index cdfc24b74..dcc5a1987 100644 --- a/hbase/hbase-side/hbase-all-side/pom.xml +++ b/hbase/hbase-side/hbase-all-side/pom.xml @@ -33,6 +33,10 @@ io.netty netty + + netty-all + io.netty + diff --git a/hbase/pom.xml b/hbase/pom.xml index 835457e29..9c82560b3 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -47,6 +47,10 @@ io.netty netty + + netty-all + io.netty + diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 395aa50a5..5fc8f62b9 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -36,6 +36,7 @@ import io.lettuce.core.api.async.RedisKeyAsyncCommands; import io.lettuce.core.cluster.RedisClusterClient; import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; +import io.lettuce.core.internal.HostAndPort; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -44,11 +45,11 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.Consumer; import java.util.regex.Matcher; /** @@ -133,9 +134,8 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ async = connection.async(); break; case CLUSTER: - RedisURI clusterURI = RedisURI.create("redis://" + url); - clusterURI.setPassword(password); - clusterClient = RedisClusterClient.create(clusterURI); + List clusterURIs = buildClusterURIs(url); + clusterClient = RedisClusterClient.create(clusterURIs); clusterConnection = clusterClient.connect(); async = clusterConnection.async(); default: @@ -143,6 +143,25 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ } } + private List buildClusterURIs(String url) { + String password = redisSideTableInfo.getPassword(); + String database = redisSideTableInfo.getDatabase(); + String[] addresses = StringUtils.split(url, ","); + List redisURIs = new ArrayList<>(addresses.length); + for (String addr : addresses){ + HostAndPort hostAndPort = HostAndPort.parse(addr); + RedisURI redisURI = RedisURI.create(hostAndPort.hostText, hostAndPort.port); + if (StringUtils.isNotEmpty(password)) { + redisURI.setPassword(password); + } + if (StringUtils.isNotEmpty(database)) { + redisURI.setDatabase(Integer.valueOf(database)); + } + redisURIs.add(redisURI); + } + return redisURIs; + } + private RedisURI.Builder buildSentinelUri( String host, String port, From 34e200b58fb8a89bedb2f3228f978fc123bdbf47 Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Thu, 15 Apr 2021 18:51:19 +0800 Subject: [PATCH 523/523] [hotfix-36198] redis async side shade netty --- hbase/hbase-side/hbase-all-side/pom.xml | 4 ---- hbase/pom.xml | 4 ---- redis5/redis5-side/redis-async-side/pom.xml | 8 +++++++- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/hbase/hbase-side/hbase-all-side/pom.xml b/hbase/hbase-side/hbase-all-side/pom.xml index dcc5a1987..cdfc24b74 100644 --- a/hbase/hbase-side/hbase-all-side/pom.xml +++ b/hbase/hbase-side/hbase-all-side/pom.xml @@ -33,10 +33,6 @@ io.netty netty - - netty-all - io.netty - diff --git a/hbase/pom.xml b/hbase/pom.xml index 9c82560b3..835457e29 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -47,10 +47,6 @@ io.netty netty - - netty-all - io.netty - diff --git a/redis5/redis5-side/redis-async-side/pom.xml b/redis5/redis5-side/redis-async-side/pom.xml index 39a6280cc..666bace2b 100644 --- a/redis5/redis5-side/redis-async-side/pom.xml +++ b/redis5/redis5-side/redis-async-side/pom.xml @@ -35,7 +35,7 @@ org.apache.maven.plugins maven-shade-plugin - 1.4 + 3.0.0 package @@ -43,6 +43,12 @@ shade + + + io.netty + com.dtstack.flink.sql.side.redis.io.netty + +