From 91ae18c33ceaf0399d2ab98c87bc9091ef163015 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 22 Jun 2020 15:54:36 +0800 Subject: [PATCH 01/80] =?UTF-8?q?local=20=E5=A2=9E=E5=8A=A0=20disableChain?= =?UTF-8?q?Operator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/environment/StreamEnvConfigManager.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index d7b772c6e..e83355f32 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -91,6 +91,8 @@ public static void streamExecutionEnvironmentConfig(StreamExecutionEnvironment s ((Configuration) exeConfig.getGlobalJobParameters()).addAll(globalJobParameters); } + disableChainOperator(streamEnv, globalJobParameters); + getEnvParallelism(confProperties).ifPresent(streamEnv::setParallelism); getMaxEnvParallelism(confProperties).ifPresent(streamEnv::setMaxParallelism); getBufferTimeoutMillis(confProperties).ifPresent(streamEnv::setBufferTimeout); @@ -350,5 +352,10 @@ private static Long getTtlTime(Integer timeNumber, String timeUnit) { } } - + private static StreamExecutionEnvironment disableChainOperator(StreamExecutionEnvironment env, Configuration configuration) { + if(configuration.getBoolean("disableChainOperator", false)) { + env.disableOperatorChaining(); + } + return env; + } } From c11c73f7065477ef33923edb4f436dbb482f9ff3 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 27 Aug 2020 20:03:10 +0800 Subject: [PATCH 02/80] =?UTF-8?q?[feat]=20=E8=84=8F=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=8F=92=E4=BB=B6=E5=8C=96=EF=BC=9A=E7=BB=93=E6=9E=84=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E8=AE=A2=E9=98=85=E8=80=85=E6=A8=A1=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/util/PluginUtil.java | 3 +- dirtyDataManager/manager/pom.xml | 34 ++++++ .../classLoader/DirtyManagerClassLoader.java | 110 ++++++++++++++++++ .../consumer/AbstractDirtyDataConsumer.java | 63 ++++++++++ .../sql/dirty/entity/DirtyDataEntity.java | 87 ++++++++++++++ .../sql/dirty/manager/DirtyDataManager.java | 85 ++++++++++++++ .../flink/sql/dirty/util/ManagerUtil.java | 43 +++++++ dirtyDataManager/mysql/pom.xml | 34 ++++++ .../dirty/mysql/MysqlDirtyDataConsumer.java | 90 ++++++++++++++ dirtyDataManager/pom.xml | 23 ++++ dirtyDataManager/print/pom.xml | 23 ++++ .../dirty/print/PrintDirtyDataConsumer.java | 49 ++++++++ pom.xml | 1 + 13 files changed, 643 insertions(+), 2 deletions(-) create mode 100644 dirtyDataManager/manager/pom.xml create mode 100644 dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/classLoader/DirtyManagerClassLoader.java create mode 100644 dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/consumer/AbstractDirtyDataConsumer.java create mode 100644 dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/entity/DirtyDataEntity.java create mode 100644 dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/manager/DirtyDataManager.java create mode 100644 dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/util/ManagerUtil.java create mode 100644 dirtyDataManager/mysql/pom.xml create mode 100644 dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java create mode 100644 dirtyDataManager/pom.xml create mode 100644 dirtyDataManager/print/pom.xml create mode 100644 dirtyDataManager/print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java diff --git a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java index 0f6755f16..3e21dc476 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java @@ -20,13 +20,12 @@ package com.dtstack.flink.sql.util; -import com.dtstack.flink.sql.classloader.DtClassLoader; import com.dtstack.flink.sql.enums.EPluginLoadMode; -import org.apache.commons.lang3.StringUtils; import com.fasterxml.jackson.core.JsonGenerationException; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dirtyDataManager/manager/pom.xml b/dirtyDataManager/manager/pom.xml new file mode 100644 index 000000000..f21f2e20c --- /dev/null +++ b/dirtyDataManager/manager/pom.xml @@ -0,0 +1,34 @@ + + + + sql.dirtyDataManager + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + manager + + + + com.fasterxml.jackson.core + jackson-databind + 2.10.3 + + + + log4j + log4j + 1.2.17 + + + + org.slf4j + slf4j-simple + 1.7.30 + + + + \ No newline at end of file diff --git a/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/classLoader/DirtyManagerClassLoader.java b/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/classLoader/DirtyManagerClassLoader.java new file mode 100644 index 000000000..c2e2cbcd5 --- /dev/null +++ b/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/classLoader/DirtyManagerClassLoader.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.dirty.classLoader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URL; +import java.net.URLClassLoader; + +/** + * @author tiezhu + * Company dtstack + * Date 2020/8/27 星期四 + */ +public class DirtyManagerClassLoader extends URLClassLoader { + private final Logger LOG = LoggerFactory.getLogger(DirtyManagerClassLoader.class); + + /** + * The parent class loader. + */ + protected ClassLoader parent; + + public DirtyManagerClassLoader(URL[] urls, ClassLoader parent) { + super(urls, parent); + this.parent = parent; + } + + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + return this.loadClass(name, false); + } + + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + synchronized (getClassLoadingLock(name)) { + if (LOG.isDebugEnabled()) { + LOG.debug("loadClass(" + name + ", " + resolve + ")"); + } + Class clazz; + + // (0.1) Check our previously loaded class cache + clazz = findLoadedClass(name); + if (clazz != null) { + if (LOG.isDebugEnabled()) { + LOG.debug(" Returning class from cache"); + } + if (resolve) { + resolveClass(clazz); + } + return (clazz); + } + + // (2) Search local repositories + if (LOG.isDebugEnabled()) { + LOG.debug(" Searching local repositories"); + } + try { + clazz = findClass(name); + if (clazz != null) { + if (LOG.isDebugEnabled()) { + LOG.debug(" Loading class from local repository"); + } + if (resolve) { + resolveClass(clazz); + } + return (clazz); + } + } catch (ClassNotFoundException e) { + // Ignore + } + + if (LOG.isDebugEnabled()) { + LOG.debug(" Delegating to parent classloader at end: " + parent); + } + + try { + clazz = Class.forName(name, false, parent); + if (LOG.isDebugEnabled()) { + LOG.debug(" Loading class from parent"); + } + if (resolve) { + resolveClass(clazz); + } + return (clazz); + } catch (ClassNotFoundException e) { + // Ignore + } + } + + throw new ClassNotFoundException(name); + } +} diff --git a/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/consumer/AbstractDirtyDataConsumer.java b/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/consumer/AbstractDirtyDataConsumer.java new file mode 100644 index 000000000..e040055f4 --- /dev/null +++ b/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/consumer/AbstractDirtyDataConsumer.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.dirty.consumer; + +import com.dtstack.flink.sql.dirty.entity.DirtyDataEntity; + +import java.io.Serializable; +import java.util.concurrent.LinkedBlockingQueue; + +/** + * @author tiezhu + * Company dtstack + * Date 2020/8/27 星期四 + */ +public abstract class AbstractDirtyDataConsumer extends Thread implements Serializable { + private static final long serialVersionUID = -6058598201315176687L; + + private long count = 0; + + protected LinkedBlockingQueue queue; + + /** + * 消费队列数据 + */ + public abstract void consume() throws InterruptedException; + + /** + * 关闭消费者，需要释放资源 + */ + public abstract void close(); + + /** + * 初始化消费者，初始化定时任务 + */ + public void init() { + + } + + @Override + public void run() { + + } + + public void setQueue(LinkedBlockingQueue queue) { + this.queue = queue; + } +} diff --git a/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/entity/DirtyDataEntity.java b/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/entity/DirtyDataEntity.java new file mode 100644 index 000000000..15f495df2 --- /dev/null +++ b/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/entity/DirtyDataEntity.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.dirty.entity; + +import java.sql.Date; + +/** + * @author tiezhu + * Company dtstack + * Date 2020/8/27 星期四 + */ +public class DirtyDataEntity { + /** + * 脏数据信息内容 + */ + private String dirtyData; + + /** + * 脏数据处理时间 + */ + private Date processDate; + + /** + * 产生脏数据的原因 + */ + private String cause; + + /** + * 产生异常的字段 + */ + private String field; + + public String getDirtyData() { + return dirtyData; + } + + public void setDirtyData(String dirtyData) { + this.dirtyData = dirtyData; + } + + public Date getProcessDate() { + return processDate; + } + + public void setProcessDate(Date processDate) { + this.processDate = processDate; + } + + public String getCause() { + return cause; + } + + public void setCause(String cause) { + this.cause = cause; + } + + public String getField() { + return field; + } + + public void setField(String field) { + this.field = field; + } + + public DirtyDataEntity(String dirtyData, Long processDate, String cause, String field) { + this.dirtyData = dirtyData; + this.processDate = new Date(processDate); + this.cause = cause; + this.field = field; + } +} diff --git a/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/manager/DirtyDataManager.java b/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/manager/DirtyDataManager.java new file mode 100644 index 000000000..d0e53cfa9 --- /dev/null +++ b/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/manager/DirtyDataManager.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.dirty.manager; + +import com.dtstack.flink.sql.dirty.consumer.AbstractDirtyDataConsumer; +import com.dtstack.flink.sql.dirty.entity.DirtyDataEntity; + +import java.io.Serializable; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author tiezhu + * Company dtstack + * Date 2020/8/27 星期四 + */ +public class DirtyDataManager implements Serializable { + private static final long serialVersionUID = 7190970299538893497L; + + /** + * 写入队列阻塞时间 + */ + private long blockingInterval = 60; + + /** + * 缓存脏数据信息队列 + */ + private LinkedBlockingQueue queue = new LinkedBlockingQueue<>(); + + /** + * 统计manager收集到的脏数据条数 + */ + private AtomicLong count = new AtomicLong(0); + + private AbstractDirtyDataConsumer consumer; + + public static DirtyDataManager newInstance() { + + return new DirtyDataManager(); + } + + public AbstractDirtyDataConsumer createConsumer(Map properties) throws Exception { + // 利用类加载的方式动态加载 + String type = properties.getOrDefault("type", "print"); + return null; + } + + public void close() throws InterruptedException { + if (!queue.isEmpty()) { + flush(); + } + } + + public void flush() throws InterruptedException { + consumer.setQueue(queue); + consumer.consume(); + } + + // 收集脏数据 + public void collectDirtyData(String dataInfo, String cause, String field) throws InterruptedException { + DirtyDataEntity dirtyDataEntity = new DirtyDataEntity(dataInfo, System.currentTimeMillis(), cause, field); + queue.offer(dirtyDataEntity, blockingInterval, TimeUnit.MILLISECONDS); + count.incrementAndGet(); + consumer.setQueue(queue); + } + +} diff --git a/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/util/ManagerUtil.java b/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/util/ManagerUtil.java new file mode 100644 index 000000000..bfa343e2e --- /dev/null +++ b/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/util/ManagerUtil.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.dirty.util; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.util.Map; + +/** + * @author tiezhu + * Company dtstack + * Date 2020/8/27 星期四 + */ +public class ManagerUtil { + private static final ObjectMapper objectMapper = new ObjectMapper(); + + @SuppressWarnings("unchecked") + public static Map convertJsonToMap(String jsonStr) throws IOException { + return (Map) objectMapper.readValue(jsonStr, Map.class); + } + + public static Map convertEntityToMap(Object obj) throws IOException { + String jsonStr = objectMapper.writeValueAsString(obj); + return convertJsonToMap(jsonStr); + } +} diff --git a/dirtyDataManager/mysql/pom.xml b/dirtyDataManager/mysql/pom.xml new file mode 100644 index 000000000..ce25c68c6 --- /dev/null +++ b/dirtyDataManager/mysql/pom.xml @@ -0,0 +1,34 @@ + + + + sql.dirtyDataManager + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + + 5.1.46 + + + mysql + + + com.dtstack.flink + manager + 1.0-SNAPSHOT + compile + + + + mysql + mysql-connector-java + ${mysql.connector.version} + + + + + + \ No newline at end of file diff --git a/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java b/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java new file mode 100644 index 000000000..49f892bc0 --- /dev/null +++ b/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.dirty.mysql; + +import com.dtstack.flink.sql.dirty.consumer.AbstractDirtyDataConsumer; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +/** + * @author tiezhu + * Company dtstack + * Date 2020/8/27 星期四 + */ +public class MysqlDirtyDataConsumer extends AbstractDirtyDataConsumer { + private static final long serialVersionUID = -2959753658786001679L; + + private static final String DRIVER_NAME = "com.mysql.jdbc.Driver"; + + private boolean isCreatedTable = false; + + private String[] tableField = {"id", "dirtyData", "processTime", "cause", "field"}; + + private String SQL = "INSERT INTO ? (?, ?, ?, ?) VALUES (?, ?, ?, ?) "; + + private PreparedStatement statement; + + private Connection connection; + + private PreparedStatement getStatement(String url, + String userName, + String password) throws ClassNotFoundException, SQLException { + Class.forName(DRIVER_NAME); + + connection = DriverManager.getConnection(url, userName, password); + statement = connection.prepareStatement(SQL); + return statement; + } + + private String quoteIdentifier(String tableName) { + return "\"" + tableName + "\""; + } + + /** + * 创建存储脏数据的表 + * @param tableName 表名 + * @return 是否创建成功 + * @throws SQLException SQL异常 + */ + private boolean createTable(String tableName) throws SQLException { + String sql = + "CREATE TABLE ` " + tableName + "` (" + + " `id` int(11) not null AUTO_INCREMENT,\n" + + " `dirtyData` varchar(100) DEFAULT NULL,\n" + + " `processTime` varchar(100) DEFAULT NULL,\n" + + " `cause` date DEFAULT NULL,\n" + + " `field` varchar(100) DEFAULT NULL,\n" + + " PRIMARY KEY (id)\n" + + ") DEFAULT CHARSET=utf8;"; + return statement.execute(sql); + } + + @Override + public void consume() { + + } + + @Override + public void close() { + + } +} diff --git a/dirtyDataManager/pom.xml b/dirtyDataManager/pom.xml new file mode 100644 index 000000000..f2ea9e9bd --- /dev/null +++ b/dirtyDataManager/pom.xml @@ -0,0 +1,23 @@ + + + + flink.sql + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.dirtyDataManager + pom + + manager + consumer + print + mysql + core + + + + \ No newline at end of file diff --git a/dirtyDataManager/print/pom.xml b/dirtyDataManager/print/pom.xml new file mode 100644 index 000000000..f57503037 --- /dev/null +++ b/dirtyDataManager/print/pom.xml @@ -0,0 +1,23 @@ + + + + sql.dirtyDataManager + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + print + + + com.dtstack.flink + manager + 1.0-SNAPSHOT + compile + + + + + \ No newline at end of file diff --git a/dirtyDataManager/print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java b/dirtyDataManager/print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java new file mode 100644 index 000000000..4cb818034 --- /dev/null +++ b/dirtyDataManager/print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.dirty.print; + +import com.dtstack.flink.sql.dirty.consumer.AbstractDirtyDataConsumer; +import com.dtstack.flink.sql.dirty.entity.DirtyDataEntity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author tiezhu + * Company dtstack + * Date 2020/8/27 星期四 + */ +public class PrintDirtyDataConsumer extends AbstractDirtyDataConsumer { + private static final long serialVersionUID = 5727194679865135189L; + + private final Logger LOG = LoggerFactory.getLogger(PrintDirtyDataConsumer.class); + + @Override + public void consume() throws InterruptedException { + DirtyDataEntity dataEntity = queue.take(); + LOG.warn("get dirtyData: " + dataEntity.getDirtyData() + "\n" + + "cause: " + dataEntity.getCause() + "\n" + + "processTime: " + dataEntity.getProcessDate() + "\n" + + "error field: " + dataEntity.getField()); + } + + @Override + public void close() { + // Do nothing + } +} diff --git a/pom.xml b/pom.xml index 45a90b6e3..a00e3835d 100644 --- a/pom.xml +++ b/pom.xml @@ -36,6 +36,7 @@ polardb oceanbase tidb + dirtyDataManager From f825e9b600fa80072241cc938e293991d8a56467 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 27 Aug 2020 21:08:48 +0800 Subject: [PATCH 03/80] [feat] temp save --- dirtyDataManager/mysql/pom.xml | 2 +- dirtyDataManager/pom.xml | 2 -- dirtyDataManager/print/pom.xml | 2 +- pom.xml | 42 +++++++++++++++++----------------- 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/dirtyDataManager/mysql/pom.xml b/dirtyDataManager/mysql/pom.xml index ce25c68c6..764cb65a6 100644 --- a/dirtyDataManager/mysql/pom.xml +++ b/dirtyDataManager/mysql/pom.xml @@ -13,7 +13,7 @@ 5.1.46 - mysql + dirtyManager.mysql com.dtstack.flink diff --git a/dirtyDataManager/pom.xml b/dirtyDataManager/pom.xml index f2ea9e9bd..360450174 100644 --- a/dirtyDataManager/pom.xml +++ b/dirtyDataManager/pom.xml @@ -13,10 +13,8 @@ pom manager - consumer print mysql - core diff --git a/dirtyDataManager/print/pom.xml b/dirtyDataManager/print/pom.xml index f57503037..71056028a 100644 --- a/dirtyDataManager/print/pom.xml +++ b/dirtyDataManager/print/pom.xml @@ -9,7 +9,7 @@ 4.0.0 - print + dirtyManager.print com.dtstack.flink diff --git a/pom.xml b/pom.xml index a00e3835d..18068f786 100644 --- a/pom.xml +++ b/pom.xml @@ -11,31 +11,31 @@ core kafka-base - kafka09 - kafka10 + + kafka11 - kafka + mysql - hbase - elasticsearch5 - elasticsearch6 - mongo - redis5 + + + + + launcher rdb - sqlserver - oracle - cassandra - kudu - postgresql - serversocket - console - clickhouse - impala - db2 - polardb - oceanbase - tidb + + + + + + + + + + + + + dirtyDataManager From 684b9b1a4d58a0fed3395e79bbc5cde9beadf83d Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 28 Aug 2020 11:22:01 +0800 Subject: [PATCH 04/80] =?UTF-8?q?[feat]=20=E9=80=9A=E8=BF=87URLClassLoader?= =?UTF-8?q?=E5=8A=A8=E6=80=81=E5=8A=A0=E8=BD=BD=20consumer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../consumer/AbstractDirtyDataConsumer.java | 4 +- .../dirtyManager}/entity/DirtyDataEntity.java | 2 +- .../manager/DirtyDataManager.java | 33 +++++- .../dirtyManager/TestDirtyDataManager.java | 27 +++++ dirtyDataManager/manager/pom.xml | 34 ------ .../classLoader/DirtyManagerClassLoader.java | 110 ------------------ .../flink/sql/dirty/util/ManagerUtil.java | 43 ------- dirtyDataManager/mysql/pom.xml | 71 +++++++++-- .../dirty/mysql/MysqlDirtyDataConsumer.java | 2 +- dirtyDataManager/pom.xml | 23 +++- dirtyDataManager/print/pom.xml | 74 ++++++++++-- .../dirty/print/PrintDirtyDataConsumer.java | 4 +- 12 files changed, 209 insertions(+), 218 deletions(-) rename {dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty => core/src/main/java/com/dtstack/flink/sql/dirtyManager}/consumer/AbstractDirtyDataConsumer.java (93%) rename {dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty => core/src/main/java/com/dtstack/flink/sql/dirtyManager}/entity/DirtyDataEntity.java (97%) rename {dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty => core/src/main/java/com/dtstack/flink/sql/dirtyManager}/manager/DirtyDataManager.java (63%) create mode 100644 core/src/test/java/com/dtstack/flink/sql/dirtyManager/TestDirtyDataManager.java delete mode 100644 dirtyDataManager/manager/pom.xml delete mode 100644 dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/classLoader/DirtyManagerClassLoader.java delete mode 100644 dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/util/ManagerUtil.java diff --git a/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/consumer/AbstractDirtyDataConsumer.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java similarity index 93% rename from dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/consumer/AbstractDirtyDataConsumer.java rename to core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java index e040055f4..00cd793a1 100644 --- a/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/consumer/AbstractDirtyDataConsumer.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package com.dtstack.flink.sql.dirty.consumer; +package com.dtstack.flink.sql.dirtyManager.consumer; -import com.dtstack.flink.sql.dirty.entity.DirtyDataEntity; +import com.dtstack.flink.sql.dirtyManager.entity.DirtyDataEntity; import java.io.Serializable; import java.util.concurrent.LinkedBlockingQueue; diff --git a/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/entity/DirtyDataEntity.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java similarity index 97% rename from dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/entity/DirtyDataEntity.java rename to core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java index 15f495df2..3ea41d8a1 100644 --- a/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/entity/DirtyDataEntity.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.dtstack.flink.sql.dirty.entity; +package com.dtstack.flink.sql.dirtyManager.entity; import java.sql.Date; diff --git a/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/manager/DirtyDataManager.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java similarity index 63% rename from dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/manager/DirtyDataManager.java rename to core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java index d0e53cfa9..f16e3daef 100644 --- a/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/manager/DirtyDataManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java @@ -16,13 +16,19 @@ * limitations under the License. */ -package com.dtstack.flink.sql.dirty.manager; +package com.dtstack.flink.sql.dirtyManager.manager; -import com.dtstack.flink.sql.dirty.consumer.AbstractDirtyDataConsumer; -import com.dtstack.flink.sql.dirty.entity.DirtyDataEntity; +import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.dtstack.flink.sql.dirtyManager.consumer.AbstractDirtyDataConsumer; +import com.dtstack.flink.sql.dirtyManager.entity.DirtyDataEntity; +import com.dtstack.flink.sql.util.PluginUtil; +import java.io.File; import java.io.Serializable; +import java.lang.reflect.Constructor; +import java.net.URL; import java.util.Map; +import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -35,6 +41,12 @@ public class DirtyDataManager implements Serializable { private static final long serialVersionUID = 7190970299538893497L; + private static final String CLASS_PRE_STR = "com.dtstack.flink.sql.dirty"; + + private static final String CLASS_POST_STR = "DirtyDataConsumer"; + + private static final String DIRTY_CONSUMER_PATH = "dirtyData"; + /** * 写入队列阻塞时间 */ @@ -50,7 +62,7 @@ public class DirtyDataManager implements Serializable { */ private AtomicLong count = new AtomicLong(0); - private AbstractDirtyDataConsumer consumer; + private AbstractDirtyDataConsumer consumer; public static DirtyDataManager newInstance() { @@ -60,7 +72,15 @@ public static DirtyDataManager newInstance() { public AbstractDirtyDataConsumer createConsumer(Map properties) throws Exception { // 利用类加载的方式动态加载 String type = properties.getOrDefault("type", "print"); - return null; + String consumerType = DIRTY_CONSUMER_PATH + File.separator + type; + String consumerJar = PluginUtil.getJarFileDirPath(consumerType, properties.getOrDefault("pluginPath", null), "shipfile"); + String className = CLASS_PRE_STR + "." + type.toLowerCase() + "." + upperCaseFirstChar(type + CLASS_POST_STR); + + return ClassLoaderManager.newInstance(consumerJar, cl -> { + Class clazz = cl.loadClass(className); + Constructor constructor = clazz.getConstructor(String.class); + return (AbstractDirtyDataConsumer) constructor.newInstance(type + CLASS_POST_STR); + }); } public void close() throws InterruptedException { @@ -82,4 +102,7 @@ public void collectDirtyData(String dataInfo, String cause, String field) throws consumer.setQueue(queue); } + private static String upperCaseFirstChar(String str) { + return str.substring(0, 1).toUpperCase() + str.substring(1); + } } diff --git a/core/src/test/java/com/dtstack/flink/sql/dirtyManager/TestDirtyDataManager.java b/core/src/test/java/com/dtstack/flink/sql/dirtyManager/TestDirtyDataManager.java new file mode 100644 index 000000000..1a9caf3ed --- /dev/null +++ b/core/src/test/java/com/dtstack/flink/sql/dirtyManager/TestDirtyDataManager.java @@ -0,0 +1,27 @@ +package com.dtstack.flink.sql.dirtyManager; + +import com.dtstack.flink.sql.dirtyManager.consumer.AbstractDirtyDataConsumer; +import com.dtstack.flink.sql.dirtyManager.manager.DirtyDataManager; +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author tiezhu + * Company dtstack + * Date 2020/8/28 星期五 + */ +public class TestDirtyDataManager { + @Test + public void testCreateConsumer() throws Exception { + Map properties = new HashMap<>(); + properties.put("type", "print"); + properties.put("pluginPath", "/Users/wtz/IdeaProjects/flinkStreamSQL/sqlplugins"); + + DirtyDataManager manager = DirtyDataManager.newInstance(); + AbstractDirtyDataConsumer consumer = manager.createConsumer(properties); + Assert.assertNotNull(consumer); + } +} diff --git a/dirtyDataManager/manager/pom.xml b/dirtyDataManager/manager/pom.xml deleted file mode 100644 index f21f2e20c..000000000 --- a/dirtyDataManager/manager/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - sql.dirtyDataManager - com.dtstack.flink - 1.0-SNAPSHOT - - 4.0.0 - - manager - - - - com.fasterxml.jackson.core - jackson-databind - 2.10.3 - - - - log4j - log4j - 1.2.17 - - - - org.slf4j - slf4j-simple - 1.7.30 - - - - \ No newline at end of file diff --git a/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/classLoader/DirtyManagerClassLoader.java b/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/classLoader/DirtyManagerClassLoader.java deleted file mode 100644 index c2e2cbcd5..000000000 --- a/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/classLoader/DirtyManagerClassLoader.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.dirty.classLoader; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.URL; -import java.net.URLClassLoader; - -/** - * @author tiezhu - * Company dtstack - * Date 2020/8/27 星期四 - */ -public class DirtyManagerClassLoader extends URLClassLoader { - private final Logger LOG = LoggerFactory.getLogger(DirtyManagerClassLoader.class); - - /** - * The parent class loader. - */ - protected ClassLoader parent; - - public DirtyManagerClassLoader(URL[] urls, ClassLoader parent) { - super(urls, parent); - this.parent = parent; - } - - - @Override - public Class loadClass(String name) throws ClassNotFoundException { - return this.loadClass(name, false); - } - - @Override - protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { - synchronized (getClassLoadingLock(name)) { - if (LOG.isDebugEnabled()) { - LOG.debug("loadClass(" + name + ", " + resolve + ")"); - } - Class clazz; - - // (0.1) Check our previously loaded class cache - clazz = findLoadedClass(name); - if (clazz != null) { - if (LOG.isDebugEnabled()) { - LOG.debug(" Returning class from cache"); - } - if (resolve) { - resolveClass(clazz); - } - return (clazz); - } - - // (2) Search local repositories - if (LOG.isDebugEnabled()) { - LOG.debug(" Searching local repositories"); - } - try { - clazz = findClass(name); - if (clazz != null) { - if (LOG.isDebugEnabled()) { - LOG.debug(" Loading class from local repository"); - } - if (resolve) { - resolveClass(clazz); - } - return (clazz); - } - } catch (ClassNotFoundException e) { - // Ignore - } - - if (LOG.isDebugEnabled()) { - LOG.debug(" Delegating to parent classloader at end: " + parent); - } - - try { - clazz = Class.forName(name, false, parent); - if (LOG.isDebugEnabled()) { - LOG.debug(" Loading class from parent"); - } - if (resolve) { - resolveClass(clazz); - } - return (clazz); - } catch (ClassNotFoundException e) { - // Ignore - } - } - - throw new ClassNotFoundException(name); - } -} diff --git a/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/util/ManagerUtil.java b/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/util/ManagerUtil.java deleted file mode 100644 index bfa343e2e..000000000 --- a/dirtyDataManager/manager/src/main/java/com/dtstack/flink/sql/dirty/util/ManagerUtil.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.dirty.util; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.io.IOException; -import java.util.Map; - -/** - * @author tiezhu - * Company dtstack - * Date 2020/8/27 星期四 - */ -public class ManagerUtil { - private static final ObjectMapper objectMapper = new ObjectMapper(); - - @SuppressWarnings("unchecked") - public static Map convertJsonToMap(String jsonStr) throws IOException { - return (Map) objectMapper.readValue(jsonStr, Map.class); - } - - public static Map convertEntityToMap(Object obj) throws IOException { - String jsonStr = objectMapper.writeValueAsString(obj); - return convertJsonToMap(jsonStr); - } -} diff --git a/dirtyDataManager/mysql/pom.xml b/dirtyDataManager/mysql/pom.xml index 764cb65a6..bd786052e 100644 --- a/dirtyDataManager/mysql/pom.xml +++ b/dirtyDataManager/mysql/pom.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - sql.dirtyDataManager + sql.dirtyConsumer com.dtstack.flink 1.0-SNAPSHOT @@ -13,14 +13,8 @@ 5.1.46 - dirtyManager.mysql + dirtyConsumer.mysql - - com.dtstack.flink - manager - 1.0-SNAPSHOT - compile - mysql @@ -29,6 +23,67 @@ + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + false + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java b/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java index 49f892bc0..24564b86f 100644 --- a/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java +++ b/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.dirty.mysql; -import com.dtstack.flink.sql.dirty.consumer.AbstractDirtyDataConsumer; +import com.dtstack.flink.sql.dirtyManager.consumer.AbstractDirtyDataConsumer; import java.sql.Connection; import java.sql.DriverManager; diff --git a/dirtyDataManager/pom.xml b/dirtyDataManager/pom.xml index 360450174..2368cefc1 100644 --- a/dirtyDataManager/pom.xml +++ b/dirtyDataManager/pom.xml @@ -9,13 +9,32 @@ 4.0.0 - sql.dirtyDataManager + sql.dirtyConsumer pom - manager print mysql + + 3.8.1 + 1.0-SNAPSHOT + + + + + junit + junit + ${junit.version} + test + + + + com.dtstack.flink + sql.core + ${sql.core.version} + provided + + \ No newline at end of file diff --git a/dirtyDataManager/print/pom.xml b/dirtyDataManager/print/pom.xml index 71056028a..e42390498 100644 --- a/dirtyDataManager/print/pom.xml +++ b/dirtyDataManager/print/pom.xml @@ -3,21 +3,75 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - sql.dirtyDataManager + sql.dirtyConsumer com.dtstack.flink 1.0-SNAPSHOT 4.0.0 - dirtyManager.print - - - com.dtstack.flink - manager - 1.0-SNAPSHOT - compile - - + dirtyConsumer.print + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + false + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dirtyDataManager/print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java b/dirtyDataManager/print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java index 4cb818034..51ba7c68d 100644 --- a/dirtyDataManager/print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java +++ b/dirtyDataManager/print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.dirty.print; -import com.dtstack.flink.sql.dirty.consumer.AbstractDirtyDataConsumer; -import com.dtstack.flink.sql.dirty.entity.DirtyDataEntity; +import com.dtstack.flink.sql.dirtyManager.consumer.AbstractDirtyDataConsumer; +import com.dtstack.flink.sql.dirtyManager.entity.DirtyDataEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From 055bcf5824f1efce8ec9d7962b5c25c1e7872c49 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 3 Sep 2020 21:49:10 +0800 Subject: [PATCH 05/80] =?UTF-8?q?[feat]=20=E5=AE=8C=E6=88=90=20print=20dir?= =?UTF-8?q?ty=20consumer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/pom.xml | 11 ++++ .../consumer/AbstractDirtyDataConsumer.java | 39 +++++++++--- .../manager/DirtyDataManager.java | 63 ++++++++++++------- .../sql/dirtyManager/manager/TestMain.java | 23 +++++++ .../dirtyManager/TestDirtyDataManager.java | 12 ---- .../dirty/mysql/MysqlDirtyDataConsumer.java | 16 ++++- dirtyDataManager/print/pom.xml | 19 ++++++ .../dirty/print/PrintDirtyDataConsumer.java | 10 ++- .../print/TestPrintDirtyDataConsumer.java | 12 ++++ 9 files changed, 160 insertions(+), 45 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java create mode 100644 dirtyDataManager/print/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java diff --git a/core/pom.xml b/core/pom.xml index ce7799f5a..178effe15 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -123,6 +123,17 @@ joda-time 2.5 + + org.slf4j + slf4j-simple + 1.7.30 + + + + log4j + log4j + 1.2.17 + diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java index 00cd793a1..830f1879f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java @@ -19,26 +19,37 @@ package com.dtstack.flink.sql.dirtyManager.consumer; import com.dtstack.flink.sql.dirtyManager.entity.DirtyDataEntity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.Serializable; +import java.util.Map; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; /** * @author tiezhu * Company dtstack * Date 2020/8/27 星期四 */ -public abstract class AbstractDirtyDataConsumer extends Thread implements Serializable { - private static final long serialVersionUID = -6058598201315176687L; +public abstract class AbstractDirtyDataConsumer implements Runnable, Serializable { + protected static final long serialVersionUID = -6058598201315176687L; - private long count = 0; + protected static final Logger LOG = LoggerFactory.getLogger(AbstractDirtyDataConsumer.class); + + protected long errorLimit = 1000; + protected long errorCount = 0; + + protected long count = 0; + + public AtomicBoolean isRunning = new AtomicBoolean(true); protected LinkedBlockingQueue queue; /** * 消费队列数据 */ - public abstract void consume() throws InterruptedException; + public abstract void consume() throws Exception; /** * 关闭消费者，需要释放资源 @@ -48,16 +59,26 @@ public abstract class AbstractDirtyDataConsumer extends Thread implements Seria /** * 初始化消费者，初始化定时任务 */ - public void init() { - - } + public abstract void init(Map properties); @Override public void run() { - + try { + LOG.info("start to consume dirty data"); + while (isRunning.get()) { + consume(); + } + } catch (Exception e) { + LOG.error("consume dirtyData error"); + errorCount++; + if (errorCount == errorLimit) { + throw new RuntimeException("脏数据消费失败达到上限，任务失败"); + } + } } - public void setQueue(LinkedBlockingQueue queue) { + public AbstractDirtyDataConsumer setQueue(LinkedBlockingQueue queue) { this.queue = queue; + return this; } } diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java index f16e3daef..749038d12 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java @@ -22,13 +22,13 @@ import com.dtstack.flink.sql.dirtyManager.consumer.AbstractDirtyDataConsumer; import com.dtstack.flink.sql.dirtyManager.entity.DirtyDataEntity; import com.dtstack.flink.sql.util.PluginUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.Serializable; import java.lang.reflect.Constructor; -import java.net.URL; import java.util.Map; -import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -41,6 +41,8 @@ public class DirtyDataManager implements Serializable { private static final long serialVersionUID = 7190970299538893497L; + private static final Logger LOG = LoggerFactory.getLogger(DirtyDataManager.class); + private static final String CLASS_PRE_STR = "com.dtstack.flink.sql.dirty"; private static final String CLASS_POST_STR = "DirtyDataConsumer"; @@ -55,22 +57,32 @@ public class DirtyDataManager implements Serializable { /** * 缓存脏数据信息队列 */ - private LinkedBlockingQueue queue = new LinkedBlockingQueue<>(); + public final LinkedBlockingQueue queue = new LinkedBlockingQueue<>(); /** * 统计manager收集到的脏数据条数 */ - private AtomicLong count = new AtomicLong(0); - - private AbstractDirtyDataConsumer consumer; + private final AtomicLong count = new AtomicLong(0); - public static DirtyDataManager newInstance() { + public static AbstractDirtyDataConsumer consumer; - return new DirtyDataManager(); + /** + * 通过参数生成manager实例，并同时将consumer实例化 + */ + public static DirtyDataManager newInstance(Map properties) throws Exception { + DirtyDataManager manager = new DirtyDataManager(); + manager.blockingInterval = Long.parseLong(properties.getOrDefault("blockingInterval", "60")); + consumer = createConsumer(properties); + consumer.init(properties); + Thread dirtyDataConsumer = new Thread(consumer.setQueue(manager.queue), "dirtyData Consumer"); + dirtyDataConsumer.start(); + return manager; } - public AbstractDirtyDataConsumer createConsumer(Map properties) throws Exception { - // 利用类加载的方式动态加载 + /** + * 通过动态加载的方式加载Consumer + */ + private static AbstractDirtyDataConsumer createConsumer(Map properties) throws Exception { String type = properties.getOrDefault("type", "print"); String consumerType = DIRTY_CONSUMER_PATH + File.separator + type; String consumerJar = PluginUtil.getJarFileDirPath(consumerType, properties.getOrDefault("pluginPath", null), "shipfile"); @@ -78,20 +90,20 @@ public AbstractDirtyDataConsumer createConsumer(Map properties) return ClassLoaderManager.newInstance(consumerJar, cl -> { Class clazz = cl.loadClass(className); - Constructor constructor = clazz.getConstructor(String.class); - return (AbstractDirtyDataConsumer) constructor.newInstance(type + CLASS_POST_STR); + Constructor constructor = clazz.getConstructor(); + return (AbstractDirtyDataConsumer) constructor.newInstance(); }); } - public void close() throws InterruptedException { - if (!queue.isEmpty()) { - flush(); + /** + * 脏数据收集任务停止，任务停止之前，需要将队列中所有的数据清空 + */ + public void close() throws Exception { + if (!queue.isEmpty() && checkConsumer()) { + consumer.consume(); } - } - - public void flush() throws InterruptedException { - consumer.setQueue(queue); - consumer.consume(); + LOG.info("dirty consumer is closing ......"); + consumer.close(); } // 收集脏数据 @@ -99,9 +111,18 @@ public void collectDirtyData(String dataInfo, String cause, String field) throws DirtyDataEntity dirtyDataEntity = new DirtyDataEntity(dataInfo, System.currentTimeMillis(), cause, field); queue.offer(dirtyDataEntity, blockingInterval, TimeUnit.MILLISECONDS); count.incrementAndGet(); - consumer.setQueue(queue); } + /** + * 查看consumer当前状态 + */ + public boolean checkConsumer() { + return consumer.isRunning.get(); + } + + /** + * 首字母大写 + */ private static String upperCaseFirstChar(String str) { return str.substring(0, 1).toUpperCase() + str.substring(1); } diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java new file mode 100644 index 000000000..78dcd7a0c --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java @@ -0,0 +1,23 @@ +package com.dtstack.flink.sql.dirtyManager.manager; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author tiezhu + * Company dtstack + * Date 2020/8/28 星期五 + */ +public class TestMain { + public static void main(String[] args) throws Exception { + Map properties = new HashMap<>(); + properties.put("type", "print"); + properties.put("pluginPath", "/Users/wtz/IdeaProjects/flinkStreamSQL/sqlplugins"); + + DirtyDataManager manager = DirtyDataManager.newInstance(properties); + for (int i = 0; i < 100; i++) { + Thread.sleep(1000); + manager.collectDirtyData("testDirtyData", new Exception("testException").getMessage(), "testField"); + } + } +} diff --git a/core/src/test/java/com/dtstack/flink/sql/dirtyManager/TestDirtyDataManager.java b/core/src/test/java/com/dtstack/flink/sql/dirtyManager/TestDirtyDataManager.java index 1a9caf3ed..ec08bb9d3 100644 --- a/core/src/test/java/com/dtstack/flink/sql/dirtyManager/TestDirtyDataManager.java +++ b/core/src/test/java/com/dtstack/flink/sql/dirtyManager/TestDirtyDataManager.java @@ -1,9 +1,6 @@ package com.dtstack.flink.sql.dirtyManager; -import com.dtstack.flink.sql.dirtyManager.consumer.AbstractDirtyDataConsumer; import com.dtstack.flink.sql.dirtyManager.manager.DirtyDataManager; -import org.junit.Assert; -import org.junit.Test; import java.util.HashMap; import java.util.Map; @@ -14,14 +11,5 @@ * Date 2020/8/28 星期五 */ public class TestDirtyDataManager { - @Test - public void testCreateConsumer() throws Exception { - Map properties = new HashMap<>(); - properties.put("type", "print"); - properties.put("pluginPath", "/Users/wtz/IdeaProjects/flinkStreamSQL/sqlplugins"); - DirtyDataManager manager = DirtyDataManager.newInstance(); - AbstractDirtyDataConsumer consumer = manager.createConsumer(properties); - Assert.assertNotNull(consumer); - } } diff --git a/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java b/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java index 24564b86f..c4a172dae 100644 --- a/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java +++ b/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java @@ -19,11 +19,14 @@ package com.dtstack.flink.sql.dirty.mysql; import com.dtstack.flink.sql.dirtyManager.consumer.AbstractDirtyDataConsumer; +import com.dtstack.flink.sql.dirtyManager.entity.DirtyDataEntity; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; /** * @author tiezhu @@ -45,6 +48,8 @@ public class MysqlDirtyDataConsumer extends AbstractDirtyDataConsumer { private Connection connection; + private String tableName; + private PreparedStatement getStatement(String url, String userName, String password) throws ClassNotFoundException, SQLException { @@ -79,12 +84,19 @@ private boolean createTable(String tableName) throws SQLException { } @Override - public void consume() { - + public void consume() throws Exception{ + if (!isCreatedTable) { + createTable("tableName"); + } } @Override public void close() { } + + @Override + public void init(Map properties) { + + } } diff --git a/dirtyDataManager/print/pom.xml b/dirtyDataManager/print/pom.xml index e42390498..8088fbcc4 100644 --- a/dirtyDataManager/print/pom.xml +++ b/dirtyDataManager/print/pom.xml @@ -73,5 +73,24 @@ + + + junit + junit + 4.8.2 + test + + + org.slf4j + slf4j-simple + 1.7.30 + + + + log4j + log4j + 1.2.17 + + \ No newline at end of file diff --git a/dirtyDataManager/print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java b/dirtyDataManager/print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java index 51ba7c68d..4e444748d 100644 --- a/dirtyDataManager/print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java +++ b/dirtyDataManager/print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java @@ -23,6 +23,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Map; + /** * @author tiezhu * Company dtstack @@ -36,6 +38,7 @@ public class PrintDirtyDataConsumer extends AbstractDirtyDataConsumer { @Override public void consume() throws InterruptedException { DirtyDataEntity dataEntity = queue.take(); + count++; LOG.warn("get dirtyData: " + dataEntity.getDirtyData() + "\n" + "cause: " + dataEntity.getCause() + "\n" + "processTime: " + dataEntity.getProcessDate() + "\n" @@ -44,6 +47,11 @@ public void consume() throws InterruptedException { @Override public void close() { - // Do nothing + isRunning.compareAndSet(true, false); + } + + @Override + public void init(Map properties) { + LOG.info("Log dirty consumer init ......"); } } diff --git a/dirtyDataManager/print/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java b/dirtyDataManager/print/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java new file mode 100644 index 000000000..de8864114 --- /dev/null +++ b/dirtyDataManager/print/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java @@ -0,0 +1,12 @@ +package com.dtstack.flink.sql.dirty.print; + +/** + * @author tiezhu + * Company dtstack + * Date 2020/8/28 星期五 + */ +public class TestPrintDirtyDataConsumer { + public static void main(String[] args) { + + } +} From 7ee482dc6e85e6f2df3f43d9a97b8b96edb606f7 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 4 Sep 2020 11:50:26 +0800 Subject: [PATCH 06/80] [feat] rename print to console and add TODO --- .../consumer/AbstractDirtyDataConsumer.java | 10 ++- .../dirtyManager/entity/DirtyDataEntity.java | 10 +++ .../manager/DirtyDataManager.java | 29 +++++-- .../sql/dirtyManager/manager/TestMain.java | 2 +- dirtyDataManager/{print => console}/pom.xml | 6 +- .../print/ConsoleDirtyDataConsumer.java} | 7 +- .../print/TestPrintDirtyDataConsumer.java | 0 .../dirty/mysql/MysqlDirtyDataConsumer.java | 78 ++++++++++++------- dirtyDataManager/pom.xml | 2 +- 9 files changed, 102 insertions(+), 42 deletions(-) rename dirtyDataManager/{print => console}/pom.xml (95%) rename dirtyDataManager/{print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java => console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java} (86%) rename dirtyDataManager/{print => console}/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java (100%) diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java index 830f1879f..c2401242d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java @@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory; import java.io.Serializable; +import java.sql.SQLException; import java.util.Map; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; @@ -59,7 +60,14 @@ public abstract class AbstractDirtyDataConsumer implements Runnable, Serializabl /** * 初始化消费者，初始化定时任务 */ - public abstract void init(Map properties); + public abstract void init(Map properties) throws Exception; + + /** + * 检验consumer是否正在执行 + */ + public boolean isRunning() { + return isRunning.get(); + } @Override public void run() { diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java index 3ea41d8a1..8a64df7a7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java @@ -84,4 +84,14 @@ public DirtyDataEntity(String dirtyData, Long processDate, String cause, String this.cause = cause; this.field = field; } + + @Override + public String toString() { + return "DirtyDataEntity{" + + "dirtyData='" + dirtyData + '\'' + + ", processDate=" + processDate + + ", cause='" + cause + '\'' + + ", field='" + field + '\'' + + '}'; + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java index 749038d12..0c90c770c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java @@ -39,6 +39,7 @@ * Date 2020/8/27 星期四 */ public class DirtyDataManager implements Serializable { + //TODO 需要确保产生脏数据后至少一次是成功发送给了consumer，至于consumer是否成功消费，manager不需要关心 private static final long serialVersionUID = 7190970299538893497L; private static final Logger LOG = LoggerFactory.getLogger(DirtyDataManager.class); @@ -64,6 +65,11 @@ public class DirtyDataManager implements Serializable { */ private final AtomicLong count = new AtomicLong(0); + /** + * 脏数据写入队列失败条数 + */ + private final AtomicLong errorCount = new AtomicLong(0); + public static AbstractDirtyDataConsumer consumer; /** @@ -74,6 +80,7 @@ public static DirtyDataManager newInstance(Map properties) throw manager.blockingInterval = Long.parseLong(properties.getOrDefault("blockingInterval", "60")); consumer = createConsumer(properties); consumer.init(properties); + //TODO 使用线程池创建线程，不要用thread Thread dirtyDataConsumer = new Thread(consumer.setQueue(manager.queue), "dirtyData Consumer"); dirtyDataConsumer.start(); return manager; @@ -102,22 +109,32 @@ public void close() throws Exception { if (!queue.isEmpty() && checkConsumer()) { consumer.consume(); } - LOG.info("dirty consumer is closing ......"); + LOG.info("dirty consumer is closing ..."); consumer.close(); } - // 收集脏数据 - public void collectDirtyData(String dataInfo, String cause, String field) throws InterruptedException { + /** + * 收集脏数据放入队列缓存中，记录放入失败的数目和存入队列中的总数目，如果放入失败的数目超过一定比列，那么manager任务失败 + */ + public void collectDirtyData(String dataInfo, String cause, String field) { DirtyDataEntity dirtyDataEntity = new DirtyDataEntity(dataInfo, System.currentTimeMillis(), cause, field); - queue.offer(dirtyDataEntity, blockingInterval, TimeUnit.MILLISECONDS); - count.incrementAndGet(); + try { + queue.offer(dirtyDataEntity, blockingInterval, TimeUnit.MILLISECONDS); + count.incrementAndGet(); + } catch (Exception ignored) { + LOG.warn("dirty Data insert error ... Failed number: " + errorCount.incrementAndGet()); + LOG.warn("error dirty data:" + dirtyDataEntity.toString()); + if (errorCount.get() > Math.ceil(count.longValue() * 0.8)) { + throw new RuntimeException("The number of failed number reaches the limit, manager fails"); + } + } } /** * 查看consumer当前状态 */ public boolean checkConsumer() { - return consumer.isRunning.get(); + return consumer.isRunning(); } /** diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java index 78dcd7a0c..b6b2b6c8f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java @@ -11,7 +11,7 @@ public class TestMain { public static void main(String[] args) throws Exception { Map properties = new HashMap<>(); - properties.put("type", "print"); + properties.put("type", "console"); properties.put("pluginPath", "/Users/wtz/IdeaProjects/flinkStreamSQL/sqlplugins"); DirtyDataManager manager = DirtyDataManager.newInstance(properties); diff --git a/dirtyDataManager/print/pom.xml b/dirtyDataManager/console/pom.xml similarity index 95% rename from dirtyDataManager/print/pom.xml rename to dirtyDataManager/console/pom.xml index 8088fbcc4..438104e4e 100644 --- a/dirtyDataManager/print/pom.xml +++ b/dirtyDataManager/console/pom.xml @@ -9,7 +9,7 @@ 4.0.0 - dirtyConsumer.print + dirtyConsumer.console @@ -64,8 +64,8 @@ - + diff --git a/dirtyDataManager/print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java b/dirtyDataManager/console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java similarity index 86% rename from dirtyDataManager/print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java rename to dirtyDataManager/console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java index 4e444748d..bf7d20156 100644 --- a/dirtyDataManager/print/src/main/java/com/dtstack/flink/sql/dirty/print/PrintDirtyDataConsumer.java +++ b/dirtyDataManager/console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java @@ -30,10 +30,10 @@ * Company dtstack * Date 2020/8/27 星期四 */ -public class PrintDirtyDataConsumer extends AbstractDirtyDataConsumer { +public class ConsoleDirtyDataConsumer extends AbstractDirtyDataConsumer { private static final long serialVersionUID = 5727194679865135189L; - private final Logger LOG = LoggerFactory.getLogger(PrintDirtyDataConsumer.class); + private final Logger LOG = LoggerFactory.getLogger(ConsoleDirtyDataConsumer.class); @Override public void consume() throws InterruptedException { @@ -48,10 +48,11 @@ public void consume() throws InterruptedException { @Override public void close() { isRunning.compareAndSet(true, false); + LOG.info("console dirty consumer close ..."); } @Override public void init(Map properties) { - LOG.info("Log dirty consumer init ......"); + LOG.info("console dirty consumer init ..."); } } diff --git a/dirtyDataManager/print/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java b/dirtyDataManager/console/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java similarity index 100% rename from dirtyDataManager/print/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java rename to dirtyDataManager/console/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java diff --git a/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java b/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java index c4a172dae..54bc19b82 100644 --- a/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java +++ b/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java @@ -19,14 +19,12 @@ package com.dtstack.flink.sql.dirty.mysql; import com.dtstack.flink.sql.dirtyManager.consumer.AbstractDirtyDataConsumer; -import com.dtstack.flink.sql.dirtyManager.entity.DirtyDataEntity; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.Map; -import java.util.concurrent.LinkedBlockingQueue; /** * @author tiezhu @@ -34,13 +32,16 @@ * Date 2020/8/27 星期四 */ public class MysqlDirtyDataConsumer extends AbstractDirtyDataConsumer { + //TODO 添加batchSize 和 定时任务 private static final long serialVersionUID = -2959753658786001679L; private static final String DRIVER_NAME = "com.mysql.jdbc.Driver"; + private final Object LOCK_STR = new Object(); + private boolean isCreatedTable = false; - private String[] tableField = {"id", "dirtyData", "processTime", "cause", "field"}; + private final String[] tableField = {"id", "dirtyData", "processTime", "cause", "field"}; private String SQL = "INSERT INTO ? (?, ?, ?, ?) VALUES (?, ?, ?, ?) "; @@ -50,14 +51,15 @@ public class MysqlDirtyDataConsumer extends AbstractDirtyDataConsumer { private String tableName; - private PreparedStatement getStatement(String url, - String userName, - String password) throws ClassNotFoundException, SQLException { - Class.forName(DRIVER_NAME); + private void setStatement(String url, + String userName, + String password) throws ClassNotFoundException, SQLException { + synchronized (LOCK_STR) { + Class.forName(DRIVER_NAME); - connection = DriverManager.getConnection(url, userName, password); - statement = connection.prepareStatement(SQL); - return statement; + connection = DriverManager.getConnection(url, userName, password); + statement = connection.prepareStatement(SQL); + } } private String quoteIdentifier(String tableName) { @@ -66,37 +68,59 @@ private String quoteIdentifier(String tableName) { /** * 创建存储脏数据的表 - * @param tableName 表名 + * + * @param tableName 表名 * @return 是否创建成功 * @throws SQLException SQL异常 */ - private boolean createTable(String tableName) throws SQLException { - String sql = - "CREATE TABLE ` " + tableName + "` (" + - " `id` int(11) not null AUTO_INCREMENT,\n" + - " `dirtyData` varchar(100) DEFAULT NULL,\n" + - " `processTime` varchar(100) DEFAULT NULL,\n" + - " `cause` date DEFAULT NULL,\n" + - " `field` varchar(100) DEFAULT NULL,\n" + - " PRIMARY KEY (id)\n" + - ") DEFAULT CHARSET=utf8;"; - return statement.execute(sql); + private boolean createTable(String tableName) { + try { + String defaultTable = ""; + String sql = + "CREATE TABLE ` " + tableName + "` (" + + " `id` int(11) not null AUTO_INCREMENT,\n" + + " `dirtyData` varchar(100) DEFAULT NULL,\n" + + " `processTime` varchar(100) DEFAULT NULL,\n" + + " `cause` date DEFAULT NULL,\n" + + " `field` varchar(100) DEFAULT NULL,\n" + + " PRIMARY KEY (id)\n" + + ") DEFAULT CHARSET=utf8;"; + return statement.execute(sql); + } catch (SQLException e) { + throw new RuntimeException("create table error !", e); + } } @Override - public void consume() throws Exception{ + public void consume() throws Exception { if (!isCreatedTable) { - createTable("tableName"); + createTable(tableName); } + } @Override public void close() { - + try { + if (connection != null && !connection.isValid(1000)) { + connection.close(); + } + + if (statement != null && !statement.isClosed()) { + statement.close(); + } + } catch (SQLException e) { + throw new RuntimeException("close mysql resource error !"); + } } @Override - public void init(Map properties) { - + public void init(Map properties) throws Exception { + tableName = properties.get("tableName"); + String userName = properties.get("userName"); + String password = properties.get("password"); + String url = properties.get("url"); + isCreatedTable = Boolean.parseBoolean(properties.get("isCreatedTable")); + setStatement(url, userName, password); } } diff --git a/dirtyDataManager/pom.xml b/dirtyDataManager/pom.xml index 2368cefc1..c077855f2 100644 --- a/dirtyDataManager/pom.xml +++ b/dirtyDataManager/pom.xml @@ -12,7 +12,7 @@ sql.dirtyConsumer pom - print + console mysql From 3c753c1dc30420ef1b57cc9d77ae13f7a88f7861 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 10 Sep 2020 20:04:43 +0800 Subject: [PATCH 07/80] [tmp] temp save --- .../executionHistory/executionHistory.bin | Bin 0 -> 20147 bytes .../executionHistory/executionHistory.lock | Bin 0 -> 17 bytes .gradle/6.1.1/fileChanges/last-build.bin | Bin 0 -> 1 bytes .gradle/6.1.1/fileHashes/fileHashes.bin | Bin 0 -> 18697 bytes .gradle/6.1.1/fileHashes/fileHashes.lock | Bin 0 -> 17 bytes .gradle/6.1.1/gc.properties | 0 .../buildOutputCleanup.lock | Bin 0 -> 17 bytes .gradle/buildOutputCleanup/cache.properties | 2 + .gradle/buildOutputCleanup/outputFiles.bin | Bin 0 -> 18695 bytes .gradle/checksums/checksums.lock | Bin 0 -> 17 bytes .gradle/vcs-1/gc.properties | 0 .../consumer/AbstractDirtyDataConsumer.java | 1 + .../manager/DirtyDataManager.java | 25 ++++++------ .../dirty/mysql/MysqlDirtyDataConsumer.java | 36 +++++++++++------- gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 58695 bytes gradle/wrapper/gradle-wrapper.properties | 5 +++ 16 files changed, 45 insertions(+), 24 deletions(-) create mode 100644 .gradle/6.1.1/executionHistory/executionHistory.bin create mode 100644 .gradle/6.1.1/executionHistory/executionHistory.lock create mode 100644 .gradle/6.1.1/fileChanges/last-build.bin create mode 100644 .gradle/6.1.1/fileHashes/fileHashes.bin create mode 100644 .gradle/6.1.1/fileHashes/fileHashes.lock create mode 100644 .gradle/6.1.1/gc.properties create mode 100644 .gradle/buildOutputCleanup/buildOutputCleanup.lock create mode 100644 .gradle/buildOutputCleanup/cache.properties create mode 100644 .gradle/buildOutputCleanup/outputFiles.bin create mode 100644 .gradle/checksums/checksums.lock create mode 100644 .gradle/vcs-1/gc.properties create mode 100644 gradle/wrapper/gradle-wrapper.jar create mode 100644 gradle/wrapper/gradle-wrapper.properties diff --git a/.gradle/6.1.1/executionHistory/executionHistory.bin b/.gradle/6.1.1/executionHistory/executionHistory.bin new file mode 100644 index 0000000000000000000000000000000000000000..a191b4fe20f13b943ea709880699c920aae01c63 GIT binary patch literal 20147 zcmeI%%}W$v90%|jCu+&WB~*-%E*1zmvoG$hhC#}j3RdQ72_44SnP+#$eVd+{by0{e zA-#yT5D^63RH9Hy2oI5m4v_>!r=m-LKnD%#6x8g_4l3KhwLL_92WIABW}ffwc|QB> z>|uip)9CzF+?dU+ezS=d1Rwwb2tWV=5P$##AOHafKmY;|fB*y_009U<00RGuz=%1K z{nlg}k5}azwONL79WWcLOFleWRWtw4y4}}h|9@bg&Fd}JJscqb0SG_<0uX=z1Rwwb z2tWV=5P$##AOHafKmY;|fB*y_009U<00Izz00bZa0SG_<0uX=z1Rwwb2tWV=5cn4a zybQBfHFqh$KZt9Wvm~Azm!_fTgRYt+8^in58t0@=+=&3BpU7${Ug% z*ZC}!G>uUHqIDL7L{3x5Ac}MGarM|-r|Zk37oP?{)K5pN{yKqq&5GR_HKBxLs%nPV zEYeI$loM$^Ly5cDZh1skTOb^9dzFZb1iTTVcmi!6$w!n9zc&!*a1m$C8avwbq#-w# z+6XUnH7q-3Br3<$3DPa;#Id-OnKlnfMlAn$u6L*_IM5@GgnEX=zH?`K?5z9)+-wm@ zP*mN}bNM4x!x=+Orz%lvTi_oG-rHI*6q7tYe<+h=wsjkx)QIExj^-TPGW>kEYu){4 z?`$nGLKEC-%rG?lgdpgIPN*``?NwuzC@|Onx?nA*RzL6N-K}9Y#ZRf)tab4Mp}L90 zO&4x5duGCtA;+vwo?RA3%y3=E8dE~QLZm^O9w)M)3z3AHiiZqJq-5y)Kruhdn|(Rw zs!;2D`*M9DdBm}C>D2tnZc=vBr=;-pafzN*6NG)kmNgH-UQ0zb`@3hxan7qZK0p0l zeKJ0|lG6_Mep{(vcm<E`;!_D=TD*zmiT zY{hF-KCxubxAek0uVA!P)~d;J&Xy}jjeWlRb)o3bx@T)Y+SxJ_%6-1Hp!p+{`V9!r Bkg)&& literal 0 HcmV?d00001 diff --git a/.gradle/6.1.1/executionHistory/executionHistory.lock b/.gradle/6.1.1/executionHistory/executionHistory.lock new file mode 100644 index 0000000000000000000000000000000000000000..af7ea0a9bab9521238aba7486ef526f312a4c6bd GIT binary patch literal 17 TcmZRs2>pJ8Ra8id0Ror-C;bBH literal 0 HcmV?d00001 diff --git a/.gradle/6.1.1/fileChanges/last-build.bin b/.gradle/6.1.1/fileChanges/last-build.bin new file mode 100644 index 0000000000000000000000000000000000000000..f76dd238ade08917e6712764a16a22005a50573d GIT binary patch literal 1 IcmZPo000310RR91 literal 0 HcmV?d00001 diff --git a/.gradle/6.1.1/fileHashes/fileHashes.bin b/.gradle/6.1.1/fileHashes/fileHashes.bin new file mode 100644 index 0000000000000000000000000000000000000000..ebcb5d2291320ab033c01022fc6819cf388779eb GIT binary patch literal 18697 zcmeI%Pbfq|0LSrp+m>b3+qR{3kPunbJk8b)-&70{r{hH=;eG@`NL*L#I)zMVlvO@p?1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~f&U^fCm+(F9uw{Jxf?Sl#89K$_z#U-KK$Hqq>iWZ z`v1Z8R`0Z(@z;C#n9hBjOLyj1%JA}4od@ib(}l^IelNe*xn5Y6eR;w@1_#$aqCjwH?uclr=&=7l3^G9)ZcL;_YHAYtK}%aoZBavu zLek_=yVz2+g=5rR_dH0n1wzF49qwMf?;UsdS?^siCOP&Y!}LeIKV^mh0tg_000Iag zfB*srAbVB>0%*OvsU zoOa{Y>y^@+bVqx$l5r01-u9$twYSRCt9C9tEPY=4eY)J)Ei|;d+FyIsEtAjI5HYubnG`o+fL+N|`R_C#^}d?#fWq~B^^zuezk&b1z-SF{(OZ`xN+CsFB# z+VyOVH4&_FL;wK<5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** s5I_I{1Q0*~0R#|0;BN|~z1Q+L>ie7TE&nk3t$1NT<#*5edDmC@1j#3#{Qv*} literal 0 HcmV?d00001 diff --git a/.gradle/checksums/checksums.lock b/.gradle/checksums/checksums.lock new file mode 100644 index 0000000000000000000000000000000000000000..8adaf1c201b0d4720183021e1990da996429d551 GIT binary patch literal 17 ScmZQ}{cr5!Z_6Ic00jUeuL7+A literal 0 HcmV?d00001 diff --git a/.gradle/vcs-1/gc.properties b/.gradle/vcs-1/gc.properties new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java index c2401242d..1dc39356a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java @@ -76,6 +76,7 @@ public void run() { while (isRunning.get()) { consume(); } + LOG.info("consume dirty data end"); } catch (Exception e) { LOG.error("consume dirtyData error"); errorCount++; diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java index 0c90c770c..849363bbb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java @@ -29,6 +29,9 @@ import java.io.Serializable; import java.lang.reflect.Constructor; import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -39,7 +42,6 @@ * Date 2020/8/27 星期四 */ public class DirtyDataManager implements Serializable { - //TODO 需要确保产生脏数据后至少一次是成功发送给了consumer，至于consumer是否成功消费，manager不需要关心 private static final long serialVersionUID = 7190970299538893497L; private static final Logger LOG = LoggerFactory.getLogger(DirtyDataManager.class); @@ -70,7 +72,9 @@ public class DirtyDataManager implements Serializable { */ private final AtomicLong errorCount = new AtomicLong(0); - public static AbstractDirtyDataConsumer consumer; + public AbstractDirtyDataConsumer consumer; + + private final ExecutorService executor = Executors.newSingleThreadExecutor(); /** * 通过参数生成manager实例，并同时将consumer实例化 @@ -78,11 +82,9 @@ public class DirtyDataManager implements Serializable { public static DirtyDataManager newInstance(Map properties) throws Exception { DirtyDataManager manager = new DirtyDataManager(); manager.blockingInterval = Long.parseLong(properties.getOrDefault("blockingInterval", "60")); - consumer = createConsumer(properties); - consumer.init(properties); - //TODO 使用线程池创建线程，不要用thread - Thread dirtyDataConsumer = new Thread(consumer.setQueue(manager.queue), "dirtyData Consumer"); - dirtyDataConsumer.start(); + manager.consumer = createConsumer(properties); + manager.consumer.init(properties); + manager.executor.execute(manager.consumer); return manager; } @@ -105,12 +107,13 @@ private static AbstractDirtyDataConsumer createConsumer(Map prop /** * 脏数据收集任务停止，任务停止之前，需要将队列中所有的数据清空 */ - public void close() throws Exception { + public void close() { if (!queue.isEmpty() && checkConsumer()) { - consumer.consume(); + executor.shutdown(); } LOG.info("dirty consumer is closing ..."); - consumer.close(); + this.consumer.isRunning.compareAndSet(true, false); + executor.shutdownNow(); } /** @@ -134,7 +137,7 @@ public void collectDirtyData(String dataInfo, String cause, String field) { * 查看consumer当前状态 */ public boolean checkConsumer() { - return consumer.isRunning(); + return this.consumer.isRunning(); } /** diff --git a/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java b/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java index 54bc19b82..4c52ebe02 100644 --- a/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java +++ b/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java @@ -19,12 +19,15 @@ package com.dtstack.flink.sql.dirty.mysql; import com.dtstack.flink.sql.dirtyManager.consumer.AbstractDirtyDataConsumer; +import com.dtstack.flink.sql.dirtyManager.entity.DirtyDataEntity; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.util.Arrays; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; /** * @author tiezhu @@ -39,26 +42,27 @@ public class MysqlDirtyDataConsumer extends AbstractDirtyDataConsumer { private final Object LOCK_STR = new Object(); - private boolean isCreatedTable = false; - private final String[] tableField = {"id", "dirtyData", "processTime", "cause", "field"}; - private String SQL = "INSERT INTO ? (?, ?, ?, ?) VALUES (?, ?, ?, ?) "; + private final String SQL = "INSERT INTO ? (?, ?, ?, ?) VALUES (?, ?, ?, ?) "; private PreparedStatement statement; private Connection connection; private String tableName; + private final String defaultTable = "dirtyData_" + System.currentTimeMillis(); private void setStatement(String url, String userName, - String password) throws ClassNotFoundException, SQLException { + String password, + String tableName) throws ClassNotFoundException, SQLException { synchronized (LOCK_STR) { Class.forName(DRIVER_NAME); connection = DriverManager.getConnection(url, userName, password); statement = connection.prepareStatement(SQL); + statement.setString(1, tableName); } } @@ -71,11 +75,9 @@ private String quoteIdentifier(String tableName) { * * @param tableName 表名 * @return 是否创建成功 - * @throws SQLException SQL异常 */ private boolean createTable(String tableName) { try { - String defaultTable = ""; String sql = "CREATE TABLE ` " + tableName + "` (" + " `id` int(11) not null AUTO_INCREMENT,\n" + @@ -93,10 +95,8 @@ private boolean createTable(String tableName) { @Override public void consume() throws Exception { - if (!isCreatedTable) { - createTable(tableName); - } - + DirtyDataEntity dataEntity = queue.take(); + count++; } @Override @@ -109,6 +109,8 @@ public void close() { if (statement != null && !statement.isClosed()) { statement.close(); } + + isRunning.compareAndSet(true, false); } catch (SQLException e) { throw new RuntimeException("close mysql resource error !"); } @@ -116,11 +118,19 @@ public void close() { @Override public void init(Map properties) throws Exception { - tableName = properties.get("tableName"); + tableName = properties.get("tableName") == null ? defaultTable : properties.get("tableName"); String userName = properties.get("userName"); String password = properties.get("password"); String url = properties.get("url"); - isCreatedTable = Boolean.parseBoolean(properties.get("isCreatedTable")); - setStatement(url, userName, password); + + boolean isCreatedTable = Boolean.parseBoolean(properties.get("isCreatedTable")); + if (!isCreatedTable) { + if (!createTable(tableName)) { + throw new RuntimeException("create table for dirty Data error, please check privilege for database"); + } + LOG.info("create " + tableName + " succeed!"); + } + + setStatement(url, userName, password, tableName); } } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..f3d88b1c2faf2fc91d853cd5d4242b5547257070 GIT binary patch literal 58695 zcma&OV~}Oh(k5J8>Mq;vvTfV8ZQE5{wr$(iDciPf+tV}m-if*I+;_h3N1nY;M6TF7 zBc7A_WUgl&IY|&uNFbnJzkq;%`2QLZ5b*!{1OkHidzBVe;-?mu5upVElKVGD>pC88 zzP}E3wRHBgaO?2nzdZ5pL;m-xf&RU>buj(E-s=DK zf%>P9se`_emGS@673tqyT^;o8?2H}$uO&&u^TlmHfPgSSfPiTK^AZ7DTPH`Szw4#- z&21E&^c|dx9f;^@46XDX9itS+ZRYuqx#wG*>5Bs&gxwSQbj8grds#xkl;ikls1%(2 zR-`Tn(#9}E_aQ!zu~_iyc0gXp2I`O?erY?=JK{M`Ew(*RP3vy^0=b2E0^PSZgm(P6 z+U<&w#)I=>0z=IC4 zh4Q;eq94OGttUh7AGWu7m){;^Qk*5F6eTn+Ky$x>9Ntl~n0KDzFmB0lBI6?o!({iX zQt=|-9TPjAmCP!eA{r|^71cIvI(1#UCSzPw(L2>8OG0O_RQeJ{{MG)tLQ*aSX{AMS zP-;|nj+9{J&c9UV5Ww|#OE*Ah6?9WaR?B04N|#`m0G-IqwdN~Z{8)!$@UsK>l9H81 z?z`Z@`dWZEvuABvItgYLk-FA(u-$4mfW@2(Eh(9fe`5?WUda#wQa54 z3dXE&-*@lsrR~U#4NqkGM7Yu4#pfGqAmxmGr&Ep?&MwQ9?Z*twtODbi;vK|nQ~d_N z;T5Gtj_HZKu&oTfqQ~i`K!L||U1U=EfW@FzKSx!_`brOs#}9d(!Cu>cN51(FstP_2dJh>IHldL~vIwjZChS-*KcKk5Gz zyoiecAu;ImgF&DPrY6!68)9CM-S8*T5$damK&KdK4S6yg#i9%YBH>Yuw0f280eAv3 za@9e0+I>F}6&QZE5*T8$5__$L>39+GL+Q(}j71dS!_w%B5BdDS56%xX1~(pKYRjT; zbVy6V@Go&vbd_OzK^&!o{)$xIfnHbMJZMOo``vQfBpg7dzc^+&gfh7_=oxk5n(SO3 zr$pV6O0%ZXyK~yn++5#x`M^HzFb3N>Vb-4J%(TAy#3qjo2RzzD*|8Y} z7fEdoY5x9b3idE~-!45v?HQ$IQWc(c>@OZ>p*o&Om#YU904cMNGuEfV=7=&sEBWEO z0*!=GVSv0>d^i9z7Sg{z#So+GM2TEu7$KXJ6>)Bor8P5J(xrxgx+fTLn1?Jlotz*U z(ekS*a2*ml5ft&R;h3Gc2ndTElB!bdMa>UptgIl{pA+&b+z_Y&aS7SWUlwJf-+PRv z$#v|!SP92+41^ppe}~aariwztUtwKA8BBLa5=?j3@~qHfjxkvID8CD`t5*+4s|u4T zLJ9iEfhO4YuAl$)?VsWcln|?(P=CA|!u}ab3c3fL8ej9fW;K|@3-c@y4I;^8?K!i0 zS(5Cm#i85BGZov}qp+<-5!Fh+KZev3(sA2D_4Z~ZLmB5B$_Yw2aY{kA$zuzggbD{T zE>#yd3ilpjM4F^dmfW#p#*;@RgBg{!_3b6cW?^iYcP!mjj!}pkNi{2da-ZCD2TKKz zH^x^+YgBb=dtg@_(Cy33D|#IZ&8t?w8$E8P0fmX#GIzq~w51uYmFs{aY76e0_~z2M z(o%PNTIipeOIq(H5O>OJ*v8KZE>U@kw5(LkumNrY>Rv7BlW7{_R9v@N63rK)*tu|S zKzq|aNs@81YUVZ5vm>+pc42CDPwQa>oxrsXkRdowWP!w?=M(fn3y6frEV*;WwfUV$s31D!S_;_~E@MEZ>|~wmIr05#z2J+& zBme6rnxfCp&kP@sP)NwG>!#WqzG>KN7VC~Gdg493So%%-P%Rk!<|~-U|L3VASMj9K zk(Pfm1oj~>$A>MFFdAC8M&X0i9-cV7Q($(R5C&nR5RH$T&7M=pCDl`MpAHPOha!4r zQnYz$7B1iLK$>_Ai%kZQaj-9)nH$)tESWUSDGs2|7plF4cq1Oj-U|+l4Ga}>k!efC z*ecEudbliG+%wI8J#qI!s@t%0y9R$MBUFB)4d47VmI`FjtzNd_xit&l1T@drx z&4>Aj<2{1gUW8&EihwT1mZeliwrCN{R|4@w4@@Btov?x5ZVzrs&gF0n4jGSE33ddUnBg_nO4Zw)yB$J-{@a8 z);m%fvX2fvXxogriNb}}A8HxA)1P-oK+Da4C3pofK3>U_6%DsXFpPX}3F8O`uIpLn zdKjq(QxJTJ4xh->(=lxWO#^XAa~<7UxQl8~8=izS!TcPmAiBP5Et7y?qEbFd9Q=%IJ;%Kn$lto-~3`}&`x=AVS+Uo7N*hbUxhqVH_w^sn!74z{Ka#*U6s z=8jIrHpUMBC@@9Jn~GS<$lse*EKuX%3Swl5&3~GiK_$vn8Vjqe{mjhBlH}m4I8qK+ ztU50COh7)d-gXpq-|}T;biGa^e=VjxjjFuoGIA8`2jJ}wNBRcsx24?7lJ7W4ksNPv zA7|gcXT@~7KTID#0|EX#OAXvgaBJ8Jg!7X#kc1^Tvl;I(=~(jtn-(5bhB=~J^w5bw z8^Hifeupm;nwsSDkT{?x?E(DgLC~Nh8HKQGv`~2jMYrz9PwS^8qs3@nz4ZBCP5}%i z=w}jr2*$X-f(zDhu%D8(hWCpix>TQpi{e`-{p^y?x4?9%)^wWc?L}UMcfp~lL|;g) zmtkcXGi9#?cFOQQi_!Z8b;4R%4y{$SN~fkFedDJ&3eBfHg|DRSx09!tjoDHgD510Z z_aJLHdS&7;Dl;X|WBVyl_+d+2_MK07^X1JEi_)v$Z*ny-()VrD6VWx|Un{)gO0*FQ zX{8Ss3JMrV15zXyfCTsVO@hs49m&mN(QMdL3&x@uQqOyh2gnGJYocz0G=?BX7qxA{ zXe0bn4ij^;wfZfnRlIYkWS^usYI@goI9PccI>}Ih*B!%zv6P$DoXsS%?G)|HHevkG z>`b#vtP=Lx$Ee(t??%_+jh(nuc0Q&mCU{E3U z1NqNK!XOE#H2Pybjg0_tYz^bzX`^RR{F2ML^+<8Q{a;t(#&af8@c6K2y2m zP|parK=qf`I`#YxwL=NTP>tMiLR(d|<#gEu=L-c!r&(+CpSMB5ChYW1pUmTVdCWw|!Ao?j&-*~50S`=) z9#Knf7GPA19g%Y7wip@`nj$aJcV|SakXZ*Q2k$_SZlNMx!eY8exF;navr&R)?NO9k z#V&~KLZ0c9m|Mf4Gic}+<=w9YPlY@|Pw*z?70dwOtb<9-(0GOg>{sZaMkZc9DVk0r zKt%g5B1-8xj$Z)>tWK-Gl4{%XF55_Ra3}pSY<@Y&9mw`1jW8|&Zm{BmHt^g=FlE{` z9Lu7fI2v3_0u~apyA;wa|S4NaaG>eHEw&3lNFVd_R9E=Y? zgpVQxc9{drFt2pP#ZiN~(PL%9daP4pWd*5ABZYK{a@e&Vb`TYiLt$1S>KceK36Ehz z;;MI%V;I`#VoSVAgK3I%-c>ViA>nt=5EZ zjr$Jv~$_vg<$q<@CpZ1gdqP_3v^)uaqZ`?RS_>f(pWx3(H;gWpjR?W8L++YPW;)Vw3)~tozdySrB3A2;O<%1F8?Il4G|rO0mEZYHDz!?ke!$^bEiWRC1B%j~ws0+hHS;B8l5Wh)e+Ms7f4M4CbL%Q_*i~cP}5-B(UkE&f7*pW6OtYk5okQCEoN4v|7;(+~~nyViqo5 z(bMGQi$)KN6EmfVHv4pf2zZMJbcAKyYy>jY@>LB5eId|2Vsp{>NMlsee-tmh({;@b z@g;wiv8@a1qrDf-@7$(MR^M^*dKYBewhIDFX%;*8s zR#u?E;DJO;VnTY6IfbO=dQ61V0DisUAs4~t|9`9ZE(jG}ax#-xikDhsO_4^RaK ziZ?9AJQP_{9WuzVk^s_U+3V8gOvVl5(#1>}a|RL>};+uJB%nQM-J>M4~yK)cioytFXtnmOaJZSiE+3g}C`Im~6H z*+-vjI>ng5w>>Y!L(+DwX2gs0!&-BFEaDie4i5ln*NGP$te7$F9iUlJl4`XpkAsPm z0l?GQ17uN^=g~u1*$)S`30xL%!`LW*flwT*#svAtY(kHXFfvA`dj*pDfr0pBZ`!La zWmX$Z@qyv|{nNsRS|+CzN-Pvb>47HEDeUGFhpp5C_NL0Vp~{Wc{bsm_5J!#tuqW@? z)Be zb&Gj&(l*bHQDq7w-b`F9MHEH*{Dh~0`Gn8t`pz}!R+q~4u$T@cVaUu`E^%0f-q*hM z1To6V31UGJN7a-QW5;nhk#C26vmHyjTVZkdV zqYMI9jQY)3oZt=V0L7JZQ=^c2k){Y_lHp&V_LIi*iX^Ih3vZ_K<@Di(hY<&g^f?c$wwF-wX1VLj>ZC4{0#e`XhbL_$a9uXS zKph*4LupSV2TQBCJ4AfOXD8fs2;bAGz-qU4=Qj$^1ZJX z2TtaVdq>OjaWGvv9)agwV)QW9eTZ-xv`us2!yXSARnD5DwX_Vg*@g4w!-zT|5<}-7 zsnllGRQz>k!LwdU`|i&!Bw^W7CTUU3x`Zg8>XgHj=bo!cd<#pI8*pa*1N`gg~I0ace!wzZoJ)oGScm~D_Sc;#wFed zUo;-*0LaWVCC2yqr6IbeW3`hvXyMfAH94qP2|cN``Z%dSuz8HcQ!WT0k38!X34<6l zHtMV%4fH5<6z-lYcK;CTvzzT6-^xSP>~a*8LfbByHyp$|X*#I6HCAi){gCu1nvN%& zvlSbNFJRCc&8>f`$2Qa`fb@w!C11v1KCn)P9<}ei0}g*cl~9A9h=7(}FO!=cVllq3 z7nD)E%gt;&AYdo{Ljb2~Fm5jy{I><%i*GUlU8crR4k(zwQf#nima@xb%O71M#t-4< z(yjX(m^mp_Y;5()naqt2-VibylPS)Oof9uBp$3Gj`>7@gjKwnwRCc>rx%$esn);gI z5B9;~uz57n7Rpm8K^o=_sFPyU?>liHM&8&#O%f)}C5F7gvj#n#TLp@!M~Q?iW~lS}(gy%d&G3p?iBP z(PZQUv07@7!o3~1_l|m5m;Xr)^QK_JaVAY3v1UREC*6>v;AT$BO`nA~KZa1x3kV2F z%iwG7SaaAcT8kalCa^Hg&|eINWmBQA_d8$}B+-Q_@6j_{>a- zwT3CMWG!A}Ef$EvQsjK>o)lJ;q!~#F%wo`k-_mT=+yo%6+`iGe9(XeUl;*-4(`G;M zc@+ep^Xv&<3e7l4wt48iwaLIC1RhSsYrf6>7zXfVD zNNJ1#zM;CjKgfqCabzacX7#oEN{koCnq1-stV+-CMQ=ZX7Fpd*n9`+AEg9=p&q7mTAKXvcbo?$AVvOOp{F>#a;S?joYZl_f}BECS%u&0x!95DR;|QkR9i}`FEAsPb=)I z8nb=4iwjiLRgAF}8WTwAb^eA>QjL4Srqb#n zTwx^-*Z38Uzh@bX$_1tq>m{o8PBX*t3Lqaf$EBqiOU*2NFp{LJX#3}p9{|v{^Hg4f zlhllKI>F+>*%mu6i9V7TT*Wx-zdK z(p8faUOwGOm5mBC%UGA1jO0@IKkG;i&+6Ur8XR2ZuRb$*a}R^-H6eKxcYodlXsF`& z{NkO+;_Yh-Ni@vV9iyzM43Yibn;oC7hPAzC24zs&+RYdY&r`3&&fg2hs62ysV^G`N zHMfBEFo8E3S$0C_m({bL8QCe$B@M{n1dLsaJYIU;(!n*V?0I1OvBB=iYh&`?u8 z&~n-$nbVIhO3mMhCQRlq%XRr1;Hvl=9E_F0sc9!VLnM>@mY~=Cx3K5}wxHKEZF9pC zIdyu1qucM!gEiomw7bW0-RwbX7?o=FE#K0l4`U2KhC8*kMWaEWJyVNZVu_tY2e&4F zb54Lh=Oz>(3?V$!ArXFXh8Cb3i;%KQGCrW$W#;kvx$YA2gofNeu?@nt>Yq8?2uJQp zUTo14hS%&dHF3Uhm~Z1>W)yb%&HoM!3z?%a%dmKT#>}}kKy2B=V3{Nu=bae%V%wU$ zb4%^m?&qn==QeHo`nAs3H}wtiK~!!&i|iBLfazh6!y9F)ToKNyE0B385!zq{p)5vB zvu`R#ULIS|2{3w52c*c$4}Pe>9Fw&U^>Bb_LUWn!xPx3X-uQsv(b1XFvFzn#voq0* z5~o`V_G805QXdgAOwOjoqmZ?uzwBVYSNP0Ie8FL`P0VK1J4CzV@t&%0duHB{;yIL$FZ9 zz#s#%ZG6ya&AwE;0_~^$1K

Hnj76Oym1QVh(3qRgs)GmgnEt-KxP|nCFY3uezZn zmtR0CZ$Z_-+f07?lu_tr~IC{&U6+QOth>ZgYk4V2FI$B2V3`M`Jk zsr>>lupymPeK129PfpDt9?GA2;I>03Ktz8NxwvTroqu8oaRB&bXT}G=^2UyOW}(4H z;9sG^YwV8K7pC&&viM^X_pfeFoN!cIhrE>OPQ5E<4KKDyPhRV^BGb_^Y6GO6#w}c= zu`0fC-@F4qXQtnB^nPmfI7Uw0bLhY^09TCO+H2(nvg8jdPjMAi4oSX%GP3oeo0`ks z%DoV|waU-Q7_libJCwnnOL9~LoapKqFPpZx?5FygX zsA~*ZR7X=@i{smf?fgxbcY6Y`JvD50P=R;Xv^sANPRp-Hc8n~Wb*gLIaoZJ2Q^CFe z_=G}y&{_NXT|Ob??}$cF7)$oPQMaeN_va1f%>C>V2E01uDU=h~<_fQKjtnl_aho2i zmI|R9jrNdhtl+q*X@}>l08Izz&UJygYkbsqu?4OOclV{GI5h98vfszu2QPiF?{Tvh19u_-C^+NjdAq!tq&Rd`ejXw#` z@U15c$Nmylco)Yj4kctX{L+lz$&CqTT5~}Q>0r-Xe!m5+?du6R&XY|YD5r5C-k*`s zOq-NOg%}RJr5ZWV4)?EO%XzZg&e8qVFQ?40r=8BI-~L%9T7@_{1X@<7RjboXqMzsV z8FiSINMjV*vC^FCv_;`jdJ-{U1<_xjZg4g?ek z4FtsapW_vFGqiGcGHP%?8US~Dfqi8^ZqtHx!}0%dqZFg%nQB)8`mE$~;1)Fb76nFk z@rK#&>2@@)4vO&gb{9&~R8-_{8qz6Rmw`4zeckD(L9xq}{r(fUO0Zh-R(d#x{<0j| z?6xZ2sp3mWnC}40B~g2QinHs1CZqZH&`+x2yBLT8hF7oWNIs_#YK2cyHO6AoGRG|RM>Hyn(ddpXFPAOGh~^0zcat`%&WoEQf9)!@l*3Tt@m>Lb z6$+$c!zsy_=%L9!_;jfd`?VXDd*^Vn%G>n~V9Vr6+_D@#E+dWB#&zAE+6xJeDMr1j zV+Tp~ht!M%^6f?)LBf8U1O4G#CutR07SB>8C&_&;g3TdIR#~e~qRtwd>&)|-ztJJ#4y0|UMjhJZlS8gA zAA260zUh+!$+xMfWKs|Lr23bcy#)JNnY|?WOka&wTS7_u%*N7PrMl1Lp9gxJY%CF? zz4IA@VVxX{knZPlNF+$9)>YIj#+(|$aflt=Wnforgn6`^3T+vaMmbshBjDi&tR(a7 zky~xCa77poRXPPam)@_UCwPdha^X~Aum=c0I@yTyD&Z!3pkA7LKr%Y6g%;~0<`{2& zS7W$AY$Kd}3Tg9CJgx=_gKR59zTMROsos?PU6&ocyCwCs8Qx1R%2#!&5c%~B+APu( z<1EXfahbm{XtOBK%@2a3&!cJ6R^g|2iLIN1)C2|l=;uj%tgSHoq2ojec6_4@6b<8BYG1h-Pm_V6dkRB!{T?jwVIIj&;~b7#%5Ew=0Fx zc(p7D1TT&e=hVt4spli}{J6tJ^}WL>sb`k}&gz+6It`Yz6dZdI53%$TR6!kSK2CfT*Q$`P30 z;$+G$D*C$U(^kkeY!OWn$j@IUu0_a{bZQ=TCbHD1EtmZ0-IBR<_3=tT%cz$>EE!V}pvfn7EMWs^971+XK}~kxSc_ATJJD$?)1Gz^Jq!>Hz#KkdCJ~jb-Y*Xv01_}}=T_V-A1<3O!V9Ezf z%Lnjihb3>=ZV}jSeqNu5AAdVbe|`;|p<%W#-<$s1oDYrB;C({psqV>ENkhadsC{cfEx=teVSB`?FOs+}d#pssxP z(ihudAVu3%%!*vOIWY11fn1M0&W|(|<2lEShz|#%W|wV2qM%#+P9NOy1x8jytHpfU zh;_L^uiL<<$L@~NpRXSrkJgdC>9R=>FmVu3^#C?3H>P{ue=mcv7lBmnfA?mB|L)EF zHv%Nl|D}0Tb~JVnv$ZysvbD8zw)>|5NpW3foe!QHipV9>Zy`|<5?O+rsBr*nZ4OE} zUytv%Rw7>^moSMsSU?@&a9+OdVgzWZnD>QXcUd{dd7vad+=0Hy)4|0A`}rpCx6cu!Ee5AM=iJ?|6=pG^>q(ExotyZP3(2PGhgg6-FkkQHS?nHX(yU0NG;4foCV|&)7 z1YK!bnv%#5n<25|CZ>4r1nK=D39qMzLAja*^#CN(aBbMx${?Iur3t=g2EMK|KwOF?I@W~0y`al&TGqJ zwf#~(?!>@#|JbDjQV9ct%+51l%q|lcY&f{FV&ACRVW*%VY6G5DzTpC!e%=T30mvav zRk$JOTntNoxRv>PDlJG1X=uep&???K00ep|l_#7=YZPuRHYoM46Z$O=ZZuGy_njgC z>P@gd+zKH5SjpWQ!h_r*!ol1s{9DS@sD4}xgFxaw>|av!xrKzg?rGnhZ#uZeU~iod z3-i*Hl@7cge0);y{DCVU(Ni1zg{yE&CxYT7)@zJ%ZZABj-Fh}0au^)*aw`vpmym;( z5|JZ!EACYenKNXH%=Md{my$sI3!8^FgtqkMcUR%w_)EBdP5DZ64aCIR%K99tId6SU ziT8Ef)K%7{XuIpPi}N+&FCm$elE>oKY;3c$x+*mXy?~wt6~?ss$HGqCm=YL2xzVTQ zr>*2_F;7j{5}NUPQ(aY0+h~rOKN|IA28L7^4XjX!L0C^vFB+3R5*1+s@k7;4d#U=5 zXTy8JN^_BCx1a4O3HMa9rf@?Fz>>dq}uvkY7!c?oksgs~xrpCo1{}^PD?w}Ug z3MbfBtRi z$ze~eRSLW^6bDJJeAt^5El{T*i1*v9wX{T7`a2wAVA z%j>3m*g^lc*~GOHFNy?h7>f7mPU*)3J>yPosaGkok}2#?wX5d$9moM~{NTzLznVhX zKa}bFQt#De`atoWzj4Lb@ZCud_T9rA@6VcmvW(+X?oIaH-FDbEg#0Slwf|7f!zUO( z7EUzpBOODL&w~(tNt0z|<9}Filev&4y;SQPp+?kIvJgnpc!^eYmsWz1)^n`LmP&Ui z-Oi1J2&O|$I<^V@g2Z91l3OArSbCkYAD0Tuw-O(INJJ>t%`DfIj}6%zmO+=-L{b!P zLRKvZHBT=^`60YuZon~D$;8UDlb-5l8J=1erf$H(r~ryWFN)+yY@a;=CjeUGNmexR zN)@)xaHmyp$SJcl>9)buKst5_+XomJu34&QMyS zQR(N@C$@%EmfWB8dFN(@Z%xmRma@>QU}!{3=E`wrRCQ~W=Dwb}*CW8KxAJ;v@TAs3 zW}Pq5JPc)(C8Rths1LR}Bgcf6dPOX<#X08^QHkznM-S>6YF(siF;pf~!@)O{KR4q1_c`T9gxSEf`_;a-=bg6=8W zQ&t`BK^gsK-E0Jp{^gW&8F9k?L4<#}Y0icYT2r+Dvg!bnY;lNNCj_3=N=yd9cM9kY zLFg|R0X;NRMY%zD*DbAmFV`(V@IANtz4^_32CH*)XCc$A>P-v49$k@!o$8%Ug>3-- z$#Fpo9J>eUMKg>Cn+T0H!n0Hf#avZX4pp54cv}YcutP+CmKC~a745-zhZp`KNms;J zS3S49WEyS8gCRAY|B~6yDh*cehY52jOSA#MZmk2dzu`_XpBXx9jDf!H3~!`n zaGe=)1VkfIz?*$T3t>-Pwhrw447idZxrsi;ks;(NF>uVl12}zI(N~2Gxi)8yDv-TLgbZ;L&{ax&TBv;m@z6RcbakF^el{!&)<___n#_|XR%jedxzfXG!a2Eyi)4g zYAWkYK{bQzhm|=>4+*SLTG2<#7g-{oB48b05=?PeW;Jo3ebWlo5y5|cl?p8)~PVZqiT^A~w-V*st8kV%%Et1(}x(mE0br-#hyPspVehofF`{gjFXla1lrqXJqQKE9M)8Xe0ZO&s$}Q zBTPjH>N!UU%bRFqaX(O9KMoG$Zy|xt-kCDjz(E*VDaI={%q? zURR{qi>G^wNteX|?&ZfhK-93KZlPXmGMsPd1o?*f_ej~TkoQ#no}~&#{O=>RadgtR zvig@~IZMsm3)vOr`>TGKD&fbRoB*0xhK7|R?Jh-NzkmR}H6lJiAZTIM1#AXE1LOGx zm7j;4b(Lu6d6GwtnsCvImB8%KJD+8z?W{_bDEB$ulcKP*v;c z*Ymsd)aP+t$dAfC-XnbwDx3HXKrB{91~O}OBx)fsb{s-qXkY<@QK7p-q-aaX&F?GS z2};`CqoNJ$<0DuM2!NCbtIpJ9*1a8?PH#bnF#xf~AYOIc4dx1Bw@K=)9bRX;ehYs; z$_=Ro(1!iIM=kZDlHFB>Ef46#rUwLM%)(#oAG(gYp>0tc##V{#aBl!q``!iIe1GBn z+6^G^5)(nr z8h#bm1ZzI450T?!EL)>RWX8VwT1X`2f;dW!{b~S>#$Pa~D6#Hp!;85XzluH%v5325 z730-aW?rY1!EAt;j7d23qfbMEyRZqxP};uID8xmG@mGw~3#2T^B~~14K5?&dP&H@r zL|aXJsEcAAXEXfu2d-!otZTV=if~^EQD*!NkUFQaheV&b-?-zH6JfjKO)aYN=Do*5 zYZ-@m#)5U0c&sUqu_%-Editr5#%Ne&bs)DxOj2_}`f;I_ReEY9U&Cf3rb>A3LK(ZD zid0_-3RfsS*t&g!zw}C_9u(_ze-vc1L59CdBl(IS^yrvsksfvjXfm>(lcol%L3))Q z@ZT;aumO3Q#8R!-)U697NBM@11jQ>lWBPs#?M4_(w=V_73rsiZh8awEm>q1phn1Ks ze@D|zskeome3uilE8-dgG(EojlI(@Yhfm}Xh_AgueHV`SL##I@?VR+bEHH=sh21A_ zhs&pIN7YTLcmJiyf4lZ;`?pN0`8@QbzDpmT`$m0CTrTMiCq%dE&Cd_{-h`I~f8Kps zAuZt4z)}@T>w$9V@iLi=mh({yiCl}}d>JN)z;*G<6&mgl(CYhJHCAPl=PYK2D>*F zy;YK=xS@1JW7i=C)T04(2P#|fowalY=`Y`G8?eRMAKt|ddG9UF^0M5 zW=ZGZ5qb-z@}iS`4RKXvuPIfzUHT)rv<8a|b?bgB3n=ziCiX4m2~CdVBKHWxw2+Hz zLvqoAij9(0moKoo2$`dqS0?5-(?^RXfcsQB6hU2SAgq8wyeasuyFGcK+@An?8ZzVw zW8wwbZB@i=<<4fA7JKPkki6y>>qO3_bW>-uQ*>9g+g7M0U^`RV)YTrGu2Q=2K>fiI zY0dFs>+}xuOZE^efLK2K6&X@>+y10Oqejnnq^NjfXt9JpK4K_E=cl29 z(t2P;kl4AK_Jg9v{1(z)ESpyo_(Z`74D&J1A#J?l5&J^Ad1sm5;Po@s9v7wOs(=_T zkutjt`BaxT09G{-r>yzyKLlM(k`GZl5m+Tgvq=IN|VjtJ*Zu66@#Rw;qdfZqi15A@fr^vz?071F5!T`s>Lx5!TszI%UK|7dDU;rUCwrRcLh!TZZ9$UMfo z@Qzjw>tKS3&-pyWS^p4mMtx`AvwxVc?g?#8aj@jQ#YKDG0aCx{pU+36?ctAiz=f$k z05S(b&VPQgA(Sm`oP&M^eiHvBe&PcTb+j$!!Yx(j3iI5zcQLOn(QqfX5OElbSsQBUw7);5C92onieJyx`p{V!iwXk)+1v zA6vStRZo0hc>m5yz-pkby#9`iG5+qJ{x>6I@qeAK zSBFylj8{FU*0YbFd2FZ6zdt^2p?V;3F~kap`UQgf@}c33+6xP)hK)fmDo@mm=`47* z9S6rnwCSL&aqgZs959!lhEZZp`*>V8ifNmL;cqajMuaJ~t`;jLPB?X~Ylk_Z#Q;%} zV+sAJ=4505-DdnIR=@D_a`Gy#RxtSX+i-zInO@LVDOd*p>M-|X(qRrZ3S(>(=Oj>} z89d75&n?m^j>;SOXM=)vNoum|3YmzxjYx%^AU*V|5v@SjBYtESp^yz?eQ#>5pnCj} zJ_WCw23wGd2AA-iBve8Hq8`%B3K4@9q@a}sf$49IA^IPsX@QK)36mrzqOv?R_n9K@ zw3=^_m#j{gNR0;&+F~wlS(i8IQN8mIvIO)mkx|e)u*y+xDie}%mkZ*m)BQM^$R@-g z1FrP0{8A?EcxtxxxX&J;393ljwwG?2A2?y-1M0-tw$?5ssoEsbPi?sd2!s~TrwPLF zYo-5XYV7AU-c|Vb-v;>pVi^CwX(Rpt<9{Ic?@<9SrNu>F(gwij%?dC9^!Xo90o1-| z&_aPKo%+xyw64e&v<}F^-7sO0Cz-VOF@7**i@v&(Oy4Q8PbV+4&rKwmYyokM z48OZ|^%*mC_Q)RJ31D#b4o4Jzr{~BX4D#swW<31;qCil2qlim;e=9ymJAEXfv-|h3 z)>uqQ5~S+8IgiWW28Fqbq+@ukCLy+k7eGa1i5#G_tAUquw$FjFvQt6~kWa69KXvAj z-knF`5yWMEJvCbTX!K{L)VeNF?(+s?eNjtE5ivg^-#937-l()2nKr#cHShB&Pl^l8 zVYws26D^7nXPlm<_DYU{iDS>6Bq0@QsN%6n>XHVvP<^rDWscC!c+LFrK#)T@$%_0{ zob%f&oaq>1_Z8Ata@Y2K6n?GYg|l8SgUr(}hi4D!@KL~hjRv<}ZZ`tCD^ev=H&^0pP%6q2e+t=Ua`ag8xqWvNnIvCU|6ZA^L5v{DD)!mcQ@n6{=; z#Z)PrAz>*+h-|IV!&J*f@{xb!L7h3{?FEs*ifw5z2U9$&OkYseI68yb=V4xv*VK3- zVxGhtmedujX32y-kC{5ej-Wy#JvB~4oxTb{|1H825_B(A0#?CjUTc=PrGh6jAgK9h zoLAe`+NBdStZE@Y8UH^Rd*|R-|7Ke}wr$(CZQHhO+upHlCp)%n+fH_}S8%^%xqhu%20_1p=x#Dl9ia`c3iM+9Vh5?gyY8M9c$tJ5>}V_sidHN zoMl%rSgSK!7+Y8tQkYq|;Vh`4by2uMsUfnxkk2{S@a>V#d}fv}Yud*>paVi_~T zU!GoYwWbnG%92!Cte(zhZX-i9#KJ;b{$(aZs|{MerP#6||UUx$=y)4XOb zihyKn`_QhJ#~@_peJ*8yD4>I7wQyKkZG%#FTKZfb(@G+9x7-3@hG}+ZC&$7DwbaB$ zC)jLj7yituY&WpOWlG7Z4Tuxzdwo6k!3lgwhh7BYMyB? zO9Q5nvn77~g~c623b`Pe5efNzYD#2Sfmg>aMB5s?4NC|-0pIXy%%`J;+E{(irb!Szc8M8A@!}0zqJLoG4SJ5$~1*yRo0^Z`uObA+= zV?1sYNvzvWbP%AsMzoIo3Cwx~y%i8rHF(BgLS>tH5Ab|1wp$X_3o2_VB(pFxgQ5QQ zk@)Vy95$b%HVf4@ppX(wrv^Jwfrsu+9N_OUm}nD7Ch_7STj66EYsZR#`9k|Tf^@p& ziHwnO$p{TB#R(Q{Os>Un~0!r$JO zLZ&F%SP|%$TuG)mFeOhKr1?S!aa0jTV$2XIeZb_fgO&n{8HTe9s`L&(tKoy?OaS^$ zLHNrgYgq920EI~M>LyU7gK70$7*`nFKD^d>MoEAhsBU0%@*RW@%T(J z?+wVbz=mcN%4#7qlCpl_^Ay7VB%?+uW1WSNnQOj^tALyqTpV zkEN2C;qO_W)MYl^Ow5I;t3;z#iG82F(qe}#QeE;AjA=wM==dB(Gu+ez*5|RVxO4}l zt`o?*B;);-0`vR(#+Q^L4WH_9wklh-S-L-_zd%Q0LZ%|H5=>Z)-x#Z+m%p&6$2ScV zEBneIGo)r0oT)xjze*Q~AIqhB%lOM5Id}^eKwS!?b_;B&TouZsemyL&y`)#FX}ZKp zp)ZnB*^)1P@2bCoe+Z|#KhTBNrT)UN@WIuudw})fwHl)re1|b~E1F=xpH?7L77p>5 zei$aD@KO0<+zo1<&7OuZatNsPq24Whu%0jD_ z$ZZy6MzayYgTJulNEy8D$F%JDYgx|d6{6kpDg#s170<15bM#4tzvrDU$6bvu-hH@6 zgcjq&3aR3k(23$FaUA|iuoy*bO{2F6W0<+ZdsYvXjc?d@ZT8kM!GD}r@qr;TF@0Hb z2Dz-A!HZ$-qJ?F%w6_`t`8xk$f$MNBfjqwvJiVdD+pf7NVFGh?O=qp2vh%UcYvc{rFldib~rkIlo`seU%pO_6hmBWGMcUhsBSWiQYYPMX<-Cjp49@7U==iS57bG zw3T9Nbm`)m9<<4e$U74`t~zRo0JSfi}=GdQXGLLPyW zlT^I}y=t$j{Vx!wN^z8X4l0|@RNrC#)G>bK)7IT7Qop>YdS^NnI3gfP>vtp)pXkr2WSVcAAv8uN>@ z`6)kICvNYU$DA8pnkl4sQopDC6<_M8zGJ^@ANXJL(yd#n1XFj9pH;rld*gwY8om_I zdB55w@FUQ_2k}d%HtQsmUx_7Mzftky&o2X2yDQrgGcehmrDDDtUJj5``AX$gzEbMc zUj2Qzp)Lo>y-O*@HJ|g9$GR2-jgjKfB68J6OlIg;4F2@2?FlW zqj|lO7A2Ts-Kd!SO|r9XLbPt_B~pBpF40xcr0h=a&$bg(cwjp>v%d~Uk-7GUWom?1 z92p+C0~)Og*-N~daT#gQdG{&dPRZso(#{jGeDb1G`N)^nFSB`{2-UQ&!fkPyK`m03 z_Di94`{-(%3nE4}7;4MZ)Pmawf#{}lyTSs5f(r;r1Dp4<;27K=F}Oga^VsUs3*NIn zOsYstpqpRF&rq^9>m50LRORj>=;{CV2&#C$-{M5{oY9biBSoQyXvugVcwyT-19S;pf!`GSNqb4**TI%Y z*zyV)XN3Fdp3RNNr9FU+cV*tt?4L8>D@kJp^rkf_rJ~DPYL}oJngd1^l!4ITQN`0RTT^iq4xMg|S6;d}lznE$Ip^8pW-CHu zP*^!U>Lcd3*shqa)pswq;y<|ISM1g1RG#`|MSPNAsw*XH1IAD(e(Kgqp6aDHgv>fI z!P67$z{#()Pdo3;4dUoy*Xor(O?+YTRPe=g*FfRj*9q9!8p%1l>g3e^rQ_nm{(@4t z?^nMDC2J8@my5q0QyCljCSp_@)No+6bZ*y)lSdrkLFcR6YOHu*vZ-q(C);5$MmM_z z1WT>Gc8g%`Rt~6*!}JhWi0=Rc_z5c8GR9YXW+cdoK~Ea(@wyXf|89HagNuFAO-V7k zUb|9zaCCWH3^Fz(m7$8K$|0ZOP!SNpgP!ql<)!z8w$Z$?9gq2f<~koe3|zD=imLfD z>IV5?SkRZ;7JlOG%z%Tlze$GXr0A}ResyF63ZGZVDLv2k4HWtoqoCaq+Z&GaVKuLA z>@zhNjYYc=sexH?;DTe4&2vnQE}C@UFo&|qcLddvH0FwswdRUc(p*X&IT^Zu>xLpG zn(@C%3ig(l2ZPm#Fc){+0b+%O7nt4zbOt+3@GQVm|1t70=-U(>yo3VY2`FnXFHUyi zwiqf(akt0kEE5_Pa-a*VCS}Pi6?`~P%bvX6UT~r-tUAY%I4XF3^nC+tf3alyL{M`w zv?aVQ#usdwpZmkrfv19O39}tQPQM+oY**a{X?@3Qe>r$+G!>r#?Id&U&m^HU(f= zjVpSi9M||1FyNQA&PO`*94&(qTTMQv3-z`bpCXs-3bX}#Ovqec<>omYhB*VrwxqjY zF3#OXFsj`h#G?F}UAilxTQ|78-edHc-Uc-LHaH*Y(K%R#dVw>_gz}kRD4s#+U&Pq= zps)kMf_t9`GHR7CO4zI8WVj0%qiSqy50N{e_5o#GrvNhMpJf5_sCPrEa%a@ltFnss ziaWh26vEW4fQp}qa4oP(l4xIMpA)~VHD9!lP%;Tm`(HD$jYMM-5Ag>S(gC35J35$%?^gk(r|`4Ewi-W z;f&;B*fO=kC@N=r<-#nGW|yXE;`zb0Y3TJOAkw1a$SQgoTawHZTck+V%T=spmP`^BHihc(jc+S1ObX%6AYQ6LVVc+BfM*P{2s0T2z zVIs*5{ql%#CKAzv0?@S+%||z;`dpfj0Y(VtA51n$j%sG5I%A|h98VU}PkVZFrk1*G zaw75v3(N50lanvr&ND4=7Db;HS4fpi)2vTME7aD2-8N5+kcOXmYCrLE?*5&dWhvB` zbD5)ADuIwwpS*Ms;1qyns(8&tZ*)0*&_lNa`_(phwqkL}h#WdX_ zyKg%+7vP>*&Fus9E4SqIN*Ms`QLB(YOnJ|md%U|X`r#tVN$#q6nEH1|blQ?9e(3|3 z`i#;GUl~v?I6&I6%YvkvmR?*l%&z)Pv8irzVQsWrZSr%aoYuPJa#EjK|4NmiuswK= zlKP2v&;yXv3>LQ$P){aYWrb)5GICwbj;ygw>*amKP;Z{xb^cF}O@IeQ^hB-OjEK{l z>#PNyLuVkeDroL9SK2*ChHmJJSkv@YRn7)E49fy!3tqhq`HtHs_(DK|2Lyv(%9L&f zSy+H}Uk{nE2^5h7zN7;{tP3)$1GK9Xcv^L48Sodg0}ZST@}x607yJo2O*XCfs7*wT@d?G^Q6QQRb!kVn?}iZLUVoyh8M4A^ElaHD*Nn2= zkfCS=(Bg9-Mck6K{ z%ZM59Rs4(j1tSG1B#wS=$kQfXSvw6V>A(IC@>F;5RrCos`N{>Oyg|o*qR2EJ>5Gpe ze~a4CB{mmDXC7C>uS@VL&t%X#&4k<`nDx;Zjmo%?A4fV3KOhBr;VuO!cvM8s2;pG5 zcAs!j?nshFQhNA`G3HMS z?8bfRyy1LwSYktu+I7Hurb-AIU9r|rl5nMd!S&!()6xYNJ1EqJd9BkjgDH@F*! zzjtj4ezywvlkV7X@dG^oOB}T76eK=y!YZB#53LhYsZuP&HdmVL>6kH8&xwa zxv8;t-AE>D5K<{`-({E0O4%fGiLVI8#GfZ0aXR6SfYiPUJKnujMoTI5El<1ZO9w|u zS3lJFx<7XUoUD(@)$pDcs3taMb*(v2yj#G)=Mz-1M1q@Tf4o{s9}Uj9Yo?8refJwV zJ;b+7kf0M}fluzHHHS!Ph8MGJxJNks7C$58^EmlaJcp`5nx+O7?J)4}1!Y>-GHf9o zk}oTyPa>+YC$)(Qm8|MhEWbj?XEq}R=0NFH@F3ymW>&KS!e&k5*05>V@O*~my_Th; zlP05~S5@q+XG>0EuSH!~gZe_@5Dbj}oNIiPJpEOip+3l!gyze@%qOkmjmx=?FWJLF zj?b}f8Vet*yYd16KmM43rVfZo?rz3u|L6Foi*GQe4+{REUv9*}d?%a{%=8|i;I!aT z7Wxm}QJC`?cEt9+$@kSkB!@`TKZz1|yrA1^*7geq zD5Kx-zf|pvWA+8s$egLrb=kY385v2WCGL{y4I15NCz5NMnyXP_^@rsP#LN$%`2+AL zJaUyV<5;B^7f+pLzTN50Z~6KC0WI<|#bMfv+JiP3RTN^2!a7*oi+@v3w*sm5#|7zz zosF*{&;fHBXn2@uguQ1IDsh(oJzH#i4%pk;Qh^T zfQLyOW;E*NqU!Fki*f-T4j(?C$lY2CT{e!uW}8E(evb3!S%>v^NtNy@BTYAD;DkVo zn9ehVGaO7s?PQBP{p%b#orGi6Y&~<;D%XLWdUi}`Nu-(U$wBBTt*|N4##sm2JSuWc)TRoYg57cM*VDGj~ka<=&JF zo8=4>Z8F`wA?AUHtoi$_hHoK!3v?l*P0$g^yipOWlcex4?N2?Ewb1U=lu}0`QICA4 zef61j-^1p}hkA*0_(esa!p%dX6%-1e-eMfQsIp6wRgtE=6=hDe`&jel{y=6x5;78s z?5^{J|t!#x1aS8<3C`v%E%u{*wZwSXr$0Owl5_ zmXh>D>C_SjOCL^CyGZpBpM5`eymt{*rf~9`%F&&o7*S!H%3X)7~QFgn^J>6 zD+yV}u{HN-x9*_$R;a+k?4k*1f)rE~K|QvcC3dlr>!nftB?gE-cfcPMj&9mRl>|Lg zQyCe|&SuZopU0>IfRmcV3^_mhueN5oQ=J+H4%UsSIum4r4!`^DJqZr?1j3BU)Ttzg z6LwM)W&UEMIe*H2T6|{rQ;x9qGbp7ca#-!Egm4|ECNTMN);`>2Q&%|BpOdIJ4l|fp zk!qEhl;n(Y7~R1YNt7FnY10bQZXRna2X`E_D1f*}v1bW^lJorDD0_p2Rkr32n}hY! zCDB(t$)4YOd)97R60gfg3|wrlsVs#4=poh4JS7Ykg$H)vE#B|YFrxU-$Ae^~62e;! zK9mwxK?dV4(|0_sv(zY&mzkf{x@!T8@}Z6Bf)#sfGy#XyRS1{$Bl(6&+db=>uy-@y z$Eq~9fYX$06>PSKAs#|7RqJ3GFb;@(^e`jpo-14%^{|%}&|6h{CD(w@8(bu-m=dVl zoWmYtxTjwKlI!^nwJ}^+ql`&fE#pcj*3I|_Z>#y##e@AvnlSN4po#4N#}WT)V5oNP zkG+h_Yb=fB$)i`e2Fd28kS$;$*_sI;o0Xoj#uVAtsB6CjX&|;Bk}HzQ*hJ!HDQ&qZ z^qf{}c`l^h5sg-i(pEg#_9aW(yTi?#WH=48?2Hfl_X+(SfW)_c48bG5Bf+MDNp>Y#Mpil%{IzCXD&azAq4&1U10=$#ETJzev$)C*S;Pr9papU3OabRQk_toRZ!Ge(4-=Ki8Db?eSBq~ZT#ufL6SKaXZ+9rA~ zQwyTQTI7*NXOhn?^$QOU>Y6PyCFP|pg;wi8VZ5Z$)7+(I_9cy--(;T#c9SO;Hk~|_ z0tEQ)?geu8C(E$>e1wy%f@o;Ar2e#3HZP$I#+9ar9bDa(RUOA+y!oB;NEBQ`VMb@_ zLFj{syU4mN%9GF;zCwNbx@^)jkv$|vFtbtbi7_odG)9s=q(-PtOnIVcwy(FxnEZm&O^y`vwRfhB z7Urcums9SQS6(swAgl?S|WDGUTFQu51yG$8069U zviuZ=@J&7tQ8DZG<(a->RzV+sUrmH$WG+QvZmUJhT*IoR3#3{ugW%XG0s?_ycS6V6 zS)019<_Rl@DN~8K4#w3g_lvRm4mK3&jmI$mwROr0>D`mX+228Dw4r;mvx7df zy~$zP8NjVX?xkGFaV>|BLuXMQ+BN+MMrIB4S6X)p&5l$;6=S8oI9qi&1iQbs?TroDMfCmIeJ}pbVVtVqHhS(zutEy6#UjTk29-+3@W0`KfehW`@np zhhu#)O&g%r)hTj4b$CY41NYp_)7!bYyG;v(rts z^}YDJt2W88H^H;e$LSm3dh=~yi@)mzJtEfW8=4avbeOE&;Oc>-6OHO+MW`XBZ4rO6 zS;nAi**w3Yso4&Ty+8f$uvT?Z)eaLe$KW1I~9YM2zeTIT}C%_G6FPH-s5Wi3r`=I&juGTfl zZ;4qFZV|6V0c&>t!Y>mvGx#1WWL0N5evV=u28K9**dv`}U3tJ$W?>3InXiwyc)SA% zcnH}(zb0@&wmE>J07n#DOs7~lw>5qUY0(JDQszC~KAAM}Bmd-2tGIzUpO@|yGBrJyXGJk3d+7 zJBN0$?Se(rEb0-z2m%CBd;~_4aH04%9UnSc4KP!FDAM5F_EFujJZ!KDR-fn181GX` z8A?8BUYV}D9bCE0eV~M>9SPag%iVCLWOYQJDzC4~B~Ct0{H7x|kOmVcTQ;esvyHJC zi$H0R73Z8+Z!9^3|2tNut#&MVKbm`8?65s)UM8rg6uE(|e^DYqvoc15-f;u8c=>3;Viz*T# zN%!T+Hex0>>_gUKs%+lgY9jo6CnxL6qnQ>C*RseLWRpipqI;AQE7;LUwL`zM%b`Vu z%Sa-+?a#+=)HaD|k2%_(b;pHRF96(c;QyPl6XHL8IqGQKC$M8R=US-c8;hUe?LKo&l!{V)8d&55sUXEu z5uITcO~`ipddh+Nr{7ibp^Wd{bU)^3##<5`lkuqfckxEU*9{pgNpTB2=ku1c-|3dK z|LIQF=ld@I7swq^4|G1VA}BK85&>2p#*P95W`I1FF(8G9vfNJ6MoN$+C^M89u!X=< zJSS%l?Qj>$J%9?0#0&S6#*h*(-9Z$}q*G#hP?cX7cAvM0eiVFhJJ~$`iZM!N5NhDb zi<1u_m#?jzpIaOe7h|Kiap#mHA`L|)ATnPJ7du{^ybuNx@1jA+V1l8ux#{LJ#teM(6=%gZcMq24J$2p z`wcC!qRssmwUv4H6Psw{(YdDNOv$!sq&O1SvIS}fCKZa+`T=Ayt@uZjQqEC{@Uj+| z!;i3W+p~=@fqEEhW@gT^JtCR<`m`i|Htg<TSJ&v`p;55ed zt@a|)70mq;#RP@=%76*iz>fAr7FKd|X8*@?9sWOFf$gbH$XFG zcUNu#=_+ovUd>FW*twO`+NSo*bcea=nbQ_gu^C7iR*dZtYbMkXL5mB@4a3@0wnwH! z(fZKLy+yfQRd%}-!aPC z4GB%OvPHXl(^H(BwVr6u6s=I;`SHQ1um7GPCdP-BjO%OQUH!_UKbEGvHCY}{OL`8FU$GZ;Y$SlS$-0VjK%lCP?U0shcadt4x7lN4%V}wBrLEbiEcK-OHl+pcBNSqN#mftpRj2A4Q z+av@-<#t_Dj_FN^O2~wq(ij1O*+=RVl+6gNV^~CI1UED- zn^zN@UOq8?q58b^4RA>lV}x;jA2OE=SqMYV9P#RsUlI+pp!y*jpwHgp-w3i$V)%?L z>irn1pnRc|P@r|Z0pCeMZ*k$}$`1GVGCT&QtJ`V%Mq!TXoge?8Fjn$bz}NqDn*2ZQ z$p3@F_^(}IVS76>OLNzs`O5!pF=LZ$<&gyuM$HQzHx8ww^FVxnP%Yv2i=m*1ASF~~ zP=!H}b`xl`k0pL5byku2QOS~!_1po!6vQyQL#LQ#rIRr?G5^W?yuNvw-PP{}%m35i$i+I?DJ%RGRcqekT#X~CxOjkV1UQrd&m_bbJ+gsSGbPwKS{F& zU-`QNw!*yq#Co#{)2JvP-6>lY$J$2u+e=r0&kEc#j#jh@4Tp;l*s<28wU%r= zezVPG^r*a?&Fn_(M|A7^xTPD998E-)-A4agNwT?=>FbrHz8w~w?hWBeHVYM()|buJ zvGv4j<%!U_Rh^ZKi~2(h1vk-?o9;`*Zc}m5#o@a1ncp)}rO2SDD9y!nT$_Eb%h`>% zDmssJ8Dl=gDn<-7Ug$~nTaRzd?CJh;?}nCco$7Pz<#J8;YL40#VFbAG|4nA$co;l^byBOT2Ki@gAO!{xU7-TY|rujdYTaWV(Rr{Jwu?(_TA zDR1|~ExJBfJ?MAReMF47u!oEw>JHVREmROknZUs2>yaboEyVs$Pg1f6vs06gCQp$b z?##4PWI#BxjCAVl>46V_dm4?uw=Y@h#}ER4|ACU{lddiweg`vq>gmB25`XuhNai1- zjt{?&%;TRFE+2Y_Gn;p^&&|bU44M=`9!Mc%NbHv|2E4!2+dUL z>6be$Kh|Duz}+)(R7WXsh!m`+#t^Its($x`pqDaN-^E z?*a=0Ck^rZBLQV~jY-SBliN&7%-y3s@FB;X)z(t&D=~@U0vT%xfcu`Lix=W#WVE{{ z2=C~L$>`~@JCIg8RAyk= zYG`(@w4H95n0@Fqv16~nlDU!+QZw&#w@K)hv!V>zA!ZOL$1Iykd&Su3rEln@(gxO| zxWc++T-rQEIL+j7i`TeatMfp4z7Ir31(TE4+_Ds@M|-+cwQg(z>s=S}gsSz{X*Wm+ ziKJWgOd`5^o|5a#i%?Gvw~8e?Rpi7C>nQ5dvPHVTO$PI^mnJ*7?gd3RD{|c_a>WrXT#Es3d}(k z$wpmA#$Q^zFclx{-GUL_M$i0&mRQMd4J#xq-5es)yD{kYCP1s!An(~K5JDRkv6DUSKgo^s@lVM5|V4mWjNZp zsuw^##l%rbRDKglQyj?YT!nk$lNUzh%kH705HWhiMuv(5a<~yoRDM&oCqm+1#S~|8 zA$g2Xr=}p_FX%Eaq{tUO9i*Q1i!>$+1JYZCL}flWRvF0y1=#D#y-JQTwx6uP-(bC} z_uP7)c;Xd`C6k#JVW?#Id7-|`uW+hN0>OM=C2Ta^4?G zr;EvxJ{%l|8D-heRYRM%f*LBC)krHZJ@%&CL0)FADWh14&7KV<9km6gE=o9(7keg~^rIQtthK^_8%Jk&aZLY_bc6SbY>IcwDK9{sV*t1GfKwf8aCo8t za)yALEi^-WXb!k6n>W-62Z^n8hO|eRYr&uZiW5d_URi??nl*aGu?ioQ+9RF9u8kwD z6UZ6HVd(G%l9>y7E)uyn?gAJMKeki0@tG*jdcE-}K?8(D-&n=Ld1i=A1AI<1z>u5p=B z<1}|q3@2jNxW-}Q4z~s|j&^Qc;nXIdS3K8caP_07#ig} z#KAD&ue2jXc&K#Q`Hy#x+LeT4HHUCzi1e?*3w{tK+5Tij(#2l2%p#YGI-b~{5{aS8 z!jABC*n6y~W|h;P!kn(a4$Ri2G118!?0WHDNn((QDJP^I{{wPf<^efQWW?zS>VS?X zfIUgCS{7oV$|7z2hJBt+pp1CPx4L{B_yC3oWdE)d)20WG6m5qknl}8@;kjPJE@!xP zV(Nkv^-Vz>DuwBXmKT(z>57*D<$u=Blt)IS-RK0j89omD{5Ya*ULWkoO)qeM_*)jF zIn87l{kXPp=}4ufM1h7t(lAL?-kEq>_DE-in8-!@+>E1+gCV9Fq)5V3SY?**;AKq0 zIpQ(1u*3MVh#tHRu5E5=B{W-QOI34plm`#uH(mk*;9&Re%?|v-=fvb;?qvVL@gc|l z8^L?2_0ZrVFS-stRY(E>UiQeG_sMrw5UiO znGFLOP-GO{JtBM@!)Q37k3G_p&JhdwPwtJS6@R4_($Ut^b!8HP{52-tkue8MG=Zwr z7u6WaFranJq4oNadY)>_6d~?pKVxg$2Uz`zZPnZVHOh-;M|H7qbV0OF8}z;ZPoI+| z(`e}bn6u*kJpRLC>OZ}gX#eHCMEk#d8y$XzSU;QZ|An$pQ%uZC$=Ki!h@&m8$5(xCtGaY3X1FsU?l5w^Fr{Q-?+EbUBxx+b?D z80o*@qg0juG;aZhj=tO=YHjfo=1+-NqLME~Kw7Y1A*?}M7#cOyT(vd$1tVPKKd@U! z&oV!RzZcK6gPWj`*8FIAy2I&x``h_sXPe*O{|ih(Y+V3|o68MWq~2Iy^iQ8RqK76f zC$1+hXqd^jsz`U{+EFo^VQNrLZt#R`qE*>2-Ip&(@6FmtAngx@+YnG}b5B9Y)^wg#oc z24KlT2s!H_4ZR^1_nDX#UH4(UTgl603&Q3g{G4!?6Sl9Om=Sy|8CjWO>d@e9?Q%s- z-OS3*W_H7*LW|Ne{b+^#LqQ}UKDmiZDma@no2!ydO^jcm>+z379K%=Ifs{20mT|xh zP$e7P=?N(tW4PMHJOQ`a8?n}>^&@<`1Rgo`aRevPp^1n7ibeS6sc8^GPe>c&{Kc+R z^2_F~K=HVI45Pf|<3)^;I{?H}vU7-QK3L1nHpcn3!1_)<$V;e0d_b8^d1T==rVpky zZTn~UvKrjdr11k}UO@o>aR2wn{jX5`KQQM1J1A?^wAFvi&A#NA#`_qKksu`sQ0tdM ziif17TO<{wDq_Q;OM}+1xMji^5X=syK=$QdZnS#dwe$;JYC7JozV8KpwfV}?As|^! zFlln0UitprIpuzLd$`<{_XoUV>rrHgc{cUQH-Px#(_Ul%=#ENrfJe@MRP_$E@FLMa zI`(J)Imw$o427@Oc^3(U&vz}<3Lfmy7diVpJJJ@gA>e;q-&gj zcGcBC_luF%_;**EB?o--G?AkaruJ%-b*8aX$4E+-?V@RWMnjHJ;hx27Vd7l0nUUY( z6OQb&8g8cvN3LZ%^xvIav*X|Epqm@yrTZk9U{GSZXAUJt8Lh(%7?Eaf&AzmXOVvU| zmz<@l1oMe#^POR38KT6q3@c`{%eYNu4ccurv`q?b5DzLxENjSfYOJHAI$MbSNgB*D zJsP>i*BgrFlIn?x&DH9x~UbPBtMFj{_vJ#CaAF>1$oE&k`EF&L@HCa@mN>Q7~!RU>7 zW%fv84aCKSgBacmuvg}r@)YKqO$U{D5|!`vG-Gp%An}raz2gESWm0Exhux4C)zE}} z_@kn z3t}bvm?L+@@az@<*jG>(Xopq&c*;^mttlJ!mv;5k6o%Ac<_`o`4G3qzzo(GO{!&F8 zW+~bF?S;7gO1dQ@>gwZ?iIHjE#^@;Ix!Z`R6{RYLlGB&v4A)ha(2hc`RGV-8`LcvSf+Y@lhT%(Z7$tWEF;cZs2{B|9k#&C}sPyr; zd-g~${TqY7E$9X+h4_(yMxQ%q;tm(h(lKzK)2FQ%k#b2}aMy+a=LHYgk?1|1VQ=&e z9)olOA5H}UD{%nu+!3^HsrBoX^D9Iy0pw!xNGXB6bPSpKDAaun{!fT~Z~`xp&Ii~k zdac?&*lkM+k_&+4oc6=KJ6RwIkB|st@DiQ!4`sI;@40>%zAG^!oG2@ z@eBM$2PJ@F&_3_}oc8A*7mp-0bWng^he9UYX#Ph*JL+<>y+moP^xvQF!MD_)h@b}c2GVX8Ez`x!kjAIV>y9h;2EgwMhDc~tn<2~`lf9j8-Q~yL zM=!Ahm|3JL3?@Tt(OuDDfljlbbN@nIgn#k+7VC+Ko;@iKi>~ovA)(M6rz5KP(yiH| z#iwJqOB7VmFZ#6qI~93C`&qTxT(*Q@om-Xb%ntm_?E;|58Ipd1F!r>^vEjy}*M^E(WslbfLE z<+71#sY~m$gZvoRX@=^FY}X?5qoU|Vg8(o`Om5RM6I(baU^6HmB<+n9rBl@N$CmP41^s?s1ey}wu3r3 z4~1dkyi%kA#*pLQy0phlXa-u(oK2Dwzhuex$YZv=*t*Tg5=n~H=}fJA!p2L78y3D2 zimkqC1gTU(0q||k9QM#><$b-Ilw#Ut2>JF=T^qN34^qcBEd={! zB)rxUbM2IwvMo?S;Id^aglw}-t9et}@TP;!QlFoqqcs(-HfNt9VqGFJ4*Ko*Kk#*B zGpJ>tA9(=t|4#M!kBaf%{$Kfj3-uf|ZFgiU`Bo>%k_OuAp~vnE^_Tg8*% z*?)4JdzyMTzvNDy{r$c``zBw=Vr)6c4}CBIv#mw()3h7`?V-;LF?J&N5a>kjpy;9n zQyXvuu`n?+W84QV=(i`JEJY=}Ak+u4>!Lyt2P!$nBl}T=^|pG*z@)_l!)OKB{tIV&&E@hj=OIhSBHgPV~X=R3NrTMh?VzDm?1yW^IJ&zzAn2{8rE~MRX5EE)a(-T&oE)1J4pGXBYi+nexX-?5! z{EZ4Ju=Y8MQ87=uNc2t^7@X)?85KeSoc`?BmCD;Uv_cwQaLyc}vvnJKHV zuK)H_d)xhGKB!_pRXv{$XgfZ_(8G%N3o$ZI#_ zixQj~so0*m^iuA!bT>&8R@>b%#B~zbIlwt4Ba0v&>B(`*Z;~?6!>-aQ zal+Qt4^dCcjZZMd4b4Khg~(GP#8$3BeB8j!-6l?*##)H?J$PeUy)cA_I26#0aggao zaM5PweS_Sb@{OZ@Uw*(!DNV)KTQU+BTRi?AUAv0Vowth`7mr9)ZVC+TI?@; zWGL&zydnsuE3+D7#U~P%PrxpD3nTc9#mm621iX*?ZMS_Q#n9SzOJ~Hg@`rX{d?qJ; zt}`76!H)MX#=VKifJZP$3<8@}0-llthFpq3FV;(UP$-k63MkHHq~J&}d?C<+c~*Zk z<#G&>AD7EoiAVO38TO2TOBKN>6N|JS*{+`}V-)T0j(bAzGlEUWEvWLrMOIItYexh) z?he>SJk*#bywgDF6+*&%>n%0`-3tOY72+n&Q1NJ`A-bX*2tJV(@;%b6&RxMcUd7+# z@UzOmc9DolSHc-D$5(GouinaE%&uOVMyD&CTdKaEB{Qap4_wU7_=23CULKQ;jmZuV;+Y$(`#Gh0@}s7-!qk-^&#IG>7B{yft?UoA)H5 z|B0u3Tu0TF{AB0jpT|E&RsYB$3WiQU^5p*|f)^Si_#^j+Ao^|5(gNjn+!0|NtXDt* z5fwxpajl@e0FrdEuj2s#Pg>gUvJdko9RBwEe_4@?aEM?SiA2nvm^tsLML{-AvBWM7 z_bm7%tu*MaJkUWd#?GWVrqaQ0>B%Azkxj+Yidvc$XdG1{@$U~uF|1oovneldx`h;9 zB1>H;;n1_5(h`2ECl?bu-sSY@d!QTa`3DrNj_F@vUIdW5{R7$|K{fN11_l7={h7@D z4}I;wCCq>QR6(;JbVbb4$=OBO)#zVu|0iK~SnW~{SrOq&j*_>YRzU&bHUhPPwiy($ zK0qin8U;#F@@}_P_flw`bW_v^G;ct?Pb65%=%egDBgS#YF3?E36$9xzdvYqjAZoK#hcjctJu~MF^S*$q3`o2;!L|jPnM1x*Q~qF%BH(5UDFYglsJwO zEdEuB7NihnTXK6$)F~``nmSQNFP7x7hE{WuOjTAhEjGw#XxvL@S;aZYuyu9)!yZ~X zo35D6Cwb8`shRXCCR;xlR`n`cs4aie!SSM`0)x3ykwM*k zK~w^4x2u#=jEEi`3Q9AU!wE)Zpn#)0!*~)(T^SEjIJveav(d1$RaSMC0|}<)?}nSG zRC2xEBN_YAsuKyl_3yDt%W^F`J-TyeGrcfboC_0Ta=KcW_?~RLb>xbqIVI6`%iWz; zM8Kq9QzwO8w!TntqcB;gNuV$gd+N|(4?6A9GEzYs z5f4(*N5}&ObeYA~I28r;?pKUj4N6}iloE=ok%1|X()Ahdwir?xf6QJfY7owe>pPj)Me*}c^%W-pP6`dnX1&6 z`b#*_P0PeM+1FR)t)Rnr22f!@UFBW!TxgjV)u0%_C~gIbb_D3aPhZ~Wmex0)Lj`VoZKjoW)dUoKY6*| z0|V)|XyjiKgZ}s5(SN?te*muif87vD_(wYOiOjOKNI4L*aK||2$~;s25HS#iY6r=)WW8a^dkd0Y|pPc1-9jmy&wqoCbL84`C94At6$lm_o!8m*did^?o$m?ozIp{RmZ*M%YMX_i$KYkz_Q)QK?Fdm)REqf*f=@>C-SnW{Lb;yYfk&2nAC~b}&B@@^fY7g;n(FVh_hy zW}ifIO9T7nSBHBQP5%-&GF8@A-!%wJAjDn{gAg=lV6IJv!|-QEXT+O>3yoZNCSD3V zG$B?5Xl20xQT?c%cCh?mParFHBsMGB=_5hl#!$W@JHM-vKkiwYqr8kZJ06n%w|-bS zE?p&12hR2B+YB$0GQd;40fJd6#37-qd1}xc1mNCeC%PDxb zlK=X|WE*qn2fROb4{oXtJZSyjOFleI3i8RBZ?2u?EEL1W-~L%7<`H6Vp0;cz5vv`7jlTXf-7XGwp}3|Xl6tNaII3GC z9y1w*@jFLl2iFA!<5AQ~e@S|uK4WL9<$R^??V^aM?Bgy=#|wl$D2P$o;06>{f)P+X z91};NrzVV+)b}k2#rYLF0X0-A+eRul=opDju)g0+vd79B%i!Y}*&a^L$_|C&jQN^j z9q#4<(4)3qNst^+ZYpyVF2hP;DN|OMxM9w(+)%kFQRcYVI zO-frej9x6a%-D%Xuwedcw9#3VSVkOjNF!BYRoY1KD3wFJ%?ML*3QwcarMK)@v`o%s z$w=NLrO>og`nRJpZZ(%~*hNJU#Y~k;_Ci3~gc=4UQO!Ydje^?=W^DgCKyO;Zz4LgQ zKtm($MdY;UZ((U_g5*pMY+dYGyyT1ERkaj`U#S-2yyJ47wMonCpV+2rI8zPNHDfo& zc59dFz*2#^A-R?P6Np}jhDLi4&vP%$NW#8J>=CLj1mlf$XzmQezH*F1jNOiPgXl2j zzD07AKLT*h$CA*OsOba2etPLU%|p?=XhplXo?vOu@q0{QBo++)@6U?YKv_)GFK(^Y zm&uFBbrQyzJm;c49O00PIt;|{&ei%VSS%Y3m3#~L#(3%Gso^a4#9AaB$w@vnAvdr6 z%!2#)YS0HFt%o)q6~BelT;?%oUjX%9qQCn#-~+TM(a^s%Y>&aBkL(UY{+?a9@&Q+a;t%c_6u^6_r@>MEAN9ir5q=Yo|R8z4lKYd1sv^LyTozFn$KqaJ>? zoH&+`AX>E03Gv=71+NZK2>!-NasKeCfMp;@5rZ z*m<}q2!$AgKUwWRXTVHs!E>`FcMT|fzJo30W551|6RoE#Q0WPD$fdA>IRD-C=ae&$=Fuzc6q1CNF>b3z_c<9!;))OViz@ zP58XOt`WOQS)r@tD0IiEIo4Umc(5f%J1p{y4F(1&3AzeAP%V)e#}>2%8W9~x^l}S4 zUOc9^;@m{eUDGL={35TN0+kQbN$X~)P>~L?3FD>s;=PIq9f{Xsl)b7D@8JW{!WVi=s?aqGVKrSJB zO-V&R>_|3@u=MEV1AF%!V*;mZS=ZK9u5OVbETOE$9JhOs!YRxgwRS9XMQ0TArkAi< zu1EC{6!O{djvwxWk_cF`2JgB zE{oo?Cyjy5@Et}<6+>vsYWY3T7S-EcO?8lrm&3!318GR}f~VZMy+(GQ#X9yLEXnnX z7)UaEJSIHQtj5?O(ZJQ{0W{^JrD=EqH_h`gxh^HS!~)?S)s<7ox3eeb7lS!XiKNiWDj5!S1ZVr8m*Vm(LX=PFO>N%y7l+73j-eS1>v0g}5&G zp?qu*PR0C>)@9!mP#acrxNj`*gh}21yrvqyhpQQK)U6|hk1wt3`@h^0-$GQCE z^f#SJiU zb@27$QZ^SVuNSI7qoRcwiH6H(ax|Xx!@g__4i%NN5wu0;mM`CSTZjJw96htSu%C7? z#pPQ9o4xEOJ#DT#KRu9mzu!GH0jb{vhP$nkD}v`n1`tnnNls#^_AN-c~PD;MVeGMBhLT0Ce2O2nwYOlg39xtI24v>pzQ zanl2Vr$77%weA<>>iVZQ&*K9_hfmv=tXiu#PVzNA;M@2}l&vaQsh84GX_+hrIfZC= z0Se*ilv-%zoXRHyvAQW9nOI2C$%DlFH1%zP-4r8bEfHjB3;8{WH`gOYt zg+fX)HIleuMKewYtjg+cSVRUIxAD9xCn+MT zs`DA7)Wx;B`ycL8Q&dR8+8mfhK;a^Rw9 zh9tC~qa>%5T{^8THrj^VEl5Do4j4h@nkrBG6+k8CDD~KB=57m@BL-)vXGkKIuVO9v z7t_L5rpY^0y=uu5iNw0v&Ca-zWk>v;fLJ=+SaV&V#C-o^}8 zp&Xp$v?~ccnfR=&5Df)32^d6QJLg*iuF#s|0M4zJF@Hza1p`q|f}~K)q;HC*I1_9t zQ&1jr9-kdUi8)DGxiwdqU|rPxYWDQPWY&SI&Rxkhxobp~C=Y*`d?HD4JW?WjU7dBPeuIE`ABLq95b#lfKS52IB^6KoHmm60$R}TESplQt59#mboJj+Na!P)V{ic@$yQ-&Z za^JU0T+n0Lf2VdusoNr0?g~1DMsY)zdY-63yH!Ii#aWe|;0TO>L7#YlaDrH}xvYXn zh-NYa>O>f_NTTBG=|k0qWH+X?d5@+INsQ}WcI_3z1Z4-%Gj#_{P$0A~cAye`?j0cW z8)hd(V}7rattLUSMvgZ4g96P7n` z^{55A&&29;-P992{yhkGWa3v_Z6iB4a&~NmL)IpC&dsSwe$9jS(4RVJGt=Y!b-O~1 zSCl@wlaba_cA*yt(QvulMcLUuK z>(ys_!{vqKy{%%~d#4ibQ5$yKn6|4Ky0_ngH>x-}h3pHzRt;iqs}KzajS!i!Pqs8c zCP%xI*d=F=6za_0g`{ZO^mAwRk0iwkzKB7D)SaLR0h|ovGF2w9C9g8;f#EtDN*vBP9yl;n=;B2a7#E8(%Bw()z(M$_pu zQ+9uFnlJ!5&$kk^S_+kJ>r9y8MFPpSf9;o8v;ZxsMA!p>eaAIwt5xNiQ|2_ydGkbi zkggG;Xp&I7C8R{>ten^j@MsN#V5JPs1Ezc!74->Nh0a}U){OK@j=OIoY}C7IYYd8-V9 zQ6s?v=Y7(?Y$7=P#Wwub-*0DLqli?I%kT-D^jqK?c2~HEx<2(poRWAUoC}!~6$1=I z*M(IfPmdID8i+5l@=1(+`?i`G_ew=1Y!gF?tFbdgtW2etKLOFoNozkH(i!Qa7(h^| zF`9!VeqQQwM+yO6J`;oWUWq@9l6hP~FiG8-{Pj*T`XI3~s@FfjW2Tl(llpa901$&y`F}K1uZuHEo;=mr+_8d(o z2Be#yWHEN@euC$=VUSB+3A}khJdF$)0r#<5(f3n`kx>ZT8ifaKyX*OhffeHH1?6OM z*-19$j5tMNYQoB)>cGpz@11>J%q4KW`GLNj?uB>LcNg$0G@}XN#Tqf2F5@jv<`|~p zqB^l!%v!g{R_+0GX5z0>3Q~O``%T$NFc==dsPsTj-;{b$XUS0TGoJs2BUA*H;4S?w z|Nigt|F@9hf7QLSo}JPEK#CPgYgTjrdCSChx0yJeRdbXipF(OwV)ZvghYba)5NZxS zm=L8k_7Lb?f8`=vpv(@m%gzsCs9^E$D5Jn+sf}1lep*zz&5V?~qi_@B?-$Vd1ti(rCi*I0}c}slKv@H_+g?#yarVzpYZN zIk21Bz9Z#WOF`JG&TC&C%a*3*`)GJx9I!U8+!#J4}@5rm8*jK%Xg2VLjP-a;H zFydWO;nxOZ&|{yOW;ta$ZU^6*4vFP)idD6M*M0+9buB#hK4z%YTGBdSva?Pvxim2` zF-?QVGuRQ2-1eYzd1Y%}w^`t1S7|{{8=Es#ApC0<;pc$|NJ)IU%WVK+4gnTWA7-t1 z0K{DCESXb}!y_tzrycr^%%|G4T4)`$BC8+qm|n1lS?CO=`V`1T#ykY#5g5$dc$lGt zqGHyw-*Av%C;33nEiU(rU?w^3F46!dEz#cHd3IF<(XCq)>JG?Bi)4v26MQr1A-g5RqhFoPy%^TD3sa|D^9aS>>_2-X2i#? ztVp@ZkyMB;Uo#9s!R!@G#CCaFVaxx*8YYu$kGFk4g3|9t!1nKqOaDBAe;w!(6#w)0 z?{&F2BgctT1=Z;TvjOGL_!}Vlt=kaLA7#W`mv1h%hUg983!wA*K@_r6_cd6o z6LHiCE6qwlt2H&|Ica~%b9C?Z@$dreBNR_!NKcfL)%8kGr7!IVq|^&6PKYK%EhcKu z6+uR*%EOw=rF6Q42Mx|a> z$2XrM*NV2x9ci6|X^eh1UAbJ9Ky!#*Q5w7)#o#%}d!#-^k8To=n8{UU*LmFsS-wRj zi6-p76V6g?If3S&Bj~GW&QI_WtyPY0@u3hjKtqf9`8S!wn{@P&Tc8uu8cf)YmrX7+ zrC+O3V{9}JG6ihA&^2Q7@)Kq)j(Y_oTzsoBUYQDG!}`Ame`bbcr>J-6E%gaBPEDCU zflX#1-)Ih^HJV*lew*N_SdG-4!b2}G8%U&9_V0~Qt?ZS z@H3L&5ybV8X}A@KQADl93H`}0qkNm!jGHkCJUM%r8`mP1nV?Oo%^l;yDnU6IJtbuY z`X2Sf8|r00mB_f)Q0;S{FqS1Yq?otd-BVbw`#@SDd5}n5X4lqdDi1*vtVv8-Zi10q zexCj0eyngrp`UxjEOrdzUt`?%jRlj7zSU-V-%R?y+_w7P7f1ge%t1ozmN+&)%3xQW zT3u@)))(_a<6`lTJd`DIYw>(pkb=PMKvCNEG~zza+LVNqkY^}QoGMVdS0K;gS*A3f z;6Ua!^sSV-try(M^pB6D9dsX}c>$Da#NHucp9vr(fg4pbBR*uPhYq+N>q1X4RSOCl znIQj4=A+y+8{?LQ$3L@(!Yy~~Cu4Sx72*%@dW>eP%Br7=uaynV6Mqa-49A9) z|L&5r=4K5SClwc`!2J|>(#n$4y1>lmR~2Om8q6HkcpK>d(Fk!T^NO?hM4Fc+(5J{` z&K|vrBz;;zWlNO%=a~JkMxMiZa%wYz#G901lw#+2SUaMMHrebb&|1L8tKoGJK*QhJ zU9|WkDy^-4F6U&VYSc3ScHDk@kV^0801#I|-pSK%az5=DwI}gMm)@s2O+-ESTk?QY z;y9gyucaXO(Cc+cd{B>2)euMHFT71$a6DssWU>>oLw4E-7>FC-YgZH1QAbRwmdahD zO4KAeuA^0q&yWS|zLTx%(P4VOqZv-^BO`0OFAXdBNt9>LAXmPALi3b|gt{b?e-$z0 z4n7H$eg6y_zs(c>*4FT!kN*$H`43~1p!g;IZ8-mYbUPTejaLW#BZnAPFES?ApM{TQ zE*TC%O8)apqcX|PrNjIZE-z{q`I(LwIE0kf=PLjExEX>)oIu><<@lt>-Ng9i$Lrk( znGXl|i4dP;Mt^-IbEp7K0e#*c7By@gCo@VQIW$93ujLL`)lMbA9R?C_5u~7^KopaAMj#6&>n-SOWlup_@{4 zcJ?w_!9JKPM=&Bd#IQ37F*x39y!azm$;~IRlkm>bHdABcNwW-TdDKD$pkD{j6A8d* z{vP~|<}bj_Oz#83K$ieRtsA4a@4a5cRjJ}A01{PgxXn3;fx)5ElMEPwDX_mW9)9oB z*;scve~v#HHqUj3KdC$tdV3&0)Whkp-=hKKz{SzD7g0@N!wyv;ZAime7AjB7&)!)5 zp_iVblaf)%agwJqOG2e7WTCM1&khq`{b>fN4n8hOJbvO?Y;60>LIwagLXWC@@0RSR zo%lPo1cUU=g$ahJ8D=;`v~ORUSl(1-&a@yTAC5Y8E892@{P@MM=GXUGpBSXSbSs!N z;L~0D_s7{+^F6c!WW+^yz5~o7eWtsOE}8{hKaFlHgnyBeUJ8Zz2$k7Lrh?NuMU|No zVvsq@57)8zin;&ckR1;*Z%(xH2lBw z`x%N;|H1En8au588bPDxP^$kfpO!bIzz>K=5Jiq9Rg(NGde0g!rKagLa+&yC)jg7y zq}~2IH)N*FJC31qrIH-2;%3^F?=bDD^U2Y;%ftN(v71oY;od+vh!!2z^}GHR$43rg z0In@ki}TglIsMU^O1(SiLK#oiuyw zB>-@z?&uW`ILoPupw0_cs?C|2YoX&87~us+ny%eo{A!3M<-7O7mHUBCgA~{yR!Dc^ zb= z8}s4Ly!GdxEQj7HHr<}iu@%Lu+-bV>EZ6MnB~{v7U59;q<9$h}&0WT;SKRpf2IId ztAjig0@{@!ab z{yVt$e@uJ{3R~8*vfrL03KVF2pS5`oR75rm?1c`@a8e{G$zfx^mA*~d>1x`8#dRm) zFESmEnSSsupfB>h7MipTeE!t>BayDVjH~pu&(FI%bRUpZ*H615?2(_6vNmYwbc^KX4HqSi!&mY9$w zpf%C6vy@O30&3N5#0s_!jDk|6qjb-7wE3YT3DA7q3D`Q&Y*y>XbgE7=g#rPx1hnf8 zTWd{IC!Iysq*vZup5VGrO)UM<3)6raR`rOwk(!ikf3XPp!n|gz0hS*P=VDXAyMW(s zL??-`&IusEuOMrz>m(A1W5Q~>9xJwCExAcMkOBD` zD5BJSadd{0u}%z4r!9qA`FW4;Ka_Qk>FcHxiucGw4L9qhtoge|ag8jbr`7LHSbVQz z6|xUo*^LV1SLxS>?D`m=g{8IC&1YF$e}VRGD#ZOc_15QW%J@FbEj8tE-nGxo4?X02 z@|q#k*G4xMW>q84Xc09pRj@>Hz8t^fMm3n&G;Al6KU*;=W`7Q{$^|=bnZiJ7?(s)@ zB`vW>#zJ{}!8=*|?p(~fcXSanO^j8+q7V!q16*ic!HLRdz0TzNI6}m+=OKd2b8KX< zAcDTj*%~vQlcO+%@H01gjv-1zZaOXVoM*t-+KXTR#NoTf-#{dQAm?GqK6q8Ta zu3xW?t=NE$EfYa#=0HofLn5~c#m-U#Ct_r6~X-pg6k*F zYIP7De52BBwcAnK?O(j?YEs1;q60!-!hTuKzw3T;XcA_w5HvU;tO~}byLA^cggu8i z-IP@pxFjTy&ie28m}j66dm@g78xK7aG{QSR^bAcY+W*xWu;G~I08sf(GK4>K-cbfJ z-%v9DGR77He<291M~=fg>>9&NFQlboP)pC6fT;{>_!lM`A&&HWIMd)Y6e@IL;nvRdBE*Tn({&3{-XJ9helJa{G51Ck}-_Y=5C|fEo z)7fZlsHxN&SY&ZLTdYuBBZnwIh0#VTzmyK>U0|r&SXb&GP0m)1dGV8z(^x6s5yQ-z zEyniK${#U@Y7p@Yxx}E+jA?1@{=|e6UM;iyai=0=aItVvqieogZUq@sio2#9NLW~L z{w@^H!HEGU;>;T0lu{Ad20Hr6u;?-9YHKvkjEc)}wsb4Y-ArRK8`24uBT8N)8m%Ee zYJX21)|e{peL26}VUUKYQ3L@NSe8rEbN#AIo$tjJm-$B|IJU?mu(h$Sq`XNY0@NhY z0?WeMtPwP)sUdk}dWA4qBUV^x>P|is-kPgVe)*WV>dKDL>gOq1 zUYw(nU|N#dw>97A_(c3?VA_zDfF{^A1eE#8Bucd^ON(sv-{tc@&i)Y)3V~o7U~+AA zOwnXB5`WN^z$z<9^@(?LY%7?y5X_C(j1ip-Ug^f7Tt6suI3&a=&~#EJegG4r2^tKz zJoEXCVOc1QdOSNHp2d;t&smxL%CfK@mSl)Ky}`!6kCsi#7s5&G2Q!sM9S6o)&mdx% zz|2M~pav2;Th=DTN5yB@6HFAO!pl-y+tEJsh}(? z!tIyg01O*w@mWxsFhHMi7%Gqz!v(Osc5WxK+^1PGfsozw)FE}VIxk9GexmAohPNAF*SAjxG3Al#(xQoYXdI}TR zoCHAFS6+LDqsP8L1SZH{RxJjFK_=vy4nNH^?M!OsQWe^qC~$c1r&y`H9n5;D z2F$t-Htc%2@K(>opJHE{NytI2<_J<6Kz*p$wtKUTEH}zITx?H0L%!5%i@!rLphSBrkFs>jscP6?HVQovX8!~b~ZY|0h%&souT7e5nD@OxuSgC zVW*eo0B|1POwg7;6fJSUC`g+`1%XQvwpRc*&|AtV*h!#5nQM(@m!K)-Qop!Rt3F`a z9HUO zF3w{uI_==EpjFQWV4boF^A?wc@@@U+KrKPjn6sK{OLu-~1UloSqt-aHYo*^@kQy2+ zH(9*-mFz?YV4cL7EW)9hsdmG{5jaYXLvm*&3PZ4y?8z`$9z6`q9fgsJm@*W$-QSzu zut}57hroSbTd=&RJpuy#?K?A6!-;_MowpK8eb~5T-^eye%3O-T^ktSMbd%PT0j-B?#yAKr37u%gB z*2)WJMw6Y)6BvY$JjD`(06ci7u;u$hv}gN5oS&Q^*y$J6L)0#BD<>XL|;pZgtZaxp3~$0zxA(;6Qr_AP$?8l@S)C^Hoaz#rQFK^lA}3&)Gr}Fsca? zK>9BkVcl;c*E2P9UMppEIB&38dL9R?Xg9N{Nl~4*w!qsZJElz}Xc9gz#}cwnP4u{+ z6VNTEx*>u67?3bn{sWk*P`1_$YfsB+)Ax0+jt|)0p&VS?N0k8IAp2KH_#eY3I#{Hw zB$vObUDtXyZX)*wVh*@BefnUej#jv@%uiA=>ngX0kQXaz>8(WM)fX~v__@I}7|!Il z@J%r#I!JqqFwGd4JPhmDmL>1Bh}nn_BE;hgKUesNOf9zQhiuhn%4B}O8jnxEwJiQFDaiiuXw2sb?*8a}Lr;_#7+IPfIjhVDhazSpbQZECL+4)p8lO;)!y>Rt=0X*;O# zX{s(p-*d{#{Y3gVhL;A{4a(Z5sIfpk;WMCqdFA&Mb7mp;YMXhBF@p`}$ShAug+bo`;<9fm!~F z-;1yCj$GQ^mzucrfuatilXrYLr)`izjn_m(f~);txN?D7d?Kg4wDuPXilVyeVwjzf z=4Kewf=u}X_H*viVfPWZW?Sqa3G#h3|;b!Q7>BRc7-Wox0}&>}Lqo=0v;T_i~% zqB&h;14|~nK{W0N=$obGP@O%(c8SraYS^qiu%Q`B zBHdA!`Vk7#Bz*@_3eE#bizLzjBV;F0vfSA~+7@8+F{$7Y?fwI~Pp_X`2ORgqW6g@2 z{cQV!niSsMEVr1IaeRAj8~|*4yW~X5$6o`crw4uTHhgPs^qAk?9UPu;xy5wh2^jZ; z)@27Q=QKa?8w7_C0|u`@k=%b9Ce$D7x42CdLsckF2<$wLuV2kpik8PXex2^Co$n2o z)l#H*;#>?yrPw0x6LI@x(X$nezCBa0Obi%|I5ZV|4bJSPtNHjDkS|3S?fiv(i_(n* zFbve0g!B0!MMmakRsgg_if8nwImb=kk%|s+08xGQ)J?vpkdaya3UD|RJK+LQ72|g> zc4LnwInx!2pN-5Yvp7rvRF#B=(ZO8gyVB^0Dh#ZdHA2BjjppfV<=2Nm#w_t{%6O$W z`-?7N?LwL0DWgK0Y7L#ChSHfa{=DOpJpl8L@V70cd%ei)n%SQO;Z+Xw#li#%LUfbs z&hP%UzN(qM3cw#bWQS6_B@>1^ea-AqNA12xoiQeb_Zdtf>yHljqeIHqlyC^gzH)h1 zstXTFEb0r=l9;><<$a}YWlscH7VW_xeKVZ#*#v#HiuUOs7PPj8ml4#!BiGEK)kDpO zX=2mU0ZuIDDnhfV7v_Rs)0R#ff6I6_|MrzV(R$3Nt#S7D?GQy6?a^WRvA@r2~?7f~s99*9;fuqJ(843U`hRl2O|sk>J@WMsR2O zwyZt$@J)DnSUNkF@B3MPNz|<@`72{M*S5d<1Vkg+G=q~u{8OP84Yh6VCE5pNC*#m> z*jzHy5Tc82sBVw+6W7DoR5@LXZ|+>;)Q%czg%8pyMyeE2-)R^oHg~SrO~#I8MxNc> z6pWT&F&H1mX7#2@mBY>#rRoFKszT z(gvV#j3x|7sF|Dt0*CgsJTdH1R!>inYZWp*2RDbjjQCP98L_ds!$x&{t85NRYk4ii ztJ3HyC8h2A2&`kq^Cfci>N*r&btHg_|v6=s|v=(-MQ zK4kjqoI^~y`j9poC2r{Izdlehm8!AcMP^+SwDUce1Zon(%YvxK)x|rXsJRlO?-K91 zMsmHgI&PmqT_W}C0mdA_6L!EEjgJzidRvTN;vQRJ-uBl#{dEeN?24PRwx)7c5kF^ut=M0)e@zr?z_vpYf=%;;@UYF9>9-->Qf2FW*# z5*#VFB$$-k(zphh4sAElMiLbp`$+SKm*{l6qX;Q8GZ7b|J>OhC!yg$}8dt$dx3E8b z$FlaM*K@6mSsYCoe#*QjLEB3|_Vs4GbZI#!>Ya}dzh%uMn}sw0gFQQ{+V+e|_`q)M3nK27)nAqQ-viJoPHUKdr9HN`v0 z+tZo0ORLuv_d)x}gO|~s(H!12RM(aMfqLG>KSH#kGxC{sUUj>FUC(6;ds1cOjeDYu zOrd>q@bNFq5?0s&@5nbF3-rw{{V&YYf3o_9|K-X4k861UwZ&C2bH+A7^%7nizU>b? zC2@*VlrqprJiv$rx{+^+Op9i3RM;IHq@a;34=Gn%B+rXMZi=UsHC@TEFk4{*fs96p z)wNUY?AhVkdLGQmPESuh@-!iqSZrnxIT~Mon)J+i+B~9VdL8QE`^4=2@lNaKluUVx z_^i7~5E4dN4&gVMi%;7ast@WIY21Q`+^iTC*Gx@IMVYB`BLFHzPh{Fpc6LKZTk@>P zquo2E*Pgq(0MX>h>4)YaJYbIK&V?-W}JfL@&R0I2)TOA!Teg zNa4DBO&)`Nn0$Inb|d8ea|)qqOLYVbQIBRC4T4E<5#Nzc2 z57|Bq7mYsW8y?uLA$XMj%OeK+1|DAKcLYB98-vDP<3*+SKYcPcOkm&}H|!{9l*9%L zbiYJYJ^)Cql-&wPwABGD>Ai7SUXe15m zIr^wNEU$9)D6@atm z(w(1~GuLpHi?JGgIBj`Ovy;j4M`XjrCNs?JsGh1zKsZ{8 z@%G?i>LaU7#uSQLpypocm*onI)$8zFgVWc7_8PVuuw>u`j-<@R$Of}T`glJ!@v*N^ zc(T~+N+M!ZczPSXN&?Ww(<@B=+*jZ+KmcpB8* zDY_1bZ3fwTw|urH{LLWB;DCGzz$jD|VX#Af@HC%BktA8F7VJSy&!5iTt};#U^e0_q zh6j7KCTInKqriZ1`BiF3iq2LWk;gyt0ORIFc4Mi3Bx`7WEuFq{u^C49-SYVjnv!_40m1>7x*+<8~Xkq?056 z!RBfE@osP%SxzOw>cLAQ$bioAOC0V!OzIXIc};)8HjfPtc~8tnah$PtoAz`4k)7$FDUc2O@D)g_uAo&nXMymK$##V?gYUPt^l zj{6NFDL(l-Rh(xkAHP%bBa=($r%3Y~jB!eQ1Smuq2iuQ|>n%Y=p(26SE5gFu11*Q< zaPN5G^d;Iovf`VY&Gh58z~%JpGzaeUz6QoBL^J%+U4|30w7Q&g9i}}@l61eKEfCgo zST6qMxF_Eaj7;0OC)TSU{4_m}%FOa6B{AxS$QIcmmG~IVjjf;7Uk!HBtHfm{%LsLb zu8~5VQFyOZk&!VY(wxL__haJ;>Bj?g&n`+i&=X{unJmv&0whCitWfGlOr6+Tc-lMZ z(ZRXqC-=O+GAvTXKViA9vdwu{aifhk$tYh~-9BScg!Yr*M2zw&9`pHMxHGh`dUH-1;~^6lF@ep;X9PjQ!rqmXNWJ?#P-qb%*TB%xe&3 zX*5V>xuW7)$3!Yc$y>cwBqd8+p+u>WS7p7~O80ipG{(a*#=NJ`^Ld6k-`|;Y&htFy zIi2(Sm)4eD=o+CGo~M3%qF|O9P0+ahmc%EklI?NgX05W3+OdS`_Rd#wg-}hd1&txU5wXy zy`x)05?WVZvELw`XWetIAg6$|(^4ntaE;=f$Wcpwbxm7?bLDnPs-1!bRoMcy!EeOh zpIv8ewDzcIU}mv1NxV!&(Wf7~_kqGAk=2=j&O5FA)z2!APCcDQPnIaiqMkVT4fUyX z))R|WvOJyzcU6d=z0q8JDt42*`js4g+_t{YP7lVguX+vhEejJ3TAIo*Z6jizHm#S- zZT_}-STQAa-0Gn8+RmR7V}{Ns1@jJ{^Sb!9&RSXXP;^ep)r6;&PW++~XYXC9a=zSF z?sp(JQo&MROb~b1Y*Xw4!P)>PHT>Z<)*U=Ax_75^OUw97pNudbxS1XPtNrIg zQ5YB77E@i7$2Ia}(^JcCi@OX`9a|m}PY%-th2m~y+)eCl>fTVjCP^lDOBLyhg1DZ+ z)~G{&OkDc$!;t~`gq(wz@qW3lh9B^ic$>-h#nV!H8d#l+>C(M%g}u2g=I#&W|L!VD zqHYoQkBW;`r|fW02u{7X!X;}T7X4iAaWzkeOh}7&o!F1qt4#$1|BDF;(2VlgEqJ$F zy8Ba-y(%fs`MzpvyXlQLEhS^ed$7Va2hO%?$-D>^*f$b)2Hx;}Ao$UqFt7l26<7eP z!{!C7PVrq>=794Zqmc z%LKkzIBZq@%Ja8EkH}?>c5ILG(EAMS*JHu?#9_7TsELw)8LZzN>f2Y6YN{AJC?34> zh42sPa1%2JpCeS9&E1URm+Pb}B>A1M`R{+O+2~}c(@^1Rf&J9p(4QqHl;E^4w5;I5 zM{?(A^eg*6DY_kI*-9!?If^HaNBfuh*u==X1_a?8$EQ3z!&;v2iJ``O7mZh%G)(O8 ze<4wX?N94(Ozf9`j+=TZpCbH>KVjWyLUe*SCiYO=rFZ4}S~Tq|ln75Jz7$AcKl$=hub=-0RM1s(0WMmE`(OPtAj>7_2I5&76hu2KPIA0y;9{+8yKa;9-m??hIE5t`5DrZ8DzRsQ+{p1jk-VFL9U z2NK_oIeqvyze>1K%b|V?-t;Wv`nY~?-t;tMC4ozyk8CR(hoZTno3!*8ZTc15`?MFf zDI892&g&3lshOEv4E@w-*_%)8C_<&HhV`0D5lN$WT4Q^UWHNSAE+RZe(o z%bqR^hp1IsDr47e^AajFtlppT)2F6yPcrWO9{Kw{o=P6y^HOW$Wqd_)_fwzn`ikZl zOGVc0+S(*=xZ_KbL0Nr`Sx$$CWEbw$52udl1f=X6CZEcFMA*nl>`0gn4&tc5^`!!)tGw<}^Q>P7E}$ zialDUofH*XcB3r9@tA@lnS}dA(@nK_xuw0b;FPUnNGD0;MIySCw=cSzB#=3>F37V-nni3UNB)-;;Gkk;3l9fh6FIjSZU zk=Eo2a`6i7@i*4>ym5`R?i-uZFv6+iX*Gi^I}ZU1OrLAX8aGiT@`*YnjeF>}$U}ORP`+EY5`eqVC_&4yG z;Tp>+2QbZ?lt1GB+D}q14W3dWP8lWnN zf(nlT6+XW&(zme{FbyDpP^NakA<~TK=Y}H^eS%2rt0v8Lr)B}@B!cTvC=9FM;7q4@ zf*;vb4HG>RFpY5?vFCp27VEnVIGx~-na6biU4{+UoYe=}^R#_My6wT$5d&r*=kpAA zu;=-c0|~yqi(N8&*H;aNfhyey+HHQ7J_qae*_CgG2V8j=Tq936S0DC8r3BXBql3Gz z0pLo_`|4Q+oY3rPBNaLmL{QM};9dke>ujP^j@z-N;fNlKb|edn>)YaafDaJ>GWKP$ z5}l&#$QFhN!CMT;WH&z-5E)kvM|36lV!^#3z{@2FF>HsgUO4PMqO#U$X%+U>K!xJ@ zBFs|+woG_9HZQs_Tw*vnCPGhlXG@>y|6pJT$I67!aP&b0o$AF2JwFy9OoapQAk>k7 z**+$_5L;5fKof<;NBX%_;vP@eyD=Z0(QW)5AF7 zp|=tk3p?5)*e~Inuydz-U?%Kuj4%zToS5I|lolPT!B)ZuRVkVa>f*-2aPeV3R79xh zB)3A$>X~szg#}>uNkpLPG#3IKyeMHM*pUuV5=-Jji7S6PSQ9oCLo{oXxzOZfF$PP) zrYwlmSQ-~n94uO3CD{K0QTmj@g%Yzn7_xQ4fTduU0Yqvln`e_`CdXH5iQ5qRr1 zBC;}%YZ2!4I>*=sR)O~jBPx6sxmIEBnq)s-fHz_y0z8-gPl2Us4BiBXNR5CIF!YR@ zb9B305SilU*@4|+ x6JBtc8JSt5M0pkooaq!^FqtuD_KdXXTo>Mw54>`rP&>h&58!3a6l6r9{sG7g--!SK literal 0 HcmV?d00001 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..1b16c34a7 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.1.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists From 80f0ca93f0154e3102a4fbc2f39fb2d4280c76fd Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 10 Sep 2020 20:07:34 +0800 Subject: [PATCH 08/80] [tmp] add .gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6020b6823..4daf54acc 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,6 @@ lib/ .DS_Store bin/nohup.out .DS_Store -bin/sideSql.txt \ No newline at end of file +bin/sideSql.txt +.gradle +gradle \ No newline at end of file From 1ed345a16a57fa38be656cb7bba9ed4ab2d998ee Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 14 Sep 2020 11:59:17 +0800 Subject: [PATCH 09/80] [feat] mysql consumer --- .../consumer/AbstractDirtyDataConsumer.java | 14 ++- .../dirtyManager/entity/DirtyDataEntity.java | 8 ++ .../manager/DirtyDataManager.java | 26 +++-- .../sql/dirtyManager/manager/TestMain.java | 22 +++-- dirtyDataManager/console/pom.xml | 2 +- .../dirty/mysql/MysqlDirtyDataConsumer.java | 98 ++++++++++++++----- 6 files changed, 125 insertions(+), 45 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java index c2401242d..f9d913fdf 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java @@ -23,7 +23,6 @@ import org.slf4j.LoggerFactory; import java.io.Serializable; -import java.sql.SQLException; import java.util.Map; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; @@ -38,8 +37,8 @@ public abstract class AbstractDirtyDataConsumer implements Runnable, Serializabl protected static final Logger LOG = LoggerFactory.getLogger(AbstractDirtyDataConsumer.class); - protected long errorLimit = 1000; - protected long errorCount = 0; + protected Long errorLimit = 1000L; + protected Long errorCount = 0L; protected long count = 0; @@ -49,6 +48,8 @@ public abstract class AbstractDirtyDataConsumer implements Runnable, Serializabl /** * 消费队列数据 + * + * @throws Exception throw exception */ public abstract void consume() throws Exception; @@ -59,6 +60,9 @@ public abstract class AbstractDirtyDataConsumer implements Runnable, Serializabl /** * 初始化消费者，初始化定时任务 + * + * @param properties 任务参数 + * @throws Exception throw exception */ public abstract void init(Map properties) throws Exception; @@ -77,9 +81,9 @@ public void run() { consume(); } } catch (Exception e) { - LOG.error("consume dirtyData error"); + LOG.error("consume dirtyData error", e); errorCount++; - if (errorCount == errorLimit) { + if (errorCount.equals(errorLimit)) { throw new RuntimeException("脏数据消费失败达到上限，任务失败"); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java index 8a64df7a7..a06a98356 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java @@ -94,4 +94,12 @@ public String toString() { ", field='" + field + '\'' + '}'; } + + /** + * 获取脏数据信息，返回字符数组 + * @return 脏数据信息字符数组 + */ + public String[] get() { + return new String[] {dirtyData, String.valueOf(processDate), cause, field}; + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java index 0c90c770c..c3953e02a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.dirtyManager.consumer.AbstractDirtyDataConsumer; import com.dtstack.flink.sql.dirtyManager.entity.DirtyDataEntity; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.util.PluginUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,6 +31,7 @@ import java.lang.reflect.Constructor; import java.util.Map; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -39,7 +41,7 @@ * Date 2020/8/27 星期四 */ public class DirtyDataManager implements Serializable { - //TODO 需要确保产生脏数据后至少一次是成功发送给了consumer，至于consumer是否成功消费，manager不需要关心 + private static final long serialVersionUID = 7190970299538893497L; private static final Logger LOG = LoggerFactory.getLogger(DirtyDataManager.class); @@ -72,6 +74,14 @@ public class DirtyDataManager implements Serializable { public static AbstractDirtyDataConsumer consumer; + private static ThreadPoolExecutor dirtyDataConsumer; + + public final static int MAX_POOL_SIZE_LIMIT = 5; + + private final static int MAX_TASK_QUEUE_SIZE = 100; + + private final static String DEFAULT_TYPE = "console"; + /** * 通过参数生成manager实例，并同时将consumer实例化 */ @@ -80,9 +90,11 @@ public static DirtyDataManager newInstance(Map properties) throw manager.blockingInterval = Long.parseLong(properties.getOrDefault("blockingInterval", "60")); consumer = createConsumer(properties); consumer.init(properties); - //TODO 使用线程池创建线程，不要用thread - Thread dirtyDataConsumer = new Thread(consumer.setQueue(manager.queue), "dirtyData Consumer"); - dirtyDataConsumer.start(); + consumer.setQueue(manager.queue); + dirtyDataConsumer = new ThreadPoolExecutor(MAX_POOL_SIZE_LIMIT, MAX_POOL_SIZE_LIMIT, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(MAX_TASK_QUEUE_SIZE), new DTThreadFactory("dirtyDataConsumer"), new ThreadPoolExecutor.CallerRunsPolicy()); + dirtyDataConsumer.execute(consumer); + return manager; } @@ -90,7 +102,7 @@ public static DirtyDataManager newInstance(Map properties) throw * 通过动态加载的方式加载Consumer */ private static AbstractDirtyDataConsumer createConsumer(Map properties) throws Exception { - String type = properties.getOrDefault("type", "print"); + String type = properties.getOrDefault("type", DEFAULT_TYPE); String consumerType = DIRTY_CONSUMER_PATH + File.separator + type; String consumerJar = PluginUtil.getJarFileDirPath(consumerType, properties.getOrDefault("pluginPath", null), "shipfile"); String className = CLASS_PRE_STR + "." + type.toLowerCase() + "." + upperCaseFirstChar(type + CLASS_POST_STR); @@ -106,6 +118,7 @@ private static AbstractDirtyDataConsumer createConsumer(Map prop * 脏数据收集任务停止，任务停止之前，需要将队列中所有的数据清空 */ public void close() throws Exception { + dirtyDataConsumer.shutdown(); if (!queue.isEmpty() && checkConsumer()) { consumer.consume(); } @@ -114,7 +127,7 @@ public void close() throws Exception { } /** - * 收集脏数据放入队列缓存中，记录放入失败的数目和存入队列中的总数目，如果放入失败的数目超过一定比列，那么manager任务失败 + * 收集脏数据放入队列缓存中，记录放入失败的数目和存入队列中的总数目，如果放入失败的数目超过一定比例，那么manager任务失败 */ public void collectDirtyData(String dataInfo, String cause, String field) { DirtyDataEntity dirtyDataEntity = new DirtyDataEntity(dataInfo, System.currentTimeMillis(), cause, field); @@ -124,6 +137,7 @@ public void collectDirtyData(String dataInfo, String cause, String field) { } catch (Exception ignored) { LOG.warn("dirty Data insert error ... Failed number: " + errorCount.incrementAndGet()); LOG.warn("error dirty data:" + dirtyDataEntity.toString()); + // TODO 这个失败比例可以作出调整 if (errorCount.get() > Math.ceil(count.longValue() * 0.8)) { throw new RuntimeException("The number of failed number reaches the limit, manager fails"); } diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java index b6b2b6c8f..635f72b64 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java @@ -9,15 +9,25 @@ * Date 2020/8/28 星期五 */ public class TestMain { + private static final Integer DATA_NUMBER = 10000; + public static void main(String[] args) throws Exception { - Map properties = new HashMap<>(); - properties.put("type", "console"); - properties.put("pluginPath", "/Users/wtz/IdeaProjects/flinkStreamSQL/sqlplugins"); + Map properties = new HashMap<>(8); + properties.put("type", "mysql"); + properties.put("pluginPath", "/Users/wtz/IdeaProjects/flinkStreamSQLTemp/sqlplugins"); + properties.put("url", "jdbc:mysql://kerberos01:3306/tiezhu"); + properties.put("userName", "dtstack"); + properties.put("password", "abc123"); + properties.put("isCreatedTable", "false"); + properties.put("batchSize", "1"); + properties.put("tableName", "DirtyDataFromMysql_2020_09_14_10_51_50"); DirtyDataManager manager = DirtyDataManager.newInstance(properties); - for (int i = 0; i < 100; i++) { - Thread.sleep(1000); - manager.collectDirtyData("testDirtyData", new Exception("testException").getMessage(), "testField"); + for (int i = 0; i < DATA_NUMBER; i++) { + Thread.sleep(100); + manager.collectDirtyData("testDirtyData" + i, + new Exception("testException" + i).getMessage(), "testField"); } + manager.close(); } } diff --git a/dirtyDataManager/console/pom.xml b/dirtyDataManager/console/pom.xml index 438104e4e..93aa4696a 100644 --- a/dirtyDataManager/console/pom.xml +++ b/dirtyDataManager/console/pom.xml @@ -58,7 +58,7 @@ - + diff --git a/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java b/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java index 54bc19b82..574d5f573 100644 --- a/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java +++ b/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java @@ -19,12 +19,20 @@ package com.dtstack.flink.sql.dirty.mysql; import com.dtstack.flink.sql.dirtyManager.consumer.AbstractDirtyDataConsumer; +import com.dtstack.flink.sql.dirtyManager.entity.DirtyDataEntity; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.sql.Statement; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * @author tiezhu @@ -32,77 +40,106 @@ * Date 2020/8/27 星期四 */ public class MysqlDirtyDataConsumer extends AbstractDirtyDataConsumer { - //TODO 添加batchSize 和 定时任务 + private static final long serialVersionUID = -2959753658786001679L; private static final String DRIVER_NAME = "com.mysql.jdbc.Driver"; - private final Object LOCK_STR = new Object(); + private static final int CONN_VALID_TIME = 1000; - private boolean isCreatedTable = false; + private static final Integer FIELD_NUMBER = 5; - private final String[] tableField = {"id", "dirtyData", "processTime", "cause", "field"}; + private final Object LOCK_STR = new Object(); - private String SQL = "INSERT INTO ? (?, ?, ?, ?) VALUES (?, ?, ?, ?) "; + private final String[] tableField = {"id", "dirtyData", "processTime", "cause", "field"}; private PreparedStatement statement; private Connection connection; - private String tableName; + private Long batchSize; - private void setStatement(String url, - String userName, - String password) throws ClassNotFoundException, SQLException { + private void beforeConsume(String url, + String userName, + String password, + String tableName, + boolean isCreatedTable) throws ClassNotFoundException, SQLException { synchronized (LOCK_STR) { Class.forName(DRIVER_NAME); - connection = DriverManager.getConnection(url, userName, password); - statement = connection.prepareStatement(SQL); + + // create table for dirty data + if (!isCreatedTable) { + createTable(tableName); + } + + String insertField = Arrays.stream(tableField) + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")); + String insertSql = "INSERT INTO " + quoteIdentifier(tableName) + + "(" + insertField + ") VALUES (?, ?, ?, ?, ?)"; + statement = connection.prepareStatement(insertSql); } } private String quoteIdentifier(String tableName) { - return "\"" + tableName + "\""; + return "`" + tableName + "`"; } /** * 创建存储脏数据的表 * * @param tableName 表名 - * @return 是否创建成功 * @throws SQLException SQL异常 */ - private boolean createTable(String tableName) { + private void createTable(String tableName) throws SQLException { + Statement statement = null; try { - String defaultTable = ""; String sql = - "CREATE TABLE ` " + tableName + "` (" + + "CREATE TABLE IF NOT EXISTS \n" + + quoteIdentifier(tableName) + " (\n" + " `id` int(11) not null AUTO_INCREMENT,\n" + - " `dirtyData` varchar(100) DEFAULT NULL,\n" + - " `processTime` varchar(100) DEFAULT NULL,\n" + - " `cause` date DEFAULT NULL,\n" + - " `field` varchar(100) DEFAULT NULL,\n" + + " `dirtyData` varchar(255) DEFAULT NULL,\n" + + " `processTime` varchar(255) DEFAULT NULL,\n" + + " `cause` varchar(255) DEFAULT NULL,\n" + + " `field` varchar(255) DEFAULT NULL,\n" + " PRIMARY KEY (id)\n" + ") DEFAULT CHARSET=utf8;"; - return statement.execute(sql); + statement = connection.createStatement(); + statement.execute(sql); } catch (SQLException e) { throw new RuntimeException("create table error !", e); + } finally { + if (statement != null && !statement.isClosed()) { + statement.close(); + } } } @Override public void consume() throws Exception { - if (!isCreatedTable) { - createTable(tableName); + DirtyDataEntity entity = queue.take(); + count++; + List data = new ArrayList<>(); + data.add(String.valueOf(count)); + Collections.addAll(data, entity.get()); + for (int i = 0; i < FIELD_NUMBER; i++) { + statement.setString(i + 1, data.get(i)); } + statement.addBatch(); + + if (count % batchSize == 0) { + statement.executeBatch(); + } } @Override public void close() { + isRunning.compareAndSet(true, false); + try { - if (connection != null && !connection.isValid(1000)) { + if (connection != null && !connection.isValid(CONN_VALID_TIME)) { connection.close(); } @@ -116,11 +153,18 @@ public void close() { @Override public void init(Map properties) throws Exception { - tableName = properties.get("tableName"); + SimpleDateFormat timeFormat = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss"); + String tableName = properties.getOrDefault("tableName", + "DirtyDataFromMysql_" + timeFormat.format(System.currentTimeMillis())); String userName = properties.get("userName"); String password = properties.get("password"); String url = properties.get("url"); - isCreatedTable = Boolean.parseBoolean(properties.get("isCreatedTable")); - setStatement(url, userName, password); + batchSize = Long.parseLong(properties.getOrDefault("batchSize", "10000")); + errorLimit = Long.parseLong(properties.getOrDefault("errorLimit", "1000")); + + boolean isCreatedTable = Boolean.parseBoolean( + properties.getOrDefault("isCreatedTable", "false")); + + beforeConsume(url, userName, password, tableName, isCreatedTable); } } From dbbbec0525277e587e8a2697b5615fcf6411aef4 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 14 Sep 2020 14:00:31 +0800 Subject: [PATCH 10/80] [opt] code opt --- .../consumer/AbstractDirtyDataConsumer.java | 11 +++++----- .../manager/DirtyDataManager.java | 21 ++++++++++++------- .../sql/dirtyManager/manager/TestMain.java | 6 ++++-- .../dirty/print/ConsoleDirtyDataConsumer.java | 2 +- .../dirty/mysql/MysqlDirtyDataConsumer.java | 7 ++++--- 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java index f9d913fdf..59e365f3f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; /** * @author tiezhu @@ -38,9 +39,9 @@ public abstract class AbstractDirtyDataConsumer implements Runnable, Serializabl protected static final Logger LOG = LoggerFactory.getLogger(AbstractDirtyDataConsumer.class); protected Long errorLimit = 1000L; - protected Long errorCount = 0L; + protected AtomicLong errorCount = new AtomicLong(0L); - protected long count = 0; + protected AtomicLong count = new AtomicLong(0L); public AtomicBoolean isRunning = new AtomicBoolean(true); @@ -82,9 +83,9 @@ public void run() { } } catch (Exception e) { LOG.error("consume dirtyData error", e); - errorCount++; - if (errorCount.equals(errorLimit)) { - throw new RuntimeException("脏数据消费失败达到上限，任务失败"); + errorCount.incrementAndGet(); + if (errorCount.get() == errorLimit) { + throw new RuntimeException("The task failed due to the number of dirty data consume failed reached the limit " + errorLimit); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java index c3953e02a..921bd1732 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java @@ -55,7 +55,7 @@ public class DirtyDataManager implements Serializable { /** * 写入队列阻塞时间 */ - private long blockingInterval = 60; + private long blockingInterval; /** * 缓存脏数据信息队列 @@ -72,6 +72,8 @@ public class DirtyDataManager implements Serializable { */ private final AtomicLong errorCount = new AtomicLong(0); + private double errorLimitRate; + public static AbstractDirtyDataConsumer consumer; private static ThreadPoolExecutor dirtyDataConsumer; @@ -82,12 +84,17 @@ public class DirtyDataManager implements Serializable { private final static String DEFAULT_TYPE = "console"; + private final static String DEFAULT_ERROR_LIMIT_RATE = "0.8"; + + private final static String DEFAULT_BLOCKING_INTERVAL = "60"; + /** * 通过参数生成manager实例，并同时将consumer实例化 */ public static DirtyDataManager newInstance(Map properties) throws Exception { DirtyDataManager manager = new DirtyDataManager(); - manager.blockingInterval = Long.parseLong(properties.getOrDefault("blockingInterval", "60")); + manager.blockingInterval = Long.parseLong(properties.getOrDefault("blockingInterval", DEFAULT_BLOCKING_INTERVAL)); + manager.errorLimitRate = Double.parseDouble(properties.getOrDefault("errorLimitRate", DEFAULT_ERROR_LIMIT_RATE)); consumer = createConsumer(properties); consumer.init(properties); consumer.setQueue(manager.queue); @@ -119,11 +126,10 @@ private static AbstractDirtyDataConsumer createConsumer(Map prop */ public void close() throws Exception { dirtyDataConsumer.shutdown(); - if (!queue.isEmpty() && checkConsumer()) { - consumer.consume(); + if (checkConsumer()) { + LOG.info("dirty consumer is closing ..."); + consumer.close(); } - LOG.info("dirty consumer is closing ..."); - consumer.close(); } /** @@ -137,8 +143,7 @@ public void collectDirtyData(String dataInfo, String cause, String field) { } catch (Exception ignored) { LOG.warn("dirty Data insert error ... Failed number: " + errorCount.incrementAndGet()); LOG.warn("error dirty data:" + dirtyDataEntity.toString()); - // TODO 这个失败比例可以作出调整 - if (errorCount.get() > Math.ceil(count.longValue() * 0.8)) { + if (errorCount.get() > Math.ceil(count.longValue() * errorLimitRate)) { throw new RuntimeException("The number of failed number reaches the limit, manager fails"); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java index 635f72b64..01a9f0a02 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java @@ -9,7 +9,7 @@ * Date 2020/8/28 星期五 */ public class TestMain { - private static final Integer DATA_NUMBER = 10000; + private static final Integer DATA_NUMBER = 1000; public static void main(String[] args) throws Exception { Map properties = new HashMap<>(8); @@ -27,7 +27,9 @@ public static void main(String[] args) throws Exception { Thread.sleep(100); manager.collectDirtyData("testDirtyData" + i, new Exception("testException" + i).getMessage(), "testField"); + if (i == 500) { + manager.close(); + } } - manager.close(); } } diff --git a/dirtyDataManager/console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java b/dirtyDataManager/console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java index bf7d20156..59f6151e2 100644 --- a/dirtyDataManager/console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java +++ b/dirtyDataManager/console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java @@ -38,7 +38,7 @@ public class ConsoleDirtyDataConsumer extends AbstractDirtyDataConsumer { @Override public void consume() throws InterruptedException { DirtyDataEntity dataEntity = queue.take(); - count++; + count.incrementAndGet(); LOG.warn("get dirtyData: " + dataEntity.getDirtyData() + "\n" + "cause: " + dataEntity.getCause() + "\n" + "processTime: " + dataEntity.getProcessDate() + "\n" diff --git a/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java b/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java index 574d5f573..00edb7ad3 100644 --- a/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java +++ b/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java @@ -119,9 +119,10 @@ private void createTable(String tableName) throws SQLException { @Override public void consume() throws Exception { DirtyDataEntity entity = queue.take(); - count++; + count.incrementAndGet(); + List data = new ArrayList<>(); - data.add(String.valueOf(count)); + data.add(String.valueOf(count.get())); Collections.addAll(data, entity.get()); for (int i = 0; i < FIELD_NUMBER; i++) { statement.setString(i + 1, data.get(i)); @@ -129,7 +130,7 @@ public void consume() throws Exception { statement.addBatch(); - if (count % batchSize == 0) { + if (count.get() % batchSize == 0) { statement.executeBatch(); } } From 67fc5a480af925b8ddb0292fa89dcc444953052b Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 14 Sep 2020 17:54:15 +0800 Subject: [PATCH 11/80] [fix] fix some bug --- .../flink/sql/dirtyManager/entity/DirtyDataEntity.java | 10 ++++++---- .../sql/dirtyManager/manager/DirtyDataManager.java | 5 +++-- .../flink/sql/dirtyManager/manager/TestMain.java | 2 +- dirtyDataManager/console/pom.xml | 2 +- dirtyDataManager/mysql/pom.xml | 2 +- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java index a06a98356..3d8d96653 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.dirtyManager.entity; import java.sql.Date; +import java.text.SimpleDateFormat; /** * @author tiezhu @@ -26,6 +27,7 @@ * Date 2020/8/27 星期四 */ public class DirtyDataEntity { + private final SimpleDateFormat timeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); /** * 脏数据信息内容 */ @@ -34,7 +36,7 @@ public class DirtyDataEntity { /** * 脏数据处理时间 */ - private Date processDate; + private String processDate; /** * 产生脏数据的原因 @@ -54,11 +56,11 @@ public void setDirtyData(String dirtyData) { this.dirtyData = dirtyData; } - public Date getProcessDate() { + public String getProcessDate() { return processDate; } - public void setProcessDate(Date processDate) { + public void setProcessDate(String processDate) { this.processDate = processDate; } @@ -80,7 +82,7 @@ public void setField(String field) { public DirtyDataEntity(String dirtyData, Long processDate, String cause, String field) { this.dirtyData = dirtyData; - this.processDate = new Date(processDate); + this.processDate = timeFormat.format(processDate); this.cause = cause; this.field = field; } diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java index 921bd1732..88e2b24cc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java @@ -123,12 +123,13 @@ private static AbstractDirtyDataConsumer createConsumer(Map prop /** * 脏数据收集任务停止，任务停止之前，需要将队列中所有的数据清空 + * TODO consumer 关闭时仍有数据没有消费到，假如有500条数据，在结束时实际消费数量可能只有493 */ - public void close() throws Exception { - dirtyDataConsumer.shutdown(); + public void close() { if (checkConsumer()) { LOG.info("dirty consumer is closing ..."); consumer.close(); + dirtyDataConsumer.shutdownNow(); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java index 01a9f0a02..8ec2296c1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java @@ -27,7 +27,7 @@ public static void main(String[] args) throws Exception { Thread.sleep(100); manager.collectDirtyData("testDirtyData" + i, new Exception("testException" + i).getMessage(), "testField"); - if (i == 500) { + if (i == 50) { manager.close(); } } diff --git a/dirtyDataManager/console/pom.xml b/dirtyDataManager/console/pom.xml index 93aa4696a..a9d0c1fc5 100644 --- a/dirtyDataManager/console/pom.xml +++ b/dirtyDataManager/console/pom.xml @@ -8,7 +8,7 @@ 1.0-SNAPSHOT 4.0.0 - + dirtyConsumer-console dirtyConsumer.console diff --git a/dirtyDataManager/mysql/pom.xml b/dirtyDataManager/mysql/pom.xml index bd786052e..07b6b2699 100644 --- a/dirtyDataManager/mysql/pom.xml +++ b/dirtyDataManager/mysql/pom.xml @@ -8,7 +8,7 @@ 1.0-SNAPSHOT 4.0.0 - + dirtyConsumer-mysql 5.1.46 From 1a02a6a9859b2b5e7cb5f721c21a423a6f9090a2 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 15 Sep 2020 11:48:39 +0800 Subject: [PATCH 12/80] delete gradle --- .../6.1.1/executionHistory/executionHistory.bin | Bin 20147 -> 0 bytes .../executionHistory/executionHistory.lock | Bin 17 -> 0 bytes .gradle/6.1.1/fileChanges/last-build.bin | Bin 1 -> 0 bytes .gradle/6.1.1/fileHashes/fileHashes.bin | Bin 18697 -> 0 bytes .gradle/6.1.1/fileHashes/fileHashes.lock | Bin 17 -> 0 bytes .gradle/6.1.1/gc.properties | 0 .../buildOutputCleanup/buildOutputCleanup.lock | Bin 17 -> 0 bytes .gradle/buildOutputCleanup/cache.properties | 2 -- .gradle/buildOutputCleanup/outputFiles.bin | Bin 18695 -> 0 bytes .gradle/checksums/checksums.lock | Bin 17 -> 0 bytes .gradle/vcs-1/gc.properties | 0 gradle/wrapper/gradle-wrapper.jar | Bin 58695 -> 0 bytes gradle/wrapper/gradle-wrapper.properties | 5 ----- 13 files changed, 7 deletions(-) delete mode 100644 .gradle/6.1.1/executionHistory/executionHistory.bin delete mode 100644 .gradle/6.1.1/executionHistory/executionHistory.lock delete mode 100644 .gradle/6.1.1/fileChanges/last-build.bin delete mode 100644 .gradle/6.1.1/fileHashes/fileHashes.bin delete mode 100644 .gradle/6.1.1/fileHashes/fileHashes.lock delete mode 100644 .gradle/6.1.1/gc.properties delete mode 100644 .gradle/buildOutputCleanup/buildOutputCleanup.lock delete mode 100644 .gradle/buildOutputCleanup/cache.properties delete mode 100644 .gradle/buildOutputCleanup/outputFiles.bin delete mode 100644 .gradle/checksums/checksums.lock delete mode 100644 .gradle/vcs-1/gc.properties delete mode 100644 gradle/wrapper/gradle-wrapper.jar delete mode 100644 gradle/wrapper/gradle-wrapper.properties diff --git a/.gradle/6.1.1/executionHistory/executionHistory.bin b/.gradle/6.1.1/executionHistory/executionHistory.bin deleted file mode 100644 index a191b4fe20f13b943ea709880699c920aae01c63..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20147 zcmeI%%}W$v90%|jCu+&WB~*-%E*1zmvoG$hhC#}j3RdQ72_44SnP+#$eVd+{by0{e zA-#yT5D^63RH9Hy2oI5m4v_>!r=m-LKnD%#6x8g_4l3KhwLL_92WIABW}ffwc|QB> z>|uip)9CzF+?dU+ezS=d1Rwwb2tWV=5P$##AOHafKmY;|fB*y_009U<00RGuz=%1K z{nlg}k5}azwONL79WWcLOFleWRWtw4y4}}h|9@bg&Fd}JJscqb0SG_<0uX=z1Rwwb z2tWV=5P$##AOHafKmY;|fB*y_009U<00Izz00bZa0SG_<0uX=z1Rwwb2tWV=5cn4a zybQBfHFqh$KZt9Wvm~Azm!_fTgRYt+8^in58t0@=+=&3BpU7${Ug% z*ZC}!G>uUHqIDL7L{3x5Ac}MGarM|-r|Zk37oP?{)K5pN{yKqq&5GR_HKBxLs%nPV zEYeI$loM$^Ly5cDZh1skTOb^9dzFZb1iTTVcmi!6$w!n9zc&!*a1m$C8avwbq#-w# z+6XUnH7q-3Br3<$3DPa;#Id-OnKlnfMlAn$u6L*_IM5@GgnEX=zH?`K?5z9)+-wm@ zP*mN}bNM4x!x=+Orz%lvTi_oG-rHI*6q7tYe<+h=wsjkx)QIExj^-TPGW>kEYu){4 z?`$nGLKEC-%rG?lgdpgIPN*``?NwuzC@|Onx?nA*RzL6N-K}9Y#ZRf)tab4Mp}L90 zO&4x5duGCtA;+vwo?RA3%y3=E8dE~QLZm^O9w)M)3z3AHiiZqJq-5y)Kruhdn|(Rw zs!;2D`*M9DdBm}C>D2tnZc=vBr=;-pafzN*6NG)kmNgH-UQ0zb`@3hxan7qZK0p0l zeKJ0|lG6_Mep{(vcm<E`;!_D=TD*zmiT zY{hF-KCxubxAek0uVA!P)~d;J&Xy}jjeWlRb)o3bx@T)Y+SxJ_%6-1Hp!p+{`V9!r Bkg)&& diff --git a/.gradle/6.1.1/executionHistory/executionHistory.lock b/.gradle/6.1.1/executionHistory/executionHistory.lock deleted file mode 100644 index af7ea0a9bab9521238aba7486ef526f312a4c6bd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17 TcmZRs2>pJ8Ra8id0Ror-C;bBH diff --git a/.gradle/6.1.1/fileChanges/last-build.bin b/.gradle/6.1.1/fileChanges/last-build.bin deleted file mode 100644 index f76dd238ade08917e6712764a16a22005a50573d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1 IcmZPo000310RR91 diff --git a/.gradle/6.1.1/fileHashes/fileHashes.bin b/.gradle/6.1.1/fileHashes/fileHashes.bin deleted file mode 100644 index ebcb5d2291320ab033c01022fc6819cf388779eb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18697 zcmeI%Pbfq|0LSrp+m>b3+qR{3kPunbJk8b)-&70{r{hH=;eG@`NL*L#I)zMVlvO@p?1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~f&U^fCm+(F9uw{Jxf?Sl#89K$_z#U-KK$Hqq>iWZ z`v1Z8R`0Z(@z;C#n9hBjOLyj1%JA}4od@ib(}l^IelNe*xn5Y6eR;w@1_#$aqCjwH?uclr=&=7l3^G9)ZcL;_YHAYtK}%aoZBavu zLek_=yVz2+g=5rR_dH0n1wzF49qwMf?;UsdS?^siCOP&Y!}LeIKV^mh0tg_000Iag zfB*srAbVB>0%*OvsU zoOa{Y>y^@+bVqx$l5r01-u9$twYSRCt9C9tEPY=4eY)J)Ei|;d+FyIsEtAjI5HYubnG`o+fL+N|`R_C#^}d?#fWq~B^^zuezk&b1z-SF{(OZ`xN+CsFB# z+VyOVH4&_FL;wK<5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** s5I_I{1Q0*~0R#|0;BN|~z1Q+L>ie7TE&nk3t$1NT<#*5edDmC@1j#3#{Qv*} diff --git a/.gradle/checksums/checksums.lock b/.gradle/checksums/checksums.lock deleted file mode 100644 index 8adaf1c201b0d4720183021e1990da996429d551..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17 ScmZQ}{cr5!Z_6Ic00jUeuL7+A diff --git a/.gradle/vcs-1/gc.properties b/.gradle/vcs-1/gc.properties deleted file mode 100644 index e69de29bb..000000000 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index f3d88b1c2faf2fc91d853cd5d4242b5547257070..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 58695 zcma&OV~}Oh(k5J8>Mq;vvTfV8ZQE5{wr$(iDciPf+tV}m-if*I+;_h3N1nY;M6TF7 zBc7A_WUgl&IY|&uNFbnJzkq;%`2QLZ5b*!{1OkHidzBVe;-?mu5upVElKVGD>pC88 zzP}E3wRHBgaO?2nzdZ5pL;m-xf&RU>buj(E-s=DK zf%>P9se`_emGS@673tqyT^;o8?2H}$uO&&u^TlmHfPgSSfPiTK^AZ7DTPH`Szw4#- z&21E&^c|dx9f;^@46XDX9itS+ZRYuqx#wG*>5Bs&gxwSQbj8grds#xkl;ikls1%(2 zR-`Tn(#9}E_aQ!zu~_iyc0gXp2I`O?erY?=JK{M`Ew(*RP3vy^0=b2E0^PSZgm(P6 z+U<&w#)I=>0z=IC4 zh4Q;eq94OGttUh7AGWu7m){;^Qk*5F6eTn+Ky$x>9Ntl~n0KDzFmB0lBI6?o!({iX zQt=|-9TPjAmCP!eA{r|^71cIvI(1#UCSzPw(L2>8OG0O_RQeJ{{MG)tLQ*aSX{AMS zP-;|nj+9{J&c9UV5Ww|#OE*Ah6?9WaR?B04N|#`m0G-IqwdN~Z{8)!$@UsK>l9H81 z?z`Z@`dWZEvuABvItgYLk-FA(u-$4mfW@2(Eh(9fe`5?WUda#wQa54 z3dXE&-*@lsrR~U#4NqkGM7Yu4#pfGqAmxmGr&Ep?&MwQ9?Z*twtODbi;vK|nQ~d_N z;T5Gtj_HZKu&oTfqQ~i`K!L||U1U=EfW@FzKSx!_`brOs#}9d(!Cu>cN51(FstP_2dJh>IHldL~vIwjZChS-*KcKk5Gz zyoiecAu;ImgF&DPrY6!68)9CM-S8*T5$damK&KdK4S6yg#i9%YBH>Yuw0f280eAv3 za@9e0+I>F}6&QZE5*T8$5__$L>39+GL+Q(}j71dS!_w%B5BdDS56%xX1~(pKYRjT; zbVy6V@Go&vbd_OzK^&!o{)$xIfnHbMJZMOo``vQfBpg7dzc^+&gfh7_=oxk5n(SO3 zr$pV6O0%ZXyK~yn++5#x`M^HzFb3N>Vb-4J%(TAy#3qjo2RzzD*|8Y} z7fEdoY5x9b3idE~-!45v?HQ$IQWc(c>@OZ>p*o&Om#YU904cMNGuEfV=7=&sEBWEO z0*!=GVSv0>d^i9z7Sg{z#So+GM2TEu7$KXJ6>)Bor8P5J(xrxgx+fTLn1?Jlotz*U z(ekS*a2*ml5ft&R;h3Gc2ndTElB!bdMa>UptgIl{pA+&b+z_Y&aS7SWUlwJf-+PRv z$#v|!SP92+41^ppe}~aariwztUtwKA8BBLa5=?j3@~qHfjxkvID8CD`t5*+4s|u4T zLJ9iEfhO4YuAl$)?VsWcln|?(P=CA|!u}ab3c3fL8ej9fW;K|@3-c@y4I;^8?K!i0 zS(5Cm#i85BGZov}qp+<-5!Fh+KZev3(sA2D_4Z~ZLmB5B$_Yw2aY{kA$zuzggbD{T zE>#yd3ilpjM4F^dmfW#p#*;@RgBg{!_3b6cW?^iYcP!mjj!}pkNi{2da-ZCD2TKKz zH^x^+YgBb=dtg@_(Cy33D|#IZ&8t?w8$E8P0fmX#GIzq~w51uYmFs{aY76e0_~z2M z(o%PNTIipeOIq(H5O>OJ*v8KZE>U@kw5(LkumNrY>Rv7BlW7{_R9v@N63rK)*tu|S zKzq|aNs@81YUVZ5vm>+pc42CDPwQa>oxrsXkRdowWP!w?=M(fn3y6frEV*;WwfUV$s31D!S_;_~E@MEZ>|~wmIr05#z2J+& zBme6rnxfCp&kP@sP)NwG>!#WqzG>KN7VC~Gdg493So%%-P%Rk!<|~-U|L3VASMj9K zk(Pfm1oj~>$A>MFFdAC8M&X0i9-cV7Q($(R5C&nR5RH$T&7M=pCDl`MpAHPOha!4r zQnYz$7B1iLK$>_Ai%kZQaj-9)nH$)tESWUSDGs2|7plF4cq1Oj-U|+l4Ga}>k!efC z*ecEudbliG+%wI8J#qI!s@t%0y9R$MBUFB)4d47VmI`FjtzNd_xit&l1T@drx z&4>Aj<2{1gUW8&EihwT1mZeliwrCN{R|4@w4@@Btov?x5ZVzrs&gF0n4jGSE33ddUnBg_nO4Zw)yB$J-{@a8 z);m%fvX2fvXxogriNb}}A8HxA)1P-oK+Da4C3pofK3>U_6%DsXFpPX}3F8O`uIpLn zdKjq(QxJTJ4xh->(=lxWO#^XAa~<7UxQl8~8=izS!TcPmAiBP5Et7y?qEbFd9Q=%IJ;%Kn$lto-~3`}&`x=AVS+Uo7N*hbUxhqVH_w^sn!74z{Ka#*U6s z=8jIrHpUMBC@@9Jn~GS<$lse*EKuX%3Swl5&3~GiK_$vn8Vjqe{mjhBlH}m4I8qK+ ztU50COh7)d-gXpq-|}T;biGa^e=VjxjjFuoGIA8`2jJ}wNBRcsx24?7lJ7W4ksNPv zA7|gcXT@~7KTID#0|EX#OAXvgaBJ8Jg!7X#kc1^Tvl;I(=~(jtn-(5bhB=~J^w5bw z8^Hifeupm;nwsSDkT{?x?E(DgLC~Nh8HKQGv`~2jMYrz9PwS^8qs3@nz4ZBCP5}%i z=w}jr2*$X-f(zDhu%D8(hWCpix>TQpi{e`-{p^y?x4?9%)^wWc?L}UMcfp~lL|;g) zmtkcXGi9#?cFOQQi_!Z8b;4R%4y{$SN~fkFedDJ&3eBfHg|DRSx09!tjoDHgD510Z z_aJLHdS&7;Dl;X|WBVyl_+d+2_MK07^X1JEi_)v$Z*ny-()VrD6VWx|Un{)gO0*FQ zX{8Ss3JMrV15zXyfCTsVO@hs49m&mN(QMdL3&x@uQqOyh2gnGJYocz0G=?BX7qxA{ zXe0bn4ij^;wfZfnRlIYkWS^usYI@goI9PccI>}Ih*B!%zv6P$DoXsS%?G)|HHevkG z>`b#vtP=Lx$Ee(t??%_+jh(nuc0Q&mCU{E3U z1NqNK!XOE#H2Pybjg0_tYz^bzX`^RR{F2ML^+<8Q{a;t(#&af8@c6K2y2m zP|parK=qf`I`#YxwL=NTP>tMiLR(d|<#gEu=L-c!r&(+CpSMB5ChYW1pUmTVdCWw|!Ao?j&-*~50S`=) z9#Knf7GPA19g%Y7wip@`nj$aJcV|SakXZ*Q2k$_SZlNMx!eY8exF;navr&R)?NO9k z#V&~KLZ0c9m|Mf4Gic}+<=w9YPlY@|Pw*z?70dwOtb<9-(0GOg>{sZaMkZc9DVk0r zKt%g5B1-8xj$Z)>tWK-Gl4{%XF55_Ra3}pSY<@Y&9mw`1jW8|&Zm{BmHt^g=FlE{` z9Lu7fI2v3_0u~apyA;wa|S4NaaG>eHEw&3lNFVd_R9E=Y? zgpVQxc9{drFt2pP#ZiN~(PL%9daP4pWd*5ABZYK{a@e&Vb`TYiLt$1S>KceK36Ehz z;;MI%V;I`#VoSVAgK3I%-c>ViA>nt=5EZ zjr$Jv~$_vg<$q<@CpZ1gdqP_3v^)uaqZ`?RS_>f(pWx3(H;gWpjR?W8L++YPW;)Vw3)~tozdySrB3A2;O<%1F8?Il4G|rO0mEZYHDz!?ke!$^bEiWRC1B%j~ws0+hHS;B8l5Wh)e+Ms7f4M4CbL%Q_*i~cP}5-B(UkE&f7*pW6OtYk5okQCEoN4v|7;(+~~nyViqo5 z(bMGQi$)KN6EmfVHv4pf2zZMJbcAKyYy>jY@>LB5eId|2Vsp{>NMlsee-tmh({;@b z@g;wiv8@a1qrDf-@7$(MR^M^*dKYBewhIDFX%;*8s zR#u?E;DJO;VnTY6IfbO=dQ61V0DisUAs4~t|9`9ZE(jG}ax#-xikDhsO_4^RaK ziZ?9AJQP_{9WuzVk^s_U+3V8gOvVl5(#1>}a|RL>};+uJB%nQM-J>M4~yK)cioytFXtnmOaJZSiE+3g}C`Im~6H z*+-vjI>ng5w>>Y!L(+DwX2gs0!&-BFEaDie4i5ln*NGP$te7$F9iUlJl4`XpkAsPm z0l?GQ17uN^=g~u1*$)S`30xL%!`LW*flwT*#svAtY(kHXFfvA`dj*pDfr0pBZ`!La zWmX$Z@qyv|{nNsRS|+CzN-Pvb>47HEDeUGFhpp5C_NL0Vp~{Wc{bsm_5J!#tuqW@? z)Be zb&Gj&(l*bHQDq7w-b`F9MHEH*{Dh~0`Gn8t`pz}!R+q~4u$T@cVaUu`E^%0f-q*hM z1To6V31UGJN7a-QW5;nhk#C26vmHyjTVZkdV zqYMI9jQY)3oZt=V0L7JZQ=^c2k){Y_lHp&V_LIi*iX^Ih3vZ_K<@Di(hY<&g^f?c$wwF-wX1VLj>ZC4{0#e`XhbL_$a9uXS zKph*4LupSV2TQBCJ4AfOXD8fs2;bAGz-qU4=Qj$^1ZJX z2TtaVdq>OjaWGvv9)agwV)QW9eTZ-xv`us2!yXSARnD5DwX_Vg*@g4w!-zT|5<}-7 zsnllGRQz>k!LwdU`|i&!Bw^W7CTUU3x`Zg8>XgHj=bo!cd<#pI8*pa*1N`gg~I0ace!wzZoJ)oGScm~D_Sc;#wFed zUo;-*0LaWVCC2yqr6IbeW3`hvXyMfAH94qP2|cN``Z%dSuz8HcQ!WT0k38!X34<6l zHtMV%4fH5<6z-lYcK;CTvzzT6-^xSP>~a*8LfbByHyp$|X*#I6HCAi){gCu1nvN%& zvlSbNFJRCc&8>f`$2Qa`fb@w!C11v1KCn)P9<}ei0}g*cl~9A9h=7(}FO!=cVllq3 z7nD)E%gt;&AYdo{Ljb2~Fm5jy{I><%i*GUlU8crR4k(zwQf#nima@xb%O71M#t-4< z(yjX(m^mp_Y;5()naqt2-VibylPS)Oof9uBp$3Gj`>7@gjKwnwRCc>rx%$esn);gI z5B9;~uz57n7Rpm8K^o=_sFPyU?>liHM&8&#O%f)}C5F7gvj#n#TLp@!M~Q?iW~lS}(gy%d&G3p?iBP z(PZQUv07@7!o3~1_l|m5m;Xr)^QK_JaVAY3v1UREC*6>v;AT$BO`nA~KZa1x3kV2F z%iwG7SaaAcT8kalCa^Hg&|eINWmBQA_d8$}B+-Q_@6j_{>a- zwT3CMWG!A}Ef$EvQsjK>o)lJ;q!~#F%wo`k-_mT=+yo%6+`iGe9(XeUl;*-4(`G;M zc@+ep^Xv&<3e7l4wt48iwaLIC1RhSsYrf6>7zXfVD zNNJ1#zM;CjKgfqCabzacX7#oEN{koCnq1-stV+-CMQ=ZX7Fpd*n9`+AEg9=p&q7mTAKXvcbo?$AVvOOp{F>#a;S?joYZl_f}BECS%u&0x!95DR;|QkR9i}`FEAsPb=)I z8nb=4iwjiLRgAF}8WTwAb^eA>QjL4Srqb#n zTwx^-*Z38Uzh@bX$_1tq>m{o8PBX*t3Lqaf$EBqiOU*2NFp{LJX#3}p9{|v{^Hg4f zlhllKI>F+>*%mu6i9V7TT*Wx-zdK z(p8faUOwGOm5mBC%UGA1jO0@IKkG;i&+6Ur8XR2ZuRb$*a}R^-H6eKxcYodlXsF`& z{NkO+;_Yh-Ni@vV9iyzM43Yibn;oC7hPAzC24zs&+RYdY&r`3&&fg2hs62ysV^G`N zHMfBEFo8E3S$0C_m({bL8QCe$B@M{n1dLsaJYIU;(!n*V?0I1OvBB=iYh&`?u8 z&~n-$nbVIhO3mMhCQRlq%XRr1;Hvl=9E_F0sc9!VLnM>@mY~=Cx3K5}wxHKEZF9pC zIdyu1qucM!gEiomw7bW0-RwbX7?o=FE#K0l4`U2KhC8*kMWaEWJyVNZVu_tY2e&4F zb54Lh=Oz>(3?V$!ArXFXh8Cb3i;%KQGCrW$W#;kvx$YA2gofNeu?@nt>Yq8?2uJQp zUTo14hS%&dHF3Uhm~Z1>W)yb%&HoM!3z?%a%dmKT#>}}kKy2B=V3{Nu=bae%V%wU$ zb4%^m?&qn==QeHo`nAs3H}wtiK~!!&i|iBLfazh6!y9F)ToKNyE0B385!zq{p)5vB zvu`R#ULIS|2{3w52c*c$4}Pe>9Fw&U^>Bb_LUWn!xPx3X-uQsv(b1XFvFzn#voq0* z5~o`V_G805QXdgAOwOjoqmZ?uzwBVYSNP0Ie8FL`P0VK1J4CzV@t&%0duHB{;yIL$FZ9 zz#s#%ZG6ya&AwE;0_~^$1K

Hnj76Oym1QVh(3qRgs)GmgnEt-KxP|nCFY3uezZn zmtR0CZ$Z_-+f07?lu_tr~IC{&U6+QOth>ZgYk4V2FI$B2V3`M`Jk zsr>>lupymPeK129PfpDt9?GA2;I>03Ktz8NxwvTroqu8oaRB&bXT}G=^2UyOW}(4H z;9sG^YwV8K7pC&&viM^X_pfeFoN!cIhrE>OPQ5E<4KKDyPhRV^BGb_^Y6GO6#w}c= zu`0fC-@F4qXQtnB^nPmfI7Uw0bLhY^09TCO+H2(nvg8jdPjMAi4oSX%GP3oeo0`ks z%DoV|waU-Q7_libJCwnnOL9~LoapKqFPpZx?5FygX zsA~*ZR7X=@i{smf?fgxbcY6Y`JvD50P=R;Xv^sANPRp-Hc8n~Wb*gLIaoZJ2Q^CFe z_=G}y&{_NXT|Ob??}$cF7)$oPQMaeN_va1f%>C>V2E01uDU=h~<_fQKjtnl_aho2i zmI|R9jrNdhtl+q*X@}>l08Izz&UJygYkbsqu?4OOclV{GI5h98vfszu2QPiF?{Tvh19u_-C^+NjdAq!tq&Rd`ejXw#` z@U15c$Nmylco)Yj4kctX{L+lz$&CqTT5~}Q>0r-Xe!m5+?du6R&XY|YD5r5C-k*`s zOq-NOg%}RJr5ZWV4)?EO%XzZg&e8qVFQ?40r=8BI-~L%9T7@_{1X@<7RjboXqMzsV z8FiSINMjV*vC^FCv_;`jdJ-{U1<_xjZg4g?ek z4FtsapW_vFGqiGcGHP%?8US~Dfqi8^ZqtHx!}0%dqZFg%nQB)8`mE$~;1)Fb76nFk z@rK#&>2@@)4vO&gb{9&~R8-_{8qz6Rmw`4zeckD(L9xq}{r(fUO0Zh-R(d#x{<0j| z?6xZ2sp3mWnC}40B~g2QinHs1CZqZH&`+x2yBLT8hF7oWNIs_#YK2cyHO6AoGRG|RM>Hyn(ddpXFPAOGh~^0zcat`%&WoEQf9)!@l*3Tt@m>Lb z6$+$c!zsy_=%L9!_;jfd`?VXDd*^Vn%G>n~V9Vr6+_D@#E+dWB#&zAE+6xJeDMr1j zV+Tp~ht!M%^6f?)LBf8U1O4G#CutR07SB>8C&_&;g3TdIR#~e~qRtwd>&)|-ztJJ#4y0|UMjhJZlS8gA zAA260zUh+!$+xMfWKs|Lr23bcy#)JNnY|?WOka&wTS7_u%*N7PrMl1Lp9gxJY%CF? zz4IA@VVxX{knZPlNF+$9)>YIj#+(|$aflt=Wnforgn6`^3T+vaMmbshBjDi&tR(a7 zky~xCa77poRXPPam)@_UCwPdha^X~Aum=c0I@yTyD&Z!3pkA7LKr%Y6g%;~0<`{2& zS7W$AY$Kd}3Tg9CJgx=_gKR59zTMROsos?PU6&ocyCwCs8Qx1R%2#!&5c%~B+APu( z<1EXfahbm{XtOBK%@2a3&!cJ6R^g|2iLIN1)C2|l=;uj%tgSHoq2ojec6_4@6b<8BYG1h-Pm_V6dkRB!{T?jwVIIj&;~b7#%5Ew=0Fx zc(p7D1TT&e=hVt4spli}{J6tJ^}WL>sb`k}&gz+6It`Yz6dZdI53%$TR6!kSK2CfT*Q$`P30 z;$+G$D*C$U(^kkeY!OWn$j@IUu0_a{bZQ=TCbHD1EtmZ0-IBR<_3=tT%cz$>EE!V}pvfn7EMWs^971+XK}~kxSc_ATJJD$?)1Gz^Jq!>Hz#KkdCJ~jb-Y*Xv01_}}=T_V-A1<3O!V9Ezf z%Lnjihb3>=ZV}jSeqNu5AAdVbe|`;|p<%W#-<$s1oDYrB;C({psqV>ENkhadsC{cfEx=teVSB`?FOs+}d#pssxP z(ihudAVu3%%!*vOIWY11fn1M0&W|(|<2lEShz|#%W|wV2qM%#+P9NOy1x8jytHpfU zh;_L^uiL<<$L@~NpRXSrkJgdC>9R=>FmVu3^#C?3H>P{ue=mcv7lBmnfA?mB|L)EF zHv%Nl|D}0Tb~JVnv$ZysvbD8zw)>|5NpW3foe!QHipV9>Zy`|<5?O+rsBr*nZ4OE} zUytv%Rw7>^moSMsSU?@&a9+OdVgzWZnD>QXcUd{dd7vad+=0Hy)4|0A`}rpCx6cu!Ee5AM=iJ?|6=pG^>q(ExotyZP3(2PGhgg6-FkkQHS?nHX(yU0NG;4foCV|&)7 z1YK!bnv%#5n<25|CZ>4r1nK=D39qMzLAja*^#CN(aBbMx${?Iur3t=g2EMK|KwOF?I@W~0y`al&TGqJ zwf#~(?!>@#|JbDjQV9ct%+51l%q|lcY&f{FV&ACRVW*%VY6G5DzTpC!e%=T30mvav zRk$JOTntNoxRv>PDlJG1X=uep&???K00ep|l_#7=YZPuRHYoM46Z$O=ZZuGy_njgC z>P@gd+zKH5SjpWQ!h_r*!ol1s{9DS@sD4}xgFxaw>|av!xrKzg?rGnhZ#uZeU~iod z3-i*Hl@7cge0);y{DCVU(Ni1zg{yE&CxYT7)@zJ%ZZABj-Fh}0au^)*aw`vpmym;( z5|JZ!EACYenKNXH%=Md{my$sI3!8^FgtqkMcUR%w_)EBdP5DZ64aCIR%K99tId6SU ziT8Ef)K%7{XuIpPi}N+&FCm$elE>oKY;3c$x+*mXy?~wt6~?ss$HGqCm=YL2xzVTQ zr>*2_F;7j{5}NUPQ(aY0+h~rOKN|IA28L7^4XjX!L0C^vFB+3R5*1+s@k7;4d#U=5 zXTy8JN^_BCx1a4O3HMa9rf@?Fz>>dq}uvkY7!c?oksgs~xrpCo1{}^PD?w}Ug z3MbfBtRi z$ze~eRSLW^6bDJJeAt^5El{T*i1*v9wX{T7`a2wAVA z%j>3m*g^lc*~GOHFNy?h7>f7mPU*)3J>yPosaGkok}2#?wX5d$9moM~{NTzLznVhX zKa}bFQt#De`atoWzj4Lb@ZCud_T9rA@6VcmvW(+X?oIaH-FDbEg#0Slwf|7f!zUO( z7EUzpBOODL&w~(tNt0z|<9}Filev&4y;SQPp+?kIvJgnpc!^eYmsWz1)^n`LmP&Ui z-Oi1J2&O|$I<^V@g2Z91l3OArSbCkYAD0Tuw-O(INJJ>t%`DfIj}6%zmO+=-L{b!P zLRKvZHBT=^`60YuZon~D$;8UDlb-5l8J=1erf$H(r~ryWFN)+yY@a;=CjeUGNmexR zN)@)xaHmyp$SJcl>9)buKst5_+XomJu34&QMyS zQR(N@C$@%EmfWB8dFN(@Z%xmRma@>QU}!{3=E`wrRCQ~W=Dwb}*CW8KxAJ;v@TAs3 zW}Pq5JPc)(C8Rths1LR}Bgcf6dPOX<#X08^QHkznM-S>6YF(siF;pf~!@)O{KR4q1_c`T9gxSEf`_;a-=bg6=8W zQ&t`BK^gsK-E0Jp{^gW&8F9k?L4<#}Y0icYT2r+Dvg!bnY;lNNCj_3=N=yd9cM9kY zLFg|R0X;NRMY%zD*DbAmFV`(V@IANtz4^_32CH*)XCc$A>P-v49$k@!o$8%Ug>3-- z$#Fpo9J>eUMKg>Cn+T0H!n0Hf#avZX4pp54cv}YcutP+CmKC~a745-zhZp`KNms;J zS3S49WEyS8gCRAY|B~6yDh*cehY52jOSA#MZmk2dzu`_XpBXx9jDf!H3~!`n zaGe=)1VkfIz?*$T3t>-Pwhrw447idZxrsi;ks;(NF>uVl12}zI(N~2Gxi)8yDv-TLgbZ;L&{ax&TBv;m@z6RcbakF^el{!&)<___n#_|XR%jedxzfXG!a2Eyi)4g zYAWkYK{bQzhm|=>4+*SLTG2<#7g-{oB48b05=?PeW;Jo3ebWlo5y5|cl?p8)~PVZqiT^A~w-V*st8kV%%Et1(}x(mE0br-#hyPspVehofF`{gjFXla1lrqXJqQKE9M)8Xe0ZO&s$}Q zBTPjH>N!UU%bRFqaX(O9KMoG$Zy|xt-kCDjz(E*VDaI={%q? zURR{qi>G^wNteX|?&ZfhK-93KZlPXmGMsPd1o?*f_ej~TkoQ#no}~&#{O=>RadgtR zvig@~IZMsm3)vOr`>TGKD&fbRoB*0xhK7|R?Jh-NzkmR}H6lJiAZTIM1#AXE1LOGx zm7j;4b(Lu6d6GwtnsCvImB8%KJD+8z?W{_bDEB$ulcKP*v;c z*Ymsd)aP+t$dAfC-XnbwDx3HXKrB{91~O}OBx)fsb{s-qXkY<@QK7p-q-aaX&F?GS z2};`CqoNJ$<0DuM2!NCbtIpJ9*1a8?PH#bnF#xf~AYOIc4dx1Bw@K=)9bRX;ehYs; z$_=Ro(1!iIM=kZDlHFB>Ef46#rUwLM%)(#oAG(gYp>0tc##V{#aBl!q``!iIe1GBn z+6^G^5)(nr z8h#bm1ZzI450T?!EL)>RWX8VwT1X`2f;dW!{b~S>#$Pa~D6#Hp!;85XzluH%v5325 z730-aW?rY1!EAt;j7d23qfbMEyRZqxP};uID8xmG@mGw~3#2T^B~~14K5?&dP&H@r zL|aXJsEcAAXEXfu2d-!otZTV=if~^EQD*!NkUFQaheV&b-?-zH6JfjKO)aYN=Do*5 zYZ-@m#)5U0c&sUqu_%-Editr5#%Ne&bs)DxOj2_}`f;I_ReEY9U&Cf3rb>A3LK(ZD zid0_-3RfsS*t&g!zw}C_9u(_ze-vc1L59CdBl(IS^yrvsksfvjXfm>(lcol%L3))Q z@ZT;aumO3Q#8R!-)U697NBM@11jQ>lWBPs#?M4_(w=V_73rsiZh8awEm>q1phn1Ks ze@D|zskeome3uilE8-dgG(EojlI(@Yhfm}Xh_AgueHV`SL##I@?VR+bEHH=sh21A_ zhs&pIN7YTLcmJiyf4lZ;`?pN0`8@QbzDpmT`$m0CTrTMiCq%dE&Cd_{-h`I~f8Kps zAuZt4z)}@T>w$9V@iLi=mh({yiCl}}d>JN)z;*G<6&mgl(CYhJHCAPl=PYK2D>*F zy;YK=xS@1JW7i=C)T04(2P#|fowalY=`Y`G8?eRMAKt|ddG9UF^0M5 zW=ZGZ5qb-z@}iS`4RKXvuPIfzUHT)rv<8a|b?bgB3n=ziCiX4m2~CdVBKHWxw2+Hz zLvqoAij9(0moKoo2$`dqS0?5-(?^RXfcsQB6hU2SAgq8wyeasuyFGcK+@An?8ZzVw zW8wwbZB@i=<<4fA7JKPkki6y>>qO3_bW>-uQ*>9g+g7M0U^`RV)YTrGu2Q=2K>fiI zY0dFs>+}xuOZE^efLK2K6&X@>+y10Oqejnnq^NjfXt9JpK4K_E=cl29 z(t2P;kl4AK_Jg9v{1(z)ESpyo_(Z`74D&J1A#J?l5&J^Ad1sm5;Po@s9v7wOs(=_T zkutjt`BaxT09G{-r>yzyKLlM(k`GZl5m+Tgvq=IN|VjtJ*Zu66@#Rw;qdfZqi15A@fr^vz?071F5!T`s>Lx5!TszI%UK|7dDU;rUCwrRcLh!TZZ9$UMfo z@Qzjw>tKS3&-pyWS^p4mMtx`AvwxVc?g?#8aj@jQ#YKDG0aCx{pU+36?ctAiz=f$k z05S(b&VPQgA(Sm`oP&M^eiHvBe&PcTb+j$!!Yx(j3iI5zcQLOn(QqfX5OElbSsQBUw7);5C92onieJyx`p{V!iwXk)+1v zA6vStRZo0hc>m5yz-pkby#9`iG5+qJ{x>6I@qeAK zSBFylj8{FU*0YbFd2FZ6zdt^2p?V;3F~kap`UQgf@}c33+6xP)hK)fmDo@mm=`47* z9S6rnwCSL&aqgZs959!lhEZZp`*>V8ifNmL;cqajMuaJ~t`;jLPB?X~Ylk_Z#Q;%} zV+sAJ=4505-DdnIR=@D_a`Gy#RxtSX+i-zInO@LVDOd*p>M-|X(qRrZ3S(>(=Oj>} z89d75&n?m^j>;SOXM=)vNoum|3YmzxjYx%^AU*V|5v@SjBYtESp^yz?eQ#>5pnCj} zJ_WCw23wGd2AA-iBve8Hq8`%B3K4@9q@a}sf$49IA^IPsX@QK)36mrzqOv?R_n9K@ zw3=^_m#j{gNR0;&+F~wlS(i8IQN8mIvIO)mkx|e)u*y+xDie}%mkZ*m)BQM^$R@-g z1FrP0{8A?EcxtxxxX&J;393ljwwG?2A2?y-1M0-tw$?5ssoEsbPi?sd2!s~TrwPLF zYo-5XYV7AU-c|Vb-v;>pVi^CwX(Rpt<9{Ic?@<9SrNu>F(gwij%?dC9^!Xo90o1-| z&_aPKo%+xyw64e&v<}F^-7sO0Cz-VOF@7**i@v&(Oy4Q8PbV+4&rKwmYyokM z48OZ|^%*mC_Q)RJ31D#b4o4Jzr{~BX4D#swW<31;qCil2qlim;e=9ymJAEXfv-|h3 z)>uqQ5~S+8IgiWW28Fqbq+@ukCLy+k7eGa1i5#G_tAUquw$FjFvQt6~kWa69KXvAj z-knF`5yWMEJvCbTX!K{L)VeNF?(+s?eNjtE5ivg^-#937-l()2nKr#cHShB&Pl^l8 zVYws26D^7nXPlm<_DYU{iDS>6Bq0@QsN%6n>XHVvP<^rDWscC!c+LFrK#)T@$%_0{ zob%f&oaq>1_Z8Ata@Y2K6n?GYg|l8SgUr(}hi4D!@KL~hjRv<}ZZ`tCD^ev=H&^0pP%6q2e+t=Ua`ag8xqWvNnIvCU|6ZA^L5v{DD)!mcQ@n6{=; z#Z)PrAz>*+h-|IV!&J*f@{xb!L7h3{?FEs*ifw5z2U9$&OkYseI68yb=V4xv*VK3- zVxGhtmedujX32y-kC{5ej-Wy#JvB~4oxTb{|1H825_B(A0#?CjUTc=PrGh6jAgK9h zoLAe`+NBdStZE@Y8UH^Rd*|R-|7Ke}wr$(CZQHhO+upHlCp)%n+fH_}S8%^%xqhu%20_1p=x#Dl9ia`c3iM+9Vh5?gyY8M9c$tJ5>}V_sidHN zoMl%rSgSK!7+Y8tQkYq|;Vh`4by2uMsUfnxkk2{S@a>V#d}fv}Yud*>paVi_~T zU!GoYwWbnG%92!Cte(zhZX-i9#KJ;b{$(aZs|{MerP#6||UUx$=y)4XOb zihyKn`_QhJ#~@_peJ*8yD4>I7wQyKkZG%#FTKZfb(@G+9x7-3@hG}+ZC&$7DwbaB$ zC)jLj7yituY&WpOWlG7Z4Tuxzdwo6k!3lgwhh7BYMyB? zO9Q5nvn77~g~c623b`Pe5efNzYD#2Sfmg>aMB5s?4NC|-0pIXy%%`J;+E{(irb!Szc8M8A@!}0zqJLoG4SJ5$~1*yRo0^Z`uObA+= zV?1sYNvzvWbP%AsMzoIo3Cwx~y%i8rHF(BgLS>tH5Ab|1wp$X_3o2_VB(pFxgQ5QQ zk@)Vy95$b%HVf4@ppX(wrv^Jwfrsu+9N_OUm}nD7Ch_7STj66EYsZR#`9k|Tf^@p& ziHwnO$p{TB#R(Q{Os>Un~0!r$JO zLZ&F%SP|%$TuG)mFeOhKr1?S!aa0jTV$2XIeZb_fgO&n{8HTe9s`L&(tKoy?OaS^$ zLHNrgYgq920EI~M>LyU7gK70$7*`nFKD^d>MoEAhsBU0%@*RW@%T(J z?+wVbz=mcN%4#7qlCpl_^Ay7VB%?+uW1WSNnQOj^tALyqTpV zkEN2C;qO_W)MYl^Ow5I;t3;z#iG82F(qe}#QeE;AjA=wM==dB(Gu+ez*5|RVxO4}l zt`o?*B;);-0`vR(#+Q^L4WH_9wklh-S-L-_zd%Q0LZ%|H5=>Z)-x#Z+m%p&6$2ScV zEBneIGo)r0oT)xjze*Q~AIqhB%lOM5Id}^eKwS!?b_;B&TouZsemyL&y`)#FX}ZKp zp)ZnB*^)1P@2bCoe+Z|#KhTBNrT)UN@WIuudw})fwHl)re1|b~E1F=xpH?7L77p>5 zei$aD@KO0<+zo1<&7OuZatNsPq24Whu%0jD_ z$ZZy6MzayYgTJulNEy8D$F%JDYgx|d6{6kpDg#s170<15bM#4tzvrDU$6bvu-hH@6 zgcjq&3aR3k(23$FaUA|iuoy*bO{2F6W0<+ZdsYvXjc?d@ZT8kM!GD}r@qr;TF@0Hb z2Dz-A!HZ$-qJ?F%w6_`t`8xk$f$MNBfjqwvJiVdD+pf7NVFGh?O=qp2vh%UcYvc{rFldib~rkIlo`seU%pO_6hmBWGMcUhsBSWiQYYPMX<-Cjp49@7U==iS57bG zw3T9Nbm`)m9<<4e$U74`t~zRo0JSfi}=GdQXGLLPyW zlT^I}y=t$j{Vx!wN^z8X4l0|@RNrC#)G>bK)7IT7Qop>YdS^NnI3gfP>vtp)pXkr2WSVcAAv8uN>@ z`6)kICvNYU$DA8pnkl4sQopDC6<_M8zGJ^@ANXJL(yd#n1XFj9pH;rld*gwY8om_I zdB55w@FUQ_2k}d%HtQsmUx_7Mzftky&o2X2yDQrgGcehmrDDDtUJj5``AX$gzEbMc zUj2Qzp)Lo>y-O*@HJ|g9$GR2-jgjKfB68J6OlIg;4F2@2?FlW zqj|lO7A2Ts-Kd!SO|r9XLbPt_B~pBpF40xcr0h=a&$bg(cwjp>v%d~Uk-7GUWom?1 z92p+C0~)Og*-N~daT#gQdG{&dPRZso(#{jGeDb1G`N)^nFSB`{2-UQ&!fkPyK`m03 z_Di94`{-(%3nE4}7;4MZ)Pmawf#{}lyTSs5f(r;r1Dp4<;27K=F}Oga^VsUs3*NIn zOsYstpqpRF&rq^9>m50LRORj>=;{CV2&#C$-{M5{oY9biBSoQyXvugVcwyT-19S;pf!`GSNqb4**TI%Y z*zyV)XN3Fdp3RNNr9FU+cV*tt?4L8>D@kJp^rkf_rJ~DPYL}oJngd1^l!4ITQN`0RTT^iq4xMg|S6;d}lznE$Ip^8pW-CHu zP*^!U>Lcd3*shqa)pswq;y<|ISM1g1RG#`|MSPNAsw*XH1IAD(e(Kgqp6aDHgv>fI z!P67$z{#()Pdo3;4dUoy*Xor(O?+YTRPe=g*FfRj*9q9!8p%1l>g3e^rQ_nm{(@4t z?^nMDC2J8@my5q0QyCljCSp_@)No+6bZ*y)lSdrkLFcR6YOHu*vZ-q(C);5$MmM_z z1WT>Gc8g%`Rt~6*!}JhWi0=Rc_z5c8GR9YXW+cdoK~Ea(@wyXf|89HagNuFAO-V7k zUb|9zaCCWH3^Fz(m7$8K$|0ZOP!SNpgP!ql<)!z8w$Z$?9gq2f<~koe3|zD=imLfD z>IV5?SkRZ;7JlOG%z%Tlze$GXr0A}ResyF63ZGZVDLv2k4HWtoqoCaq+Z&GaVKuLA z>@zhNjYYc=sexH?;DTe4&2vnQE}C@UFo&|qcLddvH0FwswdRUc(p*X&IT^Zu>xLpG zn(@C%3ig(l2ZPm#Fc){+0b+%O7nt4zbOt+3@GQVm|1t70=-U(>yo3VY2`FnXFHUyi zwiqf(akt0kEE5_Pa-a*VCS}Pi6?`~P%bvX6UT~r-tUAY%I4XF3^nC+tf3alyL{M`w zv?aVQ#usdwpZmkrfv19O39}tQPQM+oY**a{X?@3Qe>r$+G!>r#?Id&U&m^HU(f= zjVpSi9M||1FyNQA&PO`*94&(qTTMQv3-z`bpCXs-3bX}#Ovqec<>omYhB*VrwxqjY zF3#OXFsj`h#G?F}UAilxTQ|78-edHc-Uc-LHaH*Y(K%R#dVw>_gz}kRD4s#+U&Pq= zps)kMf_t9`GHR7CO4zI8WVj0%qiSqy50N{e_5o#GrvNhMpJf5_sCPrEa%a@ltFnss ziaWh26vEW4fQp}qa4oP(l4xIMpA)~VHD9!lP%;Tm`(HD$jYMM-5Ag>S(gC35J35$%?^gk(r|`4Ewi-W z;f&;B*fO=kC@N=r<-#nGW|yXE;`zb0Y3TJOAkw1a$SQgoTawHZTck+V%T=spmP`^BHihc(jc+S1ObX%6AYQ6LVVc+BfM*P{2s0T2z zVIs*5{ql%#CKAzv0?@S+%||z;`dpfj0Y(VtA51n$j%sG5I%A|h98VU}PkVZFrk1*G zaw75v3(N50lanvr&ND4=7Db;HS4fpi)2vTME7aD2-8N5+kcOXmYCrLE?*5&dWhvB` zbD5)ADuIwwpS*Ms;1qyns(8&tZ*)0*&_lNa`_(phwqkL}h#WdX_ zyKg%+7vP>*&Fus9E4SqIN*Ms`QLB(YOnJ|md%U|X`r#tVN$#q6nEH1|blQ?9e(3|3 z`i#;GUl~v?I6&I6%YvkvmR?*l%&z)Pv8irzVQsWrZSr%aoYuPJa#EjK|4NmiuswK= zlKP2v&;yXv3>LQ$P){aYWrb)5GICwbj;ygw>*amKP;Z{xb^cF}O@IeQ^hB-OjEK{l z>#PNyLuVkeDroL9SK2*ChHmJJSkv@YRn7)E49fy!3tqhq`HtHs_(DK|2Lyv(%9L&f zSy+H}Uk{nE2^5h7zN7;{tP3)$1GK9Xcv^L48Sodg0}ZST@}x607yJo2O*XCfs7*wT@d?G^Q6QQRb!kVn?}iZLUVoyh8M4A^ElaHD*Nn2= zkfCS=(Bg9-Mck6K{ z%ZM59Rs4(j1tSG1B#wS=$kQfXSvw6V>A(IC@>F;5RrCos`N{>Oyg|o*qR2EJ>5Gpe ze~a4CB{mmDXC7C>uS@VL&t%X#&4k<`nDx;Zjmo%?A4fV3KOhBr;VuO!cvM8s2;pG5 zcAs!j?nshFQhNA`G3HMS z?8bfRyy1LwSYktu+I7Hurb-AIU9r|rl5nMd!S&!()6xYNJ1EqJd9BkjgDH@F*! zzjtj4ezywvlkV7X@dG^oOB}T76eK=y!YZB#53LhYsZuP&HdmVL>6kH8&xwa zxv8;t-AE>D5K<{`-({E0O4%fGiLVI8#GfZ0aXR6SfYiPUJKnujMoTI5El<1ZO9w|u zS3lJFx<7XUoUD(@)$pDcs3taMb*(v2yj#G)=Mz-1M1q@Tf4o{s9}Uj9Yo?8refJwV zJ;b+7kf0M}fluzHHHS!Ph8MGJxJNks7C$58^EmlaJcp`5nx+O7?J)4}1!Y>-GHf9o zk}oTyPa>+YC$)(Qm8|MhEWbj?XEq}R=0NFH@F3ymW>&KS!e&k5*05>V@O*~my_Th; zlP05~S5@q+XG>0EuSH!~gZe_@5Dbj}oNIiPJpEOip+3l!gyze@%qOkmjmx=?FWJLF zj?b}f8Vet*yYd16KmM43rVfZo?rz3u|L6Foi*GQe4+{REUv9*}d?%a{%=8|i;I!aT z7Wxm}QJC`?cEt9+$@kSkB!@`TKZz1|yrA1^*7geq zD5Kx-zf|pvWA+8s$egLrb=kY385v2WCGL{y4I15NCz5NMnyXP_^@rsP#LN$%`2+AL zJaUyV<5;B^7f+pLzTN50Z~6KC0WI<|#bMfv+JiP3RTN^2!a7*oi+@v3w*sm5#|7zz zosF*{&;fHBXn2@uguQ1IDsh(oJzH#i4%pk;Qh^T zfQLyOW;E*NqU!Fki*f-T4j(?C$lY2CT{e!uW}8E(evb3!S%>v^NtNy@BTYAD;DkVo zn9ehVGaO7s?PQBP{p%b#orGi6Y&~<;D%XLWdUi}`Nu-(U$wBBTt*|N4##sm2JSuWc)TRoYg57cM*VDGj~ka<=&JF zo8=4>Z8F`wA?AUHtoi$_hHoK!3v?l*P0$g^yipOWlcex4?N2?Ewb1U=lu}0`QICA4 zef61j-^1p}hkA*0_(esa!p%dX6%-1e-eMfQsIp6wRgtE=6=hDe`&jel{y=6x5;78s z?5^{J|t!#x1aS8<3C`v%E%u{*wZwSXr$0Owl5_ zmXh>D>C_SjOCL^CyGZpBpM5`eymt{*rf~9`%F&&o7*S!H%3X)7~QFgn^J>6 zD+yV}u{HN-x9*_$R;a+k?4k*1f)rE~K|QvcC3dlr>!nftB?gE-cfcPMj&9mRl>|Lg zQyCe|&SuZopU0>IfRmcV3^_mhueN5oQ=J+H4%UsSIum4r4!`^DJqZr?1j3BU)Ttzg z6LwM)W&UEMIe*H2T6|{rQ;x9qGbp7ca#-!Egm4|ECNTMN);`>2Q&%|BpOdIJ4l|fp zk!qEhl;n(Y7~R1YNt7FnY10bQZXRna2X`E_D1f*}v1bW^lJorDD0_p2Rkr32n}hY! zCDB(t$)4YOd)97R60gfg3|wrlsVs#4=poh4JS7Ykg$H)vE#B|YFrxU-$Ae^~62e;! zK9mwxK?dV4(|0_sv(zY&mzkf{x@!T8@}Z6Bf)#sfGy#XyRS1{$Bl(6&+db=>uy-@y z$Eq~9fYX$06>PSKAs#|7RqJ3GFb;@(^e`jpo-14%^{|%}&|6h{CD(w@8(bu-m=dVl zoWmYtxTjwKlI!^nwJ}^+ql`&fE#pcj*3I|_Z>#y##e@AvnlSN4po#4N#}WT)V5oNP zkG+h_Yb=fB$)i`e2Fd28kS$;$*_sI;o0Xoj#uVAtsB6CjX&|;Bk}HzQ*hJ!HDQ&qZ z^qf{}c`l^h5sg-i(pEg#_9aW(yTi?#WH=48?2Hfl_X+(SfW)_c48bG5Bf+MDNp>Y#Mpil%{IzCXD&azAq4&1U10=$#ETJzev$)C*S;Pr9papU3OabRQk_toRZ!Ge(4-=Ki8Db?eSBq~ZT#ufL6SKaXZ+9rA~ zQwyTQTI7*NXOhn?^$QOU>Y6PyCFP|pg;wi8VZ5Z$)7+(I_9cy--(;T#c9SO;Hk~|_ z0tEQ)?geu8C(E$>e1wy%f@o;Ar2e#3HZP$I#+9ar9bDa(RUOA+y!oB;NEBQ`VMb@_ zLFj{syU4mN%9GF;zCwNbx@^)jkv$|vFtbtbi7_odG)9s=q(-PtOnIVcwy(FxnEZm&O^y`vwRfhB z7Urcums9SQS6(swAgl?S|WDGUTFQu51yG$8069U zviuZ=@J&7tQ8DZG<(a->RzV+sUrmH$WG+QvZmUJhT*IoR3#3{ugW%XG0s?_ycS6V6 zS)019<_Rl@DN~8K4#w3g_lvRm4mK3&jmI$mwROr0>D`mX+228Dw4r;mvx7df zy~$zP8NjVX?xkGFaV>|BLuXMQ+BN+MMrIB4S6X)p&5l$;6=S8oI9qi&1iQbs?TroDMfCmIeJ}pbVVtVqHhS(zutEy6#UjTk29-+3@W0`KfehW`@np zhhu#)O&g%r)hTj4b$CY41NYp_)7!bYyG;v(rts z^}YDJt2W88H^H;e$LSm3dh=~yi@)mzJtEfW8=4avbeOE&;Oc>-6OHO+MW`XBZ4rO6 zS;nAi**w3Yso4&Ty+8f$uvT?Z)eaLe$KW1I~9YM2zeTIT}C%_G6FPH-s5Wi3r`=I&juGTfl zZ;4qFZV|6V0c&>t!Y>mvGx#1WWL0N5evV=u28K9**dv`}U3tJ$W?>3InXiwyc)SA% zcnH}(zb0@&wmE>J07n#DOs7~lw>5qUY0(JDQszC~KAAM}Bmd-2tGIzUpO@|yGBrJyXGJk3d+7 zJBN0$?Se(rEb0-z2m%CBd;~_4aH04%9UnSc4KP!FDAM5F_EFujJZ!KDR-fn181GX` z8A?8BUYV}D9bCE0eV~M>9SPag%iVCLWOYQJDzC4~B~Ct0{H7x|kOmVcTQ;esvyHJC zi$H0R73Z8+Z!9^3|2tNut#&MVKbm`8?65s)UM8rg6uE(|e^DYqvoc15-f;u8c=>3;Viz*T# zN%!T+Hex0>>_gUKs%+lgY9jo6CnxL6qnQ>C*RseLWRpipqI;AQE7;LUwL`zM%b`Vu z%Sa-+?a#+=)HaD|k2%_(b;pHRF96(c;QyPl6XHL8IqGQKC$M8R=US-c8;hUe?LKo&l!{V)8d&55sUXEu z5uITcO~`ipddh+Nr{7ibp^Wd{bU)^3##<5`lkuqfckxEU*9{pgNpTB2=ku1c-|3dK z|LIQF=ld@I7swq^4|G1VA}BK85&>2p#*P95W`I1FF(8G9vfNJ6MoN$+C^M89u!X=< zJSS%l?Qj>$J%9?0#0&S6#*h*(-9Z$}q*G#hP?cX7cAvM0eiVFhJJ~$`iZM!N5NhDb zi<1u_m#?jzpIaOe7h|Kiap#mHA`L|)ATnPJ7du{^ybuNx@1jA+V1l8ux#{LJ#teM(6=%gZcMq24J$2p z`wcC!qRssmwUv4H6Psw{(YdDNOv$!sq&O1SvIS}fCKZa+`T=Ayt@uZjQqEC{@Uj+| z!;i3W+p~=@fqEEhW@gT^JtCR<`m`i|Htg<TSJ&v`p;55ed zt@a|)70mq;#RP@=%76*iz>fAr7FKd|X8*@?9sWOFf$gbH$XFG zcUNu#=_+ovUd>FW*twO`+NSo*bcea=nbQ_gu^C7iR*dZtYbMkXL5mB@4a3@0wnwH! z(fZKLy+yfQRd%}-!aPC z4GB%OvPHXl(^H(BwVr6u6s=I;`SHQ1um7GPCdP-BjO%OQUH!_UKbEGvHCY}{OL`8FU$GZ;Y$SlS$-0VjK%lCP?U0shcadt4x7lN4%V}wBrLEbiEcK-OHl+pcBNSqN#mftpRj2A4Q z+av@-<#t_Dj_FN^O2~wq(ij1O*+=RVl+6gNV^~CI1UED- zn^zN@UOq8?q58b^4RA>lV}x;jA2OE=SqMYV9P#RsUlI+pp!y*jpwHgp-w3i$V)%?L z>irn1pnRc|P@r|Z0pCeMZ*k$}$`1GVGCT&QtJ`V%Mq!TXoge?8Fjn$bz}NqDn*2ZQ z$p3@F_^(}IVS76>OLNzs`O5!pF=LZ$<&gyuM$HQzHx8ww^FVxnP%Yv2i=m*1ASF~~ zP=!H}b`xl`k0pL5byku2QOS~!_1po!6vQyQL#LQ#rIRr?G5^W?yuNvw-PP{}%m35i$i+I?DJ%RGRcqekT#X~CxOjkV1UQrd&m_bbJ+gsSGbPwKS{F& zU-`QNw!*yq#Co#{)2JvP-6>lY$J$2u+e=r0&kEc#j#jh@4Tp;l*s<28wU%r= zezVPG^r*a?&Fn_(M|A7^xTPD998E-)-A4agNwT?=>FbrHz8w~w?hWBeHVYM()|buJ zvGv4j<%!U_Rh^ZKi~2(h1vk-?o9;`*Zc}m5#o@a1ncp)}rO2SDD9y!nT$_Eb%h`>% zDmssJ8Dl=gDn<-7Ug$~nTaRzd?CJh;?}nCco$7Pz<#J8;YL40#VFbAG|4nA$co;l^byBOT2Ki@gAO!{xU7-TY|rujdYTaWV(Rr{Jwu?(_TA zDR1|~ExJBfJ?MAReMF47u!oEw>JHVREmROknZUs2>yaboEyVs$Pg1f6vs06gCQp$b z?##4PWI#BxjCAVl>46V_dm4?uw=Y@h#}ER4|ACU{lddiweg`vq>gmB25`XuhNai1- zjt{?&%;TRFE+2Y_Gn;p^&&|bU44M=`9!Mc%NbHv|2E4!2+dUL z>6be$Kh|Duz}+)(R7WXsh!m`+#t^Its($x`pqDaN-^E z?*a=0Ck^rZBLQV~jY-SBliN&7%-y3s@FB;X)z(t&D=~@U0vT%xfcu`Lix=W#WVE{{ z2=C~L$>`~@JCIg8RAyk= zYG`(@w4H95n0@Fqv16~nlDU!+QZw&#w@K)hv!V>zA!ZOL$1Iykd&Su3rEln@(gxO| zxWc++T-rQEIL+j7i`TeatMfp4z7Ir31(TE4+_Ds@M|-+cwQg(z>s=S}gsSz{X*Wm+ ziKJWgOd`5^o|5a#i%?Gvw~8e?Rpi7C>nQ5dvPHVTO$PI^mnJ*7?gd3RD{|c_a>WrXT#Es3d}(k z$wpmA#$Q^zFclx{-GUL_M$i0&mRQMd4J#xq-5es)yD{kYCP1s!An(~K5JDRkv6DUSKgo^s@lVM5|V4mWjNZp zsuw^##l%rbRDKglQyj?YT!nk$lNUzh%kH705HWhiMuv(5a<~yoRDM&oCqm+1#S~|8 zA$g2Xr=}p_FX%Eaq{tUO9i*Q1i!>$+1JYZCL}flWRvF0y1=#D#y-JQTwx6uP-(bC} z_uP7)c;Xd`C6k#JVW?#Id7-|`uW+hN0>OM=C2Ta^4?G zr;EvxJ{%l|8D-heRYRM%f*LBC)krHZJ@%&CL0)FADWh14&7KV<9km6gE=o9(7keg~^rIQtthK^_8%Jk&aZLY_bc6SbY>IcwDK9{sV*t1GfKwf8aCo8t za)yALEi^-WXb!k6n>W-62Z^n8hO|eRYr&uZiW5d_URi??nl*aGu?ioQ+9RF9u8kwD z6UZ6HVd(G%l9>y7E)uyn?gAJMKeki0@tG*jdcE-}K?8(D-&n=Ld1i=A1AI<1z>u5p=B z<1}|q3@2jNxW-}Q4z~s|j&^Qc;nXIdS3K8caP_07#ig} z#KAD&ue2jXc&K#Q`Hy#x+LeT4HHUCzi1e?*3w{tK+5Tij(#2l2%p#YGI-b~{5{aS8 z!jABC*n6y~W|h;P!kn(a4$Ri2G118!?0WHDNn((QDJP^I{{wPf<^efQWW?zS>VS?X zfIUgCS{7oV$|7z2hJBt+pp1CPx4L{B_yC3oWdE)d)20WG6m5qknl}8@;kjPJE@!xP zV(Nkv^-Vz>DuwBXmKT(z>57*D<$u=Blt)IS-RK0j89omD{5Ya*ULWkoO)qeM_*)jF zIn87l{kXPp=}4ufM1h7t(lAL?-kEq>_DE-in8-!@+>E1+gCV9Fq)5V3SY?**;AKq0 zIpQ(1u*3MVh#tHRu5E5=B{W-QOI34plm`#uH(mk*;9&Re%?|v-=fvb;?qvVL@gc|l z8^L?2_0ZrVFS-stRY(E>UiQeG_sMrw5UiO znGFLOP-GO{JtBM@!)Q37k3G_p&JhdwPwtJS6@R4_($Ut^b!8HP{52-tkue8MG=Zwr z7u6WaFranJq4oNadY)>_6d~?pKVxg$2Uz`zZPnZVHOh-;M|H7qbV0OF8}z;ZPoI+| z(`e}bn6u*kJpRLC>OZ}gX#eHCMEk#d8y$XzSU;QZ|An$pQ%uZC$=Ki!h@&m8$5(xCtGaY3X1FsU?l5w^Fr{Q-?+EbUBxx+b?D z80o*@qg0juG;aZhj=tO=YHjfo=1+-NqLME~Kw7Y1A*?}M7#cOyT(vd$1tVPKKd@U! z&oV!RzZcK6gPWj`*8FIAy2I&x``h_sXPe*O{|ih(Y+V3|o68MWq~2Iy^iQ8RqK76f zC$1+hXqd^jsz`U{+EFo^VQNrLZt#R`qE*>2-Ip&(@6FmtAngx@+YnG}b5B9Y)^wg#oc z24KlT2s!H_4ZR^1_nDX#UH4(UTgl603&Q3g{G4!?6Sl9Om=Sy|8CjWO>d@e9?Q%s- z-OS3*W_H7*LW|Ne{b+^#LqQ}UKDmiZDma@no2!ydO^jcm>+z379K%=Ifs{20mT|xh zP$e7P=?N(tW4PMHJOQ`a8?n}>^&@<`1Rgo`aRevPp^1n7ibeS6sc8^GPe>c&{Kc+R z^2_F~K=HVI45Pf|<3)^;I{?H}vU7-QK3L1nHpcn3!1_)<$V;e0d_b8^d1T==rVpky zZTn~UvKrjdr11k}UO@o>aR2wn{jX5`KQQM1J1A?^wAFvi&A#NA#`_qKksu`sQ0tdM ziif17TO<{wDq_Q;OM}+1xMji^5X=syK=$QdZnS#dwe$;JYC7JozV8KpwfV}?As|^! zFlln0UitprIpuzLd$`<{_XoUV>rrHgc{cUQH-Px#(_Ul%=#ENrfJe@MRP_$E@FLMa zI`(J)Imw$o427@Oc^3(U&vz}<3Lfmy7diVpJJJ@gA>e;q-&gj zcGcBC_luF%_;**EB?o--G?AkaruJ%-b*8aX$4E+-?V@RWMnjHJ;hx27Vd7l0nUUY( z6OQb&8g8cvN3LZ%^xvIav*X|Epqm@yrTZk9U{GSZXAUJt8Lh(%7?Eaf&AzmXOVvU| zmz<@l1oMe#^POR38KT6q3@c`{%eYNu4ccurv`q?b5DzLxENjSfYOJHAI$MbSNgB*D zJsP>i*BgrFlIn?x&DH9x~UbPBtMFj{_vJ#CaAF>1$oE&k`EF&L@HCa@mN>Q7~!RU>7 zW%fv84aCKSgBacmuvg}r@)YKqO$U{D5|!`vG-Gp%An}raz2gESWm0Exhux4C)zE}} z_@kn z3t}bvm?L+@@az@<*jG>(Xopq&c*;^mttlJ!mv;5k6o%Ac<_`o`4G3qzzo(GO{!&F8 zW+~bF?S;7gO1dQ@>gwZ?iIHjE#^@;Ix!Z`R6{RYLlGB&v4A)ha(2hc`RGV-8`LcvSf+Y@lhT%(Z7$tWEF;cZs2{B|9k#&C}sPyr; zd-g~${TqY7E$9X+h4_(yMxQ%q;tm(h(lKzK)2FQ%k#b2}aMy+a=LHYgk?1|1VQ=&e z9)olOA5H}UD{%nu+!3^HsrBoX^D9Iy0pw!xNGXB6bPSpKDAaun{!fT~Z~`xp&Ii~k zdac?&*lkM+k_&+4oc6=KJ6RwIkB|st@DiQ!4`sI;@40>%zAG^!oG2@ z@eBM$2PJ@F&_3_}oc8A*7mp-0bWng^he9UYX#Ph*JL+<>y+moP^xvQF!MD_)h@b}c2GVX8Ez`x!kjAIV>y9h;2EgwMhDc~tn<2~`lf9j8-Q~yL zM=!Ahm|3JL3?@Tt(OuDDfljlbbN@nIgn#k+7VC+Ko;@iKi>~ovA)(M6rz5KP(yiH| z#iwJqOB7VmFZ#6qI~93C`&qTxT(*Q@om-Xb%ntm_?E;|58Ipd1F!r>^vEjy}*M^E(WslbfLE z<+71#sY~m$gZvoRX@=^FY}X?5qoU|Vg8(o`Om5RM6I(baU^6HmB<+n9rBl@N$CmP41^s?s1ey}wu3r3 z4~1dkyi%kA#*pLQy0phlXa-u(oK2Dwzhuex$YZv=*t*Tg5=n~H=}fJA!p2L78y3D2 zimkqC1gTU(0q||k9QM#><$b-Ilw#Ut2>JF=T^qN34^qcBEd={! zB)rxUbM2IwvMo?S;Id^aglw}-t9et}@TP;!QlFoqqcs(-HfNt9VqGFJ4*Ko*Kk#*B zGpJ>tA9(=t|4#M!kBaf%{$Kfj3-uf|ZFgiU`Bo>%k_OuAp~vnE^_Tg8*% z*?)4JdzyMTzvNDy{r$c``zBw=Vr)6c4}CBIv#mw()3h7`?V-;LF?J&N5a>kjpy;9n zQyXvuu`n?+W84QV=(i`JEJY=}Ak+u4>!Lyt2P!$nBl}T=^|pG*z@)_l!)OKB{tIV&&E@hj=OIhSBHgPV~X=R3NrTMh?VzDm?1yW^IJ&zzAn2{8rE~MRX5EE)a(-T&oE)1J4pGXBYi+nexX-?5! z{EZ4Ju=Y8MQ87=uNc2t^7@X)?85KeSoc`?BmCD;Uv_cwQaLyc}vvnJKHV zuK)H_d)xhGKB!_pRXv{$XgfZ_(8G%N3o$ZI#_ zixQj~so0*m^iuA!bT>&8R@>b%#B~zbIlwt4Ba0v&>B(`*Z;~?6!>-aQ zal+Qt4^dCcjZZMd4b4Khg~(GP#8$3BeB8j!-6l?*##)H?J$PeUy)cA_I26#0aggao zaM5PweS_Sb@{OZ@Uw*(!DNV)KTQU+BTRi?AUAv0Vowth`7mr9)ZVC+TI?@; zWGL&zydnsuE3+D7#U~P%PrxpD3nTc9#mm621iX*?ZMS_Q#n9SzOJ~Hg@`rX{d?qJ; zt}`76!H)MX#=VKifJZP$3<8@}0-llthFpq3FV;(UP$-k63MkHHq~J&}d?C<+c~*Zk z<#G&>AD7EoiAVO38TO2TOBKN>6N|JS*{+`}V-)T0j(bAzGlEUWEvWLrMOIItYexh) z?he>SJk*#bywgDF6+*&%>n%0`-3tOY72+n&Q1NJ`A-bX*2tJV(@;%b6&RxMcUd7+# z@UzOmc9DolSHc-D$5(GouinaE%&uOVMyD&CTdKaEB{Qap4_wU7_=23CULKQ;jmZuV;+Y$(`#Gh0@}s7-!qk-^&#IG>7B{yft?UoA)H5 z|B0u3Tu0TF{AB0jpT|E&RsYB$3WiQU^5p*|f)^Si_#^j+Ao^|5(gNjn+!0|NtXDt* z5fwxpajl@e0FrdEuj2s#Pg>gUvJdko9RBwEe_4@?aEM?SiA2nvm^tsLML{-AvBWM7 z_bm7%tu*MaJkUWd#?GWVrqaQ0>B%Azkxj+Yidvc$XdG1{@$U~uF|1oovneldx`h;9 zB1>H;;n1_5(h`2ECl?bu-sSY@d!QTa`3DrNj_F@vUIdW5{R7$|K{fN11_l7={h7@D z4}I;wCCq>QR6(;JbVbb4$=OBO)#zVu|0iK~SnW~{SrOq&j*_>YRzU&bHUhPPwiy($ zK0qin8U;#F@@}_P_flw`bW_v^G;ct?Pb65%=%egDBgS#YF3?E36$9xzdvYqjAZoK#hcjctJu~MF^S*$q3`o2;!L|jPnM1x*Q~qF%BH(5UDFYglsJwO zEdEuB7NihnTXK6$)F~``nmSQNFP7x7hE{WuOjTAhEjGw#XxvL@S;aZYuyu9)!yZ~X zo35D6Cwb8`shRXCCR;xlR`n`cs4aie!SSM`0)x3ykwM*k zK~w^4x2u#=jEEi`3Q9AU!wE)Zpn#)0!*~)(T^SEjIJveav(d1$RaSMC0|}<)?}nSG zRC2xEBN_YAsuKyl_3yDt%W^F`J-TyeGrcfboC_0Ta=KcW_?~RLb>xbqIVI6`%iWz; zM8Kq9QzwO8w!TntqcB;gNuV$gd+N|(4?6A9GEzYs z5f4(*N5}&ObeYA~I28r;?pKUj4N6}iloE=ok%1|X()Ahdwir?xf6QJfY7owe>pPj)Me*}c^%W-pP6`dnX1&6 z`b#*_P0PeM+1FR)t)Rnr22f!@UFBW!TxgjV)u0%_C~gIbb_D3aPhZ~Wmex0)Lj`VoZKjoW)dUoKY6*| z0|V)|XyjiKgZ}s5(SN?te*muif87vD_(wYOiOjOKNI4L*aK||2$~;s25HS#iY6r=)WW8a^dkd0Y|pPc1-9jmy&wqoCbL84`C94At6$lm_o!8m*did^?o$m?ozIp{RmZ*M%YMX_i$KYkz_Q)QK?Fdm)REqf*f=@>C-SnW{Lb;yYfk&2nAC~b}&B@@^fY7g;n(FVh_hy zW}ifIO9T7nSBHBQP5%-&GF8@A-!%wJAjDn{gAg=lV6IJv!|-QEXT+O>3yoZNCSD3V zG$B?5Xl20xQT?c%cCh?mParFHBsMGB=_5hl#!$W@JHM-vKkiwYqr8kZJ06n%w|-bS zE?p&12hR2B+YB$0GQd;40fJd6#37-qd1}xc1mNCeC%PDxb zlK=X|WE*qn2fROb4{oXtJZSyjOFleI3i8RBZ?2u?EEL1W-~L%7<`H6Vp0;cz5vv`7jlTXf-7XGwp}3|Xl6tNaII3GC z9y1w*@jFLl2iFA!<5AQ~e@S|uK4WL9<$R^??V^aM?Bgy=#|wl$D2P$o;06>{f)P+X z91};NrzVV+)b}k2#rYLF0X0-A+eRul=opDju)g0+vd79B%i!Y}*&a^L$_|C&jQN^j z9q#4<(4)3qNst^+ZYpyVF2hP;DN|OMxM9w(+)%kFQRcYVI zO-frej9x6a%-D%Xuwedcw9#3VSVkOjNF!BYRoY1KD3wFJ%?ML*3QwcarMK)@v`o%s z$w=NLrO>og`nRJpZZ(%~*hNJU#Y~k;_Ci3~gc=4UQO!Ydje^?=W^DgCKyO;Zz4LgQ zKtm($MdY;UZ((U_g5*pMY+dYGyyT1ERkaj`U#S-2yyJ47wMonCpV+2rI8zPNHDfo& zc59dFz*2#^A-R?P6Np}jhDLi4&vP%$NW#8J>=CLj1mlf$XzmQezH*F1jNOiPgXl2j zzD07AKLT*h$CA*OsOba2etPLU%|p?=XhplXo?vOu@q0{QBo++)@6U?YKv_)GFK(^Y zm&uFBbrQyzJm;c49O00PIt;|{&ei%VSS%Y3m3#~L#(3%Gso^a4#9AaB$w@vnAvdr6 z%!2#)YS0HFt%o)q6~BelT;?%oUjX%9qQCn#-~+TM(a^s%Y>&aBkL(UY{+?a9@&Q+a;t%c_6u^6_r@>MEAN9ir5q=Yo|R8z4lKYd1sv^LyTozFn$KqaJ>? zoH&+`AX>E03Gv=71+NZK2>!-NasKeCfMp;@5rZ z*m<}q2!$AgKUwWRXTVHs!E>`FcMT|fzJo30W551|6RoE#Q0WPD$fdA>IRD-C=ae&$=Fuzc6q1CNF>b3z_c<9!;))OViz@ zP58XOt`WOQS)r@tD0IiEIo4Umc(5f%J1p{y4F(1&3AzeAP%V)e#}>2%8W9~x^l}S4 zUOc9^;@m{eUDGL={35TN0+kQbN$X~)P>~L?3FD>s;=PIq9f{Xsl)b7D@8JW{!WVi=s?aqGVKrSJB zO-V&R>_|3@u=MEV1AF%!V*;mZS=ZK9u5OVbETOE$9JhOs!YRxgwRS9XMQ0TArkAi< zu1EC{6!O{djvwxWk_cF`2JgB zE{oo?Cyjy5@Et}<6+>vsYWY3T7S-EcO?8lrm&3!318GR}f~VZMy+(GQ#X9yLEXnnX z7)UaEJSIHQtj5?O(ZJQ{0W{^JrD=EqH_h`gxh^HS!~)?S)s<7ox3eeb7lS!XiKNiWDj5!S1ZVr8m*Vm(LX=PFO>N%y7l+73j-eS1>v0g}5&G zp?qu*PR0C>)@9!mP#acrxNj`*gh}21yrvqyhpQQK)U6|hk1wt3`@h^0-$GQCE z^f#SJiU zb@27$QZ^SVuNSI7qoRcwiH6H(ax|Xx!@g__4i%NN5wu0;mM`CSTZjJw96htSu%C7? z#pPQ9o4xEOJ#DT#KRu9mzu!GH0jb{vhP$nkD}v`n1`tnnNls#^_AN-c~PD;MVeGMBhLT0Ce2O2nwYOlg39xtI24v>pzQ zanl2Vr$77%weA<>>iVZQ&*K9_hfmv=tXiu#PVzNA;M@2}l&vaQsh84GX_+hrIfZC= z0Se*ilv-%zoXRHyvAQW9nOI2C$%DlFH1%zP-4r8bEfHjB3;8{WH`gOYt zg+fX)HIleuMKewYtjg+cSVRUIxAD9xCn+MT zs`DA7)Wx;B`ycL8Q&dR8+8mfhK;a^Rw9 zh9tC~qa>%5T{^8THrj^VEl5Do4j4h@nkrBG6+k8CDD~KB=57m@BL-)vXGkKIuVO9v z7t_L5rpY^0y=uu5iNw0v&Ca-zWk>v;fLJ=+SaV&V#C-o^}8 zp&Xp$v?~ccnfR=&5Df)32^d6QJLg*iuF#s|0M4zJF@Hza1p`q|f}~K)q;HC*I1_9t zQ&1jr9-kdUi8)DGxiwdqU|rPxYWDQPWY&SI&Rxkhxobp~C=Y*`d?HD4JW?WjU7dBPeuIE`ABLq95b#lfKS52IB^6KoHmm60$R}TESplQt59#mboJj+Na!P)V{ic@$yQ-&Z za^JU0T+n0Lf2VdusoNr0?g~1DMsY)zdY-63yH!Ii#aWe|;0TO>L7#YlaDrH}xvYXn zh-NYa>O>f_NTTBG=|k0qWH+X?d5@+INsQ}WcI_3z1Z4-%Gj#_{P$0A~cAye`?j0cW z8)hd(V}7rattLUSMvgZ4g96P7n` z^{55A&&29;-P992{yhkGWa3v_Z6iB4a&~NmL)IpC&dsSwe$9jS(4RVJGt=Y!b-O~1 zSCl@wlaba_cA*yt(QvulMcLUuK z>(ys_!{vqKy{%%~d#4ibQ5$yKn6|4Ky0_ngH>x-}h3pHzRt;iqs}KzajS!i!Pqs8c zCP%xI*d=F=6za_0g`{ZO^mAwRk0iwkzKB7D)SaLR0h|ovGF2w9C9g8;f#EtDN*vBP9yl;n=;B2a7#E8(%Bw()z(M$_pu zQ+9uFnlJ!5&$kk^S_+kJ>r9y8MFPpSf9;o8v;ZxsMA!p>eaAIwt5xNiQ|2_ydGkbi zkggG;Xp&I7C8R{>ten^j@MsN#V5JPs1Ezc!74->Nh0a}U){OK@j=OIoY}C7IYYd8-V9 zQ6s?v=Y7(?Y$7=P#Wwub-*0DLqli?I%kT-D^jqK?c2~HEx<2(poRWAUoC}!~6$1=I z*M(IfPmdID8i+5l@=1(+`?i`G_ew=1Y!gF?tFbdgtW2etKLOFoNozkH(i!Qa7(h^| zF`9!VeqQQwM+yO6J`;oWUWq@9l6hP~FiG8-{Pj*T`XI3~s@FfjW2Tl(llpa901$&y`F}K1uZuHEo;=mr+_8d(o z2Be#yWHEN@euC$=VUSB+3A}khJdF$)0r#<5(f3n`kx>ZT8ifaKyX*OhffeHH1?6OM z*-19$j5tMNYQoB)>cGpz@11>J%q4KW`GLNj?uB>LcNg$0G@}XN#Tqf2F5@jv<`|~p zqB^l!%v!g{R_+0GX5z0>3Q~O``%T$NFc==dsPsTj-;{b$XUS0TGoJs2BUA*H;4S?w z|Nigt|F@9hf7QLSo}JPEK#CPgYgTjrdCSChx0yJeRdbXipF(OwV)ZvghYba)5NZxS zm=L8k_7Lb?f8`=vpv(@m%gzsCs9^E$D5Jn+sf}1lep*zz&5V?~qi_@B?-$Vd1ti(rCi*I0}c}slKv@H_+g?#yarVzpYZN zIk21Bz9Z#WOF`JG&TC&C%a*3*`)GJx9I!U8+!#J4}@5rm8*jK%Xg2VLjP-a;H zFydWO;nxOZ&|{yOW;ta$ZU^6*4vFP)idD6M*M0+9buB#hK4z%YTGBdSva?Pvxim2` zF-?QVGuRQ2-1eYzd1Y%}w^`t1S7|{{8=Es#ApC0<;pc$|NJ)IU%WVK+4gnTWA7-t1 z0K{DCESXb}!y_tzrycr^%%|G4T4)`$BC8+qm|n1lS?CO=`V`1T#ykY#5g5$dc$lGt zqGHyw-*Av%C;33nEiU(rU?w^3F46!dEz#cHd3IF<(XCq)>JG?Bi)4v26MQr1A-g5RqhFoPy%^TD3sa|D^9aS>>_2-X2i#? ztVp@ZkyMB;Uo#9s!R!@G#CCaFVaxx*8YYu$kGFk4g3|9t!1nKqOaDBAe;w!(6#w)0 z?{&F2BgctT1=Z;TvjOGL_!}Vlt=kaLA7#W`mv1h%hUg983!wA*K@_r6_cd6o z6LHiCE6qwlt2H&|Ica~%b9C?Z@$dreBNR_!NKcfL)%8kGr7!IVq|^&6PKYK%EhcKu z6+uR*%EOw=rF6Q42Mx|a> z$2XrM*NV2x9ci6|X^eh1UAbJ9Ky!#*Q5w7)#o#%}d!#-^k8To=n8{UU*LmFsS-wRj zi6-p76V6g?If3S&Bj~GW&QI_WtyPY0@u3hjKtqf9`8S!wn{@P&Tc8uu8cf)YmrX7+ zrC+O3V{9}JG6ihA&^2Q7@)Kq)j(Y_oTzsoBUYQDG!}`Ame`bbcr>J-6E%gaBPEDCU zflX#1-)Ih^HJV*lew*N_SdG-4!b2}G8%U&9_V0~Qt?ZS z@H3L&5ybV8X}A@KQADl93H`}0qkNm!jGHkCJUM%r8`mP1nV?Oo%^l;yDnU6IJtbuY z`X2Sf8|r00mB_f)Q0;S{FqS1Yq?otd-BVbw`#@SDd5}n5X4lqdDi1*vtVv8-Zi10q zexCj0eyngrp`UxjEOrdzUt`?%jRlj7zSU-V-%R?y+_w7P7f1ge%t1ozmN+&)%3xQW zT3u@)))(_a<6`lTJd`DIYw>(pkb=PMKvCNEG~zza+LVNqkY^}QoGMVdS0K;gS*A3f z;6Ua!^sSV-try(M^pB6D9dsX}c>$Da#NHucp9vr(fg4pbBR*uPhYq+N>q1X4RSOCl znIQj4=A+y+8{?LQ$3L@(!Yy~~Cu4Sx72*%@dW>eP%Br7=uaynV6Mqa-49A9) z|L&5r=4K5SClwc`!2J|>(#n$4y1>lmR~2Om8q6HkcpK>d(Fk!T^NO?hM4Fc+(5J{` z&K|vrBz;;zWlNO%=a~JkMxMiZa%wYz#G901lw#+2SUaMMHrebb&|1L8tKoGJK*QhJ zU9|WkDy^-4F6U&VYSc3ScHDk@kV^0801#I|-pSK%az5=DwI}gMm)@s2O+-ESTk?QY z;y9gyucaXO(Cc+cd{B>2)euMHFT71$a6DssWU>>oLw4E-7>FC-YgZH1QAbRwmdahD zO4KAeuA^0q&yWS|zLTx%(P4VOqZv-^BO`0OFAXdBNt9>LAXmPALi3b|gt{b?e-$z0 z4n7H$eg6y_zs(c>*4FT!kN*$H`43~1p!g;IZ8-mYbUPTejaLW#BZnAPFES?ApM{TQ zE*TC%O8)apqcX|PrNjIZE-z{q`I(LwIE0kf=PLjExEX>)oIu><<@lt>-Ng9i$Lrk( znGXl|i4dP;Mt^-IbEp7K0e#*c7By@gCo@VQIW$93ujLL`)lMbA9R?C_5u~7^KopaAMj#6&>n-SOWlup_@{4 zcJ?w_!9JKPM=&Bd#IQ37F*x39y!azm$;~IRlkm>bHdABcNwW-TdDKD$pkD{j6A8d* z{vP~|<}bj_Oz#83K$ieRtsA4a@4a5cRjJ}A01{PgxXn3;fx)5ElMEPwDX_mW9)9oB z*;scve~v#HHqUj3KdC$tdV3&0)Whkp-=hKKz{SzD7g0@N!wyv;ZAime7AjB7&)!)5 zp_iVblaf)%agwJqOG2e7WTCM1&khq`{b>fN4n8hOJbvO?Y;60>LIwagLXWC@@0RSR zo%lPo1cUU=g$ahJ8D=;`v~ORUSl(1-&a@yTAC5Y8E892@{P@MM=GXUGpBSXSbSs!N z;L~0D_s7{+^F6c!WW+^yz5~o7eWtsOE}8{hKaFlHgnyBeUJ8Zz2$k7Lrh?NuMU|No zVvsq@57)8zin;&ckR1;*Z%(xH2lBw z`x%N;|H1En8au588bPDxP^$kfpO!bIzz>K=5Jiq9Rg(NGde0g!rKagLa+&yC)jg7y zq}~2IH)N*FJC31qrIH-2;%3^F?=bDD^U2Y;%ftN(v71oY;od+vh!!2z^}GHR$43rg z0In@ki}TglIsMU^O1(SiLK#oiuyw zB>-@z?&uW`ILoPupw0_cs?C|2YoX&87~us+ny%eo{A!3M<-7O7mHUBCgA~{yR!Dc^ zb= z8}s4Ly!GdxEQj7HHr<}iu@%Lu+-bV>EZ6MnB~{v7U59;q<9$h}&0WT;SKRpf2IId ztAjig0@{@!ab z{yVt$e@uJ{3R~8*vfrL03KVF2pS5`oR75rm?1c`@a8e{G$zfx^mA*~d>1x`8#dRm) zFESmEnSSsupfB>h7MipTeE!t>BayDVjH~pu&(FI%bRUpZ*H615?2(_6vNmYwbc^KX4HqSi!&mY9$w zpf%C6vy@O30&3N5#0s_!jDk|6qjb-7wE3YT3DA7q3D`Q&Y*y>XbgE7=g#rPx1hnf8 zTWd{IC!Iysq*vZup5VGrO)UM<3)6raR`rOwk(!ikf3XPp!n|gz0hS*P=VDXAyMW(s zL??-`&IusEuOMrz>m(A1W5Q~>9xJwCExAcMkOBD` zD5BJSadd{0u}%z4r!9qA`FW4;Ka_Qk>FcHxiucGw4L9qhtoge|ag8jbr`7LHSbVQz z6|xUo*^LV1SLxS>?D`m=g{8IC&1YF$e}VRGD#ZOc_15QW%J@FbEj8tE-nGxo4?X02 z@|q#k*G4xMW>q84Xc09pRj@>Hz8t^fMm3n&G;Al6KU*;=W`7Q{$^|=bnZiJ7?(s)@ zB`vW>#zJ{}!8=*|?p(~fcXSanO^j8+q7V!q16*ic!HLRdz0TzNI6}m+=OKd2b8KX< zAcDTj*%~vQlcO+%@H01gjv-1zZaOXVoM*t-+KXTR#NoTf-#{dQAm?GqK6q8Ta zu3xW?t=NE$EfYa#=0HofLn5~c#m-U#Ct_r6~X-pg6k*F zYIP7De52BBwcAnK?O(j?YEs1;q60!-!hTuKzw3T;XcA_w5HvU;tO~}byLA^cggu8i z-IP@pxFjTy&ie28m}j66dm@g78xK7aG{QSR^bAcY+W*xWu;G~I08sf(GK4>K-cbfJ z-%v9DGR77He<291M~=fg>>9&NFQlboP)pC6fT;{>_!lM`A&&HWIMd)Y6e@IL;nvRdBE*Tn({&3{-XJ9helJa{G51Ck}-_Y=5C|fEo z)7fZlsHxN&SY&ZLTdYuBBZnwIh0#VTzmyK>U0|r&SXb&GP0m)1dGV8z(^x6s5yQ-z zEyniK${#U@Y7p@Yxx}E+jA?1@{=|e6UM;iyai=0=aItVvqieogZUq@sio2#9NLW~L z{w@^H!HEGU;>;T0lu{Ad20Hr6u;?-9YHKvkjEc)}wsb4Y-ArRK8`24uBT8N)8m%Ee zYJX21)|e{peL26}VUUKYQ3L@NSe8rEbN#AIo$tjJm-$B|IJU?mu(h$Sq`XNY0@NhY z0?WeMtPwP)sUdk}dWA4qBUV^x>P|is-kPgVe)*WV>dKDL>gOq1 zUYw(nU|N#dw>97A_(c3?VA_zDfF{^A1eE#8Bucd^ON(sv-{tc@&i)Y)3V~o7U~+AA zOwnXB5`WN^z$z<9^@(?LY%7?y5X_C(j1ip-Ug^f7Tt6suI3&a=&~#EJegG4r2^tKz zJoEXCVOc1QdOSNHp2d;t&smxL%CfK@mSl)Ky}`!6kCsi#7s5&G2Q!sM9S6o)&mdx% zz|2M~pav2;Th=DTN5yB@6HFAO!pl-y+tEJsh}(? z!tIyg01O*w@mWxsFhHMi7%Gqz!v(Osc5WxK+^1PGfsozw)FE}VIxk9GexmAohPNAF*SAjxG3Al#(xQoYXdI}TR zoCHAFS6+LDqsP8L1SZH{RxJjFK_=vy4nNH^?M!OsQWe^qC~$c1r&y`H9n5;D z2F$t-Htc%2@K(>opJHE{NytI2<_J<6Kz*p$wtKUTEH}zITx?H0L%!5%i@!rLphSBrkFs>jscP6?HVQovX8!~b~ZY|0h%&souT7e5nD@OxuSgC zVW*eo0B|1POwg7;6fJSUC`g+`1%XQvwpRc*&|AtV*h!#5nQM(@m!K)-Qop!Rt3F`a z9HUO zF3w{uI_==EpjFQWV4boF^A?wc@@@U+KrKPjn6sK{OLu-~1UloSqt-aHYo*^@kQy2+ zH(9*-mFz?YV4cL7EW)9hsdmG{5jaYXLvm*&3PZ4y?8z`$9z6`q9fgsJm@*W$-QSzu zut}57hroSbTd=&RJpuy#?K?A6!-;_MowpK8eb~5T-^eye%3O-T^ktSMbd%PT0j-B?#yAKr37u%gB z*2)WJMw6Y)6BvY$JjD`(06ci7u;u$hv}gN5oS&Q^*y$J6L)0#BD<>XL|;pZgtZaxp3~$0zxA(;6Qr_AP$?8l@S)C^Hoaz#rQFK^lA}3&)Gr}Fsca? zK>9BkVcl;c*E2P9UMppEIB&38dL9R?Xg9N{Nl~4*w!qsZJElz}Xc9gz#}cwnP4u{+ z6VNTEx*>u67?3bn{sWk*P`1_$YfsB+)Ax0+jt|)0p&VS?N0k8IAp2KH_#eY3I#{Hw zB$vObUDtXyZX)*wVh*@BefnUej#jv@%uiA=>ngX0kQXaz>8(WM)fX~v__@I}7|!Il z@J%r#I!JqqFwGd4JPhmDmL>1Bh}nn_BE;hgKUesNOf9zQhiuhn%4B}O8jnxEwJiQFDaiiuXw2sb?*8a}Lr;_#7+IPfIjhVDhazSpbQZECL+4)p8lO;)!y>Rt=0X*;O# zX{s(p-*d{#{Y3gVhL;A{4a(Z5sIfpk;WMCqdFA&Mb7mp;YMXhBF@p`}$ShAug+bo`;<9fm!~F z-;1yCj$GQ^mzucrfuatilXrYLr)`izjn_m(f~);txN?D7d?Kg4wDuPXilVyeVwjzf z=4Kewf=u}X_H*viVfPWZW?Sqa3G#h3|;b!Q7>BRc7-Wox0}&>}Lqo=0v;T_i~% zqB&h;14|~nK{W0N=$obGP@O%(c8SraYS^qiu%Q`B zBHdA!`Vk7#Bz*@_3eE#bizLzjBV;F0vfSA~+7@8+F{$7Y?fwI~Pp_X`2ORgqW6g@2 z{cQV!niSsMEVr1IaeRAj8~|*4yW~X5$6o`crw4uTHhgPs^qAk?9UPu;xy5wh2^jZ; z)@27Q=QKa?8w7_C0|u`@k=%b9Ce$D7x42CdLsckF2<$wLuV2kpik8PXex2^Co$n2o z)l#H*;#>?yrPw0x6LI@x(X$nezCBa0Obi%|I5ZV|4bJSPtNHjDkS|3S?fiv(i_(n* zFbve0g!B0!MMmakRsgg_if8nwImb=kk%|s+08xGQ)J?vpkdaya3UD|RJK+LQ72|g> zc4LnwInx!2pN-5Yvp7rvRF#B=(ZO8gyVB^0Dh#ZdHA2BjjppfV<=2Nm#w_t{%6O$W z`-?7N?LwL0DWgK0Y7L#ChSHfa{=DOpJpl8L@V70cd%ei)n%SQO;Z+Xw#li#%LUfbs z&hP%UzN(qM3cw#bWQS6_B@>1^ea-AqNA12xoiQeb_Zdtf>yHljqeIHqlyC^gzH)h1 zstXTFEb0r=l9;><<$a}YWlscH7VW_xeKVZ#*#v#HiuUOs7PPj8ml4#!BiGEK)kDpO zX=2mU0ZuIDDnhfV7v_Rs)0R#ff6I6_|MrzV(R$3Nt#S7D?GQy6?a^WRvA@r2~?7f~s99*9;fuqJ(843U`hRl2O|sk>J@WMsR2O zwyZt$@J)DnSUNkF@B3MPNz|<@`72{M*S5d<1Vkg+G=q~u{8OP84Yh6VCE5pNC*#m> z*jzHy5Tc82sBVw+6W7DoR5@LXZ|+>;)Q%czg%8pyMyeE2-)R^oHg~SrO~#I8MxNc> z6pWT&F&H1mX7#2@mBY>#rRoFKszT z(gvV#j3x|7sF|Dt0*CgsJTdH1R!>inYZWp*2RDbjjQCP98L_ds!$x&{t85NRYk4ii ztJ3HyC8h2A2&`kq^Cfci>N*r&btHg_|v6=s|v=(-MQ zK4kjqoI^~y`j9poC2r{Izdlehm8!AcMP^+SwDUce1Zon(%YvxK)x|rXsJRlO?-K91 zMsmHgI&PmqT_W}C0mdA_6L!EEjgJzidRvTN;vQRJ-uBl#{dEeN?24PRwx)7c5kF^ut=M0)e@zr?z_vpYf=%;;@UYF9>9-->Qf2FW*# z5*#VFB$$-k(zphh4sAElMiLbp`$+SKm*{l6qX;Q8GZ7b|J>OhC!yg$}8dt$dx3E8b z$FlaM*K@6mSsYCoe#*QjLEB3|_Vs4GbZI#!>Ya}dzh%uMn}sw0gFQQ{+V+e|_`q)M3nK27)nAqQ-viJoPHUKdr9HN`v0 z+tZo0ORLuv_d)x}gO|~s(H!12RM(aMfqLG>KSH#kGxC{sUUj>FUC(6;ds1cOjeDYu zOrd>q@bNFq5?0s&@5nbF3-rw{{V&YYf3o_9|K-X4k861UwZ&C2bH+A7^%7nizU>b? zC2@*VlrqprJiv$rx{+^+Op9i3RM;IHq@a;34=Gn%B+rXMZi=UsHC@TEFk4{*fs96p z)wNUY?AhVkdLGQmPESuh@-!iqSZrnxIT~Mon)J+i+B~9VdL8QE`^4=2@lNaKluUVx z_^i7~5E4dN4&gVMi%;7ast@WIY21Q`+^iTC*Gx@IMVYB`BLFHzPh{Fpc6LKZTk@>P zquo2E*Pgq(0MX>h>4)YaJYbIK&V?-W}JfL@&R0I2)TOA!Teg zNa4DBO&)`Nn0$Inb|d8ea|)qqOLYVbQIBRC4T4E<5#Nzc2 z57|Bq7mYsW8y?uLA$XMj%OeK+1|DAKcLYB98-vDP<3*+SKYcPcOkm&}H|!{9l*9%L zbiYJYJ^)Cql-&wPwABGD>Ai7SUXe15m zIr^wNEU$9)D6@atm z(w(1~GuLpHi?JGgIBj`Ovy;j4M`XjrCNs?JsGh1zKsZ{8 z@%G?i>LaU7#uSQLpypocm*onI)$8zFgVWc7_8PVuuw>u`j-<@R$Of}T`glJ!@v*N^ zc(T~+N+M!ZczPSXN&?Ww(<@B=+*jZ+KmcpB8* zDY_1bZ3fwTw|urH{LLWB;DCGzz$jD|VX#Af@HC%BktA8F7VJSy&!5iTt};#U^e0_q zh6j7KCTInKqriZ1`BiF3iq2LWk;gyt0ORIFc4Mi3Bx`7WEuFq{u^C49-SYVjnv!_40m1>7x*+<8~Xkq?056 z!RBfE@osP%SxzOw>cLAQ$bioAOC0V!OzIXIc};)8HjfPtc~8tnah$PtoAz`4k)7$FDUc2O@D)g_uAo&nXMymK$##V?gYUPt^l zj{6NFDL(l-Rh(xkAHP%bBa=($r%3Y~jB!eQ1Smuq2iuQ|>n%Y=p(26SE5gFu11*Q< zaPN5G^d;Iovf`VY&Gh58z~%JpGzaeUz6QoBL^J%+U4|30w7Q&g9i}}@l61eKEfCgo zST6qMxF_Eaj7;0OC)TSU{4_m}%FOa6B{AxS$QIcmmG~IVjjf;7Uk!HBtHfm{%LsLb zu8~5VQFyOZk&!VY(wxL__haJ;>Bj?g&n`+i&=X{unJmv&0whCitWfGlOr6+Tc-lMZ z(ZRXqC-=O+GAvTXKViA9vdwu{aifhk$tYh~-9BScg!Yr*M2zw&9`pHMxHGh`dUH-1;~^6lF@ep;X9PjQ!rqmXNWJ?#P-qb%*TB%xe&3 zX*5V>xuW7)$3!Yc$y>cwBqd8+p+u>WS7p7~O80ipG{(a*#=NJ`^Ld6k-`|;Y&htFy zIi2(Sm)4eD=o+CGo~M3%qF|O9P0+ahmc%EklI?NgX05W3+OdS`_Rd#wg-}hd1&txU5wXy zy`x)05?WVZvELw`XWetIAg6$|(^4ntaE;=f$Wcpwbxm7?bLDnPs-1!bRoMcy!EeOh zpIv8ewDzcIU}mv1NxV!&(Wf7~_kqGAk=2=j&O5FA)z2!APCcDQPnIaiqMkVT4fUyX z))R|WvOJyzcU6d=z0q8JDt42*`js4g+_t{YP7lVguX+vhEejJ3TAIo*Z6jizHm#S- zZT_}-STQAa-0Gn8+RmR7V}{Ns1@jJ{^Sb!9&RSXXP;^ep)r6;&PW++~XYXC9a=zSF z?sp(JQo&MROb~b1Y*Xw4!P)>PHT>Z<)*U=Ax_75^OUw97pNudbxS1XPtNrIg zQ5YB77E@i7$2Ia}(^JcCi@OX`9a|m}PY%-th2m~y+)eCl>fTVjCP^lDOBLyhg1DZ+ z)~G{&OkDc$!;t~`gq(wz@qW3lh9B^ic$>-h#nV!H8d#l+>C(M%g}u2g=I#&W|L!VD zqHYoQkBW;`r|fW02u{7X!X;}T7X4iAaWzkeOh}7&o!F1qt4#$1|BDF;(2VlgEqJ$F zy8Ba-y(%fs`MzpvyXlQLEhS^ed$7Va2hO%?$-D>^*f$b)2Hx;}Ao$UqFt7l26<7eP z!{!C7PVrq>=794Zqmc z%LKkzIBZq@%Ja8EkH}?>c5ILG(EAMS*JHu?#9_7TsELw)8LZzN>f2Y6YN{AJC?34> zh42sPa1%2JpCeS9&E1URm+Pb}B>A1M`R{+O+2~}c(@^1Rf&J9p(4QqHl;E^4w5;I5 zM{?(A^eg*6DY_kI*-9!?If^HaNBfuh*u==X1_a?8$EQ3z!&;v2iJ``O7mZh%G)(O8 ze<4wX?N94(Ozf9`j+=TZpCbH>KVjWyLUe*SCiYO=rFZ4}S~Tq|ln75Jz7$AcKl$=hub=-0RM1s(0WMmE`(OPtAj>7_2I5&76hu2KPIA0y;9{+8yKa;9-m??hIE5t`5DrZ8DzRsQ+{p1jk-VFL9U z2NK_oIeqvyze>1K%b|V?-t;Wv`nY~?-t;tMC4ozyk8CR(hoZTno3!*8ZTc15`?MFf zDI892&g&3lshOEv4E@w-*_%)8C_<&HhV`0D5lN$WT4Q^UWHNSAE+RZe(o z%bqR^hp1IsDr47e^AajFtlppT)2F6yPcrWO9{Kw{o=P6y^HOW$Wqd_)_fwzn`ikZl zOGVc0+S(*=xZ_KbL0Nr`Sx$$CWEbw$52udl1f=X6CZEcFMA*nl>`0gn4&tc5^`!!)tGw<}^Q>P7E}$ zialDUofH*XcB3r9@tA@lnS}dA(@nK_xuw0b;FPUnNGD0;MIySCw=cSzB#=3>F37V-nni3UNB)-;;Gkk;3l9fh6FIjSZU zk=Eo2a`6i7@i*4>ym5`R?i-uZFv6+iX*Gi^I}ZU1OrLAX8aGiT@`*YnjeF>}$U}ORP`+EY5`eqVC_&4yG z;Tp>+2QbZ?lt1GB+D}q14W3dWP8lWnN zf(nlT6+XW&(zme{FbyDpP^NakA<~TK=Y}H^eS%2rt0v8Lr)B}@B!cTvC=9FM;7q4@ zf*;vb4HG>RFpY5?vFCp27VEnVIGx~-na6biU4{+UoYe=}^R#_My6wT$5d&r*=kpAA zu;=-c0|~yqi(N8&*H;aNfhyey+HHQ7J_qae*_CgG2V8j=Tq936S0DC8r3BXBql3Gz z0pLo_`|4Q+oY3rPBNaLmL{QM};9dke>ujP^j@z-N;fNlKb|edn>)YaafDaJ>GWKP$ z5}l&#$QFhN!CMT;WH&z-5E)kvM|36lV!^#3z{@2FF>HsgUO4PMqO#U$X%+U>K!xJ@ zBFs|+woG_9HZQs_Tw*vnCPGhlXG@>y|6pJT$I67!aP&b0o$AF2JwFy9OoapQAk>k7 z**+$_5L;5fKof<;NBX%_;vP@eyD=Z0(QW)5AF7 zp|=tk3p?5)*e~Inuydz-U?%Kuj4%zToS5I|lolPT!B)ZuRVkVa>f*-2aPeV3R79xh zB)3A$>X~szg#}>uNkpLPG#3IKyeMHM*pUuV5=-Jji7S6PSQ9oCLo{oXxzOZfF$PP) zrYwlmSQ-~n94uO3CD{K0QTmj@g%Yzn7_xQ4fTduU0Yqvln`e_`CdXH5iQ5qRr1 zBC;}%YZ2!4I>*=sR)O~jBPx6sxmIEBnq)s-fHz_y0z8-gPl2Us4BiBXNR5CIF!YR@ zb9B305SilU*@4|+ x6JBtc8JSt5M0pkooaq!^FqtuD_KdXXTo>Mw54>`rP&>h&58!3a6l6r9{sG7g--!SK diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index 1b16c34a7..000000000 --- a/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,5 +0,0 @@ -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.1.1-bin.zip -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists From 60d1d0dd233fd0c749a48fa02dc2b58491d645f7 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 12 Oct 2020 11:21:19 +0800 Subject: [PATCH 13/80] =?UTF-8?q?[feat]=20impala=20=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=8A=A8=E6=80=81=E5=88=86=E5=8C=BA=E5=8F=8A=E5=AF=B9kudu?= =?UTF-8?q?=E8=A1=A8=E4=B8=8D=E5=81=9A=E5=88=86=E5=8C=BA=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/impala/ImpalaDialect.java | 38 +++++++++- .../sql/sink/impala/ImpalaOutputFormat.java | 75 +++++++++---------- .../flink/sql/sink/impala/ImpalaSink.java | 8 +- .../sink/impala/table/ImpalaSinkParser.java | 12 ++- .../sink/impala/table/ImpalaTableInfo.java | 22 +++++- 5 files changed, 107 insertions(+), 48 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java index 754be49d6..334846e8e 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java @@ -18,18 +18,23 @@ package com.dtstack.flink.sql.sink.impala; +import com.dtstack.flink.sql.sink.impala.table.ImpalaTableInfo; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.google.common.collect.Lists; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import java.util.*; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; /** * Date: 2020/1/17 * Company: www.dtstack.com + * * @author maqi */ public class ImpalaDialect implements JDBCDialect { @@ -41,9 +46,14 @@ public class ImpalaDialect implements JDBCDialect { private List primaryKeys; - public ImpalaDialect(TypeInformation[] fieldTypes, List primaryKeys){ + private String storeType; + + public ImpalaDialect(TypeInformation[] fieldTypes, + List primaryKeys, + String storeType) { this.fieldTypes = fieldTypes; this.primaryKeys = primaryKeys; + this.storeType = storeType; } @Override @@ -65,7 +75,7 @@ public String quoteIdentifier(String identifier) { public String getUpdateStatement(String tableName, String[] fieldNames, String[] conditionFields) { //跳过primary key字段 String setClause = Arrays.stream(fieldNames) - .filter(f -> CollectionUtils.isNotEmpty(primaryKeys) ? !primaryKeys.contains(f) : true) + .filter(f -> !CollectionUtils.isNotEmpty(primaryKeys) || !primaryKeys.contains(f)) .map(f -> quoteIdentifier(f) + "=?") .collect(Collectors.joining(", ")); String conditionClause = Arrays.stream(conditionFields) @@ -83,6 +93,10 @@ public String getInsertIntoStatement(String schema, String tableName, String[] f List partitionFieldsList = Objects.isNull(partitionFields) ? Lists.newArrayList() : Arrays.asList(partitionFields); + if (storeType.equalsIgnoreCase(ImpalaTableInfo.KUDU_TYPE)) { + return buildKuduInsertSql(schemaInfo, tableName, fieldNames, fieldTypes); + } + String columns = Arrays.stream(fieldNames) .filter(f -> !partitionFieldsList.contains(f)) .map(this::quoteIdentifier) @@ -90,7 +104,7 @@ public String getInsertIntoStatement(String schema, String tableName, String[] f String placeholders = Arrays.stream(fieldTypes) .map(f -> { - if(String.class.getName().equals(f.getTypeClass().getName())){ + if (String.class.getName().equals(f.getTypeClass().getName())) { return "cast( ? as string)"; } return "?"; @@ -106,4 +120,20 @@ public String getInsertIntoStatement(String schema, String tableName, String[] f return "INSERT INTO " + schemaInfo + quoteIdentifier(tableName) + "(" + columns + ")" + partitionStatement + " VALUES (" + placeholders + ")"; } + + private String buildKuduInsertSql(String schemaInfo, String tableName, String[] fieldNames, TypeInformation[] fieldTypes) { // kudu表的Insert语句 + String columns = Arrays.stream(fieldNames) + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")); + String placeholders = Arrays.stream(fieldTypes) + .map(f -> { + if (String.class.getName().equals(f.getTypeClass().getName())) { + return "cast( ? as string)"; + } + return "?"; + }) + .collect(Collectors.joining(", ")); + return "INSERT INTO " + schemaInfo + quoteIdentifier(tableName) + + "(" + columns + ")" + " VALUES (" + placeholders + ")"; + } } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index f67d8f372..a26a27a14 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -6,7 +6,6 @@ import org.apache.hadoop.security.UserGroupInformation; import java.io.IOException; -import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; import java.util.List; @@ -24,28 +23,28 @@ public class ImpalaOutputFormat extends JDBCUpsertOutputFormat { private Integer authMech; public ImpalaOutputFormat( - JDBCOptions options, - String[] fieldNames, - String[] keyFields, - String[] partitionFields, - int[] fieldTypes, - int flushMaxSize, - long flushIntervalMills, - boolean allReplace, - String updateMode, - Integer authMech, - String keytabPath, - String krb5confPath, - String principal) { + JDBCOptions options, + String[] fieldNames, + String[] keyFields, + String[] partitionFields, + int[] fieldTypes, + int flushMaxSize, + long flushIntervalMills, + boolean allReplace, + String updateMode, + Integer authMech, + String keytabPath, + String krb5confPath, + String principal) { super(options, - fieldNames, - keyFields, - partitionFields, - fieldTypes, - flushMaxSize, - flushIntervalMills, - allReplace, - updateMode); + fieldNames, + keyFields, + partitionFields, + fieldTypes, + flushMaxSize, + flushIntervalMills, + allReplace, + updateMode); this.authMech = authMech; this.keytabPath = keytabPath; this.krb5confPath = krb5confPath; @@ -71,11 +70,11 @@ public Void run() throws IOException { super.open(taskNumber, numTasks); } } - + public static Builder impalaBuilder() { return new Builder(); } - + public static class Builder { private Integer authMech; private String keytabPath; @@ -178,24 +177,24 @@ public Builder setPrincipal(String principal) { this.principal = principal; return this; } - + public ImpalaOutputFormat build() { checkNotNull(options, "No options supplied."); checkNotNull(fieldNames, "No fieldNames supplied."); return new ImpalaOutputFormat( - options, - fieldNames, - keyFields, - partitionFields, - fieldTypes, - flushMaxSize, - flushIntervalMills, - allReplace, - updateMode, - authMech, - keytabPath, - krb5confPath, - principal); + options, + fieldNames, + keyFields, + partitionFields, + fieldTypes, + flushMaxSize, + flushIntervalMills, + allReplace, + updateMode, + authMech, + keytabPath, + krb5confPath, + principal); } } } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index 9c4da810f..ad1945e94 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -41,9 +41,15 @@ public ImpalaSink() { @Override public ImpalaOutputFormat getOutputFormat() { + ImpalaDialect impalaDialect = new ImpalaDialect( + getFieldTypes(), + primaryKeys, + impalaTableInfo.getStoreType() + ); + JDBCOptions jdbcOptions = JDBCOptions.builder() .setDbUrl(getImpalaJdbcUrl()) - .setDialect(new ImpalaDialect(getFieldTypes(), primaryKeys)) + .setDialect(impalaDialect) .setUsername(userName) .setPassword(password) .setTableName(tableName) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java index 0d4857809..fccfc9c42 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java @@ -59,7 +59,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map authMechs = Arrays.asList(0, 1, 2, 3); if (!authMechs.contains(authMech)) { throw new IllegalArgumentException("The value of authMech is illegal, Please select 0, 1, 2, 3"); @@ -79,20 +79,26 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map 0, "impala field of partitionFields is required"); } From c4ab92cad0ef85d899f9ee1a57f62a8611686c03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Fri, 27 Nov 2020 18:02:00 +0800 Subject: [PATCH 14/80] [feat][hbase-sink] hbase add batchSize and batchInterval. --- .../sql/sink/hbase/HbaseOutputFormat.java | 132 +++++++++++++----- .../flink/sql/sink/hbase/HbaseSink.java | 8 ++ .../sql/sink/hbase/table/HbaseSinkParser.java | 28 ++-- .../sql/sink/hbase/table/HbaseTableInfo.java | 20 +++ 4 files changed, 143 insertions(+), 45 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 4fbaa9f39..84d9c3826 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -16,16 +16,15 @@ * limitations under the License. */ - package com.dtstack.flink.sql.sink.hbase; -import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.util.ExecutorThreadFactory; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; import org.apache.hadoop.hbase.AuthUtil; @@ -35,10 +34,8 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,10 +43,14 @@ import java.io.File; import java.io.IOException; import java.security.PrivilegedAction; -import java.util.LinkedHashMap; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; /** * @author: jingzhen@dtstack.com @@ -85,6 +86,15 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private transient ChoreService choreService; + private Integer batchSize; + private Long batchWaitInterval; + + private transient ScheduledExecutorService executor; + private transient ScheduledFuture scheduledFuture; + + private final List records = new ArrayList<>(); + + @Override public void configure(Configuration parameters) { LOG.warn("---configure---"); @@ -98,10 +108,22 @@ public void open(int taskNumber, int numTasks) throws IOException { table = conn.getTable(TableName.valueOf(tableName)); LOG.warn("---open end(get table from hbase) ---"); initMetric(); + + // 设置定时任务 + if (batchWaitInterval > 0) { + this.executor = Executors.newScheduledThreadPool( + 1, new ExecutorThreadFactory("hbase-sink-flusher")); + this.scheduledFuture = this.executor.scheduleAtFixedRate(() -> { + if (!records.isEmpty()) { + dealBatchOperation(records); + records.clear(); + } + }, batchWaitInterval, batchWaitInterval, TimeUnit.MILLISECONDS); + } } - private void openConn(){ - try{ + private void openConn() { + try { if (kerberosAuthEnable) { LOG.info("open kerberos conn"); openKerberosConn(); @@ -111,7 +133,7 @@ private void openConn(){ conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); conn = ConnectionFactory.createConnection(conf); } - }catch (Exception e){ + } catch (Exception e) { throw new RuntimeException(e); } @@ -151,16 +173,59 @@ private void openKerberosConn() throws Exception { } - @Override public void writeRecord(Tuple2 tuple2) { Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.f0; Row row = tupleTrans.f1; if (retract) { - dealInsert(row); - } else if (!retract && StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { - dealDelete(row); + if (this.batchSize != 0) { + writeBatchRecord(row); + } else { + dealInsert(row); + } + } + } + + public void writeBatchRecord(Row row) { + records.add(row); + // 数据累计到batchSize之后开始处理 + if (records.size() == this.batchSize) { + dealBatchOperation(records); + // 添加完数据之后数据清空records + records.clear(); + } + } + + protected void dealBatchOperation(List records) { + // A null in the result array means that the call for that action failed, even after retries. + Object[] results = new Object[records.size()]; + try { + List puts = new ArrayList<>(); + for (Row record : records) { + puts.add(getPutByRow(record)); + } + table.batch(puts, results); + + // 判断数据是否插入成功 + for (int i = 0; i < results.length; i++) { + if (results[i] == null) { + if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { + LOG.error("record insert failed ..{}", records.get(i).toString()); + } + // 脏数据记录 + outDirtyRecords.inc(); + } else { + // 输出结果条数记录 + outRecords.inc(); + } + } + // 打印结果 + if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0) { + LOG.info(records.toString()); + } + } catch (IOException | InterruptedException e) { + LOG.error("", e); } } @@ -187,26 +252,6 @@ protected void dealInsert(Row record) { outRecords.inc(); } - protected void dealDelete(Row record) { - String rowKey = buildRowKey(record); - if (!StringUtils.isEmpty(rowKey)) { - Delete delete = new Delete(Bytes.toBytes(rowKey)); - try { - table.delete(delete); - } catch (IOException e) { - if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { - LOG.error("record insert failed ..{}", record.toString()); - LOG.error("", e); - } - outDirtyRecords.inc(); - } - if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0) { - LOG.info(record.toString()); - } - outRecords.inc(); - } - } - private Put getPutByRow(Row record) { String rowKey = buildRowKey(record); if (StringUtils.isEmpty(rowKey)) { @@ -244,9 +289,9 @@ private String getRowKeyValues(Row record) { return rowKeyBuilder.getRowKey(row); } - private Map rowConvertMap(Row record){ + private Map rowConvertMap(Row record) { Map rowValue = Maps.newHashMap(); - for(int i = 0; i < columnNames.length; i++){ + for (int i = 0; i < columnNames.length; i++) { rowValue.put(columnNames[i], record.getField(i)); } return rowValue; @@ -258,7 +303,15 @@ public void close() throws IOException { conn.close(); conn = null; } + + if (scheduledFuture != null) { + scheduledFuture.cancel(false); + if (executor != null) { + executor.shutdownNow(); + } + } } + private HbaseOutputFormat() { } @@ -345,6 +398,15 @@ public HbaseOutputFormatBuilder setClientKeytabFile(String clientKeytabFile) { return this; } + public HbaseOutputFormatBuilder setBatchSize(Integer batchSize) { + format.batchSize = batchSize; + return this; + } + + public HbaseOutputFormatBuilder setBatchWaitInterval(Long batchWaitInterval) { + format.batchWaitInterval = batchWaitInterval; + return this; + } public HbaseOutputFormat finish() { Preconditions.checkNotNull(format.host, "zookeeperQuorum should be specified"); @@ -405,6 +467,8 @@ public String toString() { ", zookeeperSaslClient='" + zookeeperSaslClient + '\'' + ", clientPrincipal='" + clientPrincipal + '\'' + ", clientKeytabFile='" + clientKeytabFile + '\'' + + ", batchSize='" + batchSize + '\'' + + ", batchWaitInterval='" + batchWaitInterval + '\'' + '}'; } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index c19dfa5ad..3a7d1145c 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -65,6 +65,8 @@ public class HbaseSink implements RetractStreamTableSink, IStreamSinkGener< private String clientKeytabFile; private int parallelism = 1; + protected String batchSize; + protected String batchWaitInterval; public HbaseSink() { // TO DO NOTHING @@ -94,6 +96,9 @@ public HbaseSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { if (tmpSinkParallelism != null) { this.parallelism = tmpSinkParallelism; } + + this.batchSize = hbaseTableInfo.getBatchSize(); + this.batchWaitInterval = hbaseTableInfo.getBatchWaitInterval(); return this; } @@ -119,6 +124,9 @@ public DataStreamSink> consumeDataStream(DataStream hbaseConfig = Maps.newHashMap(); public HbaseTableInfo(){ @@ -233,4 +237,20 @@ public void setClientKeytabFile(String clientKeytabFile) { this.clientKeytabFile = clientKeytabFile; } + public String getBatchSize() { + return batchSize; + } + + public void setBatchSize(String batchSize) { + this.batchSize = batchSize; + } + + public String getBatchWaitInterval() { + return batchWaitInterval; + } + + public void setBatchWaitInterval(String batchWaitInterval) { + this.batchWaitInterval = batchWaitInterval; + } + } From ae2aaf9467eb9a2ecc4c53cda712b5b1e08e9d66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Sat, 28 Nov 2020 17:25:32 +0800 Subject: [PATCH 15/80] [opt] use copyOnWriteList replace ArrayList because of concurrent modification exception --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 84d9c3826..2c65713ec 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -47,6 +47,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -92,7 +93,7 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private transient ScheduledExecutorService executor; private transient ScheduledFuture scheduledFuture; - private final List records = new ArrayList<>(); + private final List records = new CopyOnWriteArrayList<>(); @Override From acb0e69389217adf9ae03f8ac24301a2bba56436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Mon, 7 Dec 2020 22:55:02 +0800 Subject: [PATCH 16/80] =?UTF-8?q?[fix]=20=E4=BF=AE=E5=A4=8D=E5=90=88?= =?UTF-8?q?=E5=B9=B6=E5=88=86=E6=94=AF=E4=B8=A2=E5=A4=B1=E7=9A=84=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index f0d5518d2..d0aa93927 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -30,8 +30,10 @@ import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.configuration.Configuration; import org.apache.flink.table.dataformat.BaseRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; import org.slf4j.Logger; @@ -41,6 +43,8 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.sql.Timestamp; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Calendar; import java.util.List; @@ -133,6 +137,7 @@ public void flatMap(Row value, Collector out) throws Exception { * @param obj * @return */ + @Override protected Object dealTimeAttributeType(Class entry, Object obj) { boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(entry); if (obj instanceof LocalDateTime && isTimeIndicatorTypeInfo) { From 241e7bd8bb7a39fde35835ab44c9cdb5fa238ced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Tue, 8 Dec 2020 17:23:22 +0800 Subject: [PATCH 17/80] [opt] resolve conflict from test 4.1.x --- .../java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java | 0 .../com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java | 2 -- 2 files changed, 2 deletions(-) delete mode 100644 impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java deleted file mode 100644 index e69de29bb..000000000 diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index e677e65bf..e093c5501 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -34,7 +34,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.rmi.RemoteException; import java.security.PrivilegedExceptionAction; import java.sql.Connection; import java.sql.DriverManager; @@ -52,7 +51,6 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; From 8ba87064d360ccf651ab3e740d032932889b7bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Wed, 9 Dec 2020 15:10:27 +0800 Subject: [PATCH 18/80] [fix][hbase] rowKey not available. --- .../com/dtstack/flink/sql/parser/CreateTableParser.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java index 925531240..d788885ea 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java @@ -20,6 +20,7 @@ import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Map; @@ -72,13 +73,18 @@ private Map parseProp(String propsStr){ for (String str : strings) { List ss = DtStringUtil.splitIgnoreQuota(str, '='); String key = ss.get(0).trim(); - String value = ss.get(1).trim().replaceAll("'", "").trim(); + String value = replaceWithQuota(ss.get(1).trim()); propMap.put(key, value); } return propMap; } + private static String replaceWithQuota(String str) { + String removeStart = StringUtils.removeStart(str, "'"); + return StringUtils.removeEnd(removeStart, "'"); + } + public static class SqlParserResult{ private String tableName; From 3d9139387e6e16ebe941d47791178fe5c7cbd2a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Wed, 16 Dec 2020 17:07:43 +0800 Subject: [PATCH 19/80] =?UTF-8?q?[feat][hbase]=20=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E7=BB=93=E6=9D=9F=E6=97=B6=E5=B0=86=E6=95=B0=E6=8D=AE=E5=86=99?= =?UTF-8?q?=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/hbase/HbaseOutputFormat.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 2c65713ec..beda5cfbc 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -300,9 +300,8 @@ private Map rowConvertMap(Row record) { @Override public void close() throws IOException { - if (conn != null) { - conn.close(); - conn = null; + if (!records.isEmpty()) { + dealBatchOperation(records); } if (scheduledFuture != null) { @@ -311,6 +310,12 @@ public void close() throws IOException { executor.shutdownNow(); } } + + if (conn != null) { + conn.close(); + conn = null; + } + } private HbaseOutputFormat() { From f934602e481863737976cc2ffc812b99a509d802 Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 17 Dec 2020 17:25:16 +0800 Subject: [PATCH 20/80] =?UTF-8?q?[fix-31342][core]=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E4=B8=8D=E4=B8=8A=EF=BC=8C=E9=9C=80=E8=A6=81?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E6=8F=90=E7=A4=BA=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/resource/ResourceCheck.java | 40 +++++++ .../sql/side/mysql/table/MysqlSideParser.java | 6 +- .../side/rdb/resource/JdbcResourceCheck.java | 107 ++++++++++++++++++ .../sql/side/rdb/table/RdbSideParser.java | 3 + .../sql/side/rdb/table/RdbSideTableInfo.java | 19 +++- 5 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java create mode 100644 rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/resource/JdbcResourceCheck.java diff --git a/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java b/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java new file mode 100644 index 000000000..d002a246e --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.resource; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; + +/** + * @author: chuixue + * @create: 2020-12-08 17:21 + * @description:资源检测 + **/ +public abstract class ResourceCheck { + /** + * 数据库连接的最大重试次数 + */ + protected static final int MAX_RETRY_TIMES = 3; + + /** + * 资源可用性检测 + * + * @param abstractSideTableInfo 连接信息 + */ + public abstract void checkResourceStatus(AbstractSideTableInfo abstractSideTableInfo); +} diff --git a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java index b0a38e344..6f0ad0c32 100644 --- a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java +++ b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java @@ -20,10 +20,13 @@ package com.dtstack.flink.sql.side.mysql.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; +import static com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo.DRIVER_NAME; + /** * Reason: * Date: 2018/7/25 @@ -38,7 +41,8 @@ public class MysqlSideParser extends RdbSideParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + props.put(DRIVER_NAME, "com.mysql.jdbc.Driver"); + RdbSideTableInfo mysqlTableInfo = (RdbSideTableInfo) super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/resource/JdbcResourceCheck.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/resource/JdbcResourceCheck.java new file mode 100644 index 000000000..cfdd7e420 --- /dev/null +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/resource/JdbcResourceCheck.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.rdb.resource; + +import com.dtstack.flink.sql.resource.ResourceCheck; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import org.apache.flink.runtime.execution.SuppressRestartsException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.concurrent.TimeUnit; + +/** + * @author: chuixue + * @create: 2020-12-08 17:24 + * @description:jdbc资源检测 + **/ +public class JdbcResourceCheck extends ResourceCheck { + private static final Logger LOG = LoggerFactory.getLogger(JdbcResourceCheck.class); + private static final JdbcResourceCheck Instance = new JdbcResourceCheck(); + + private JdbcResourceCheck() { + } + + public static JdbcResourceCheck getInstance() { + return Instance; + } + + @Override + public void checkResourceStatus(AbstractSideTableInfo abstractSideTableInfo) { + RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) abstractSideTableInfo; + Connection connection = null; + forName(rdbSideTableInfo.getDriverName()); + for (int i = 0; i < MAX_RETRY_TIMES; i++) { + try { + if (rdbSideTableInfo.getUserName() == null) { + connection = DriverManager.getConnection(rdbSideTableInfo.getUrl()); + } else { + connection = DriverManager.getConnection(rdbSideTableInfo.getUrl(), rdbSideTableInfo.getUserName(), rdbSideTableInfo.getPassword()); + } + if (null != connection) { + // 这里不能抛出RuntimeException，用户可能只有某个table的读取权限，而没有读取meta表权限 + try { + // 如果没有异常，说明能有权限读取meta表，如果找不到，则抛出RuntimeException + if (!connection.getMetaData().getTables(null, rdbSideTableInfo.getSchema(), rdbSideTableInfo.getTableName(), null).next()) { + LOG.error("Table " + rdbSideTableInfo.getTableName() + " doesn't exist"); + throw new SuppressRestartsException(new Throwable("Table " + rdbSideTableInfo.getTableName() + " doesn't exist")); + } + } catch (SQLException e) { + LOG.error(e.getMessage()); + } + break; + } + } catch (SQLException sqe) { + if (i == MAX_RETRY_TIMES - 1) { + throw new SuppressRestartsException(new Throwable(sqe)); + } + LOG.error(sqe.getMessage() + " retry " + (i + 1) + " time"); + try { + TimeUnit.SECONDS.sleep(3); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } finally { + if (null != connection) { + try { + connection.close(); + } catch (SQLException e) { + throw new SuppressRestartsException(new Throwable(e)); + } + } + } + } + } + + /** + * @param clazz + */ + public synchronized static void forName(String clazz) { + try { + Class driverClass = Class.forName(clazz); + driverClass.newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java index 79942ae11..3d25a3599 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java @@ -25,6 +25,8 @@ import java.util.Map; +import static com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo.DRIVER_NAME; + /** * Reason: * Date: 2018/11/26 @@ -48,6 +50,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map * http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.rdb.table; import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.resource.JdbcResourceCheck; import com.google.common.base.Preconditions; /** @@ -31,6 +32,8 @@ public class RdbSideTableInfo extends AbstractSideTableInfo { private static final long serialVersionUID = -1L; + public static final String DRIVER_NAME = "driverName"; + public static final String URL_KEY = "url"; public static final String TABLE_NAME_KEY = "tableName"; @@ -49,9 +52,12 @@ public boolean check() { Preconditions.checkNotNull(password, "rdb of password is required"); Preconditions.checkArgument(getFieldList().size() == getFieldExtraInfoList().size(), "fields and fieldExtraInfoList attributes must be the same length"); + JdbcResourceCheck.getInstance().checkResourceStatus(this); return true; } + private String driverName; + private String url; private String tableName; @@ -102,6 +108,14 @@ public void setPassword(String password) { this.password = password; } + public String getDriverName() { + return driverName; + } + + public void setDriverName(String driverName) { + this.driverName = driverName; + } + @Override public String toString() { String cacheInfo = super.toString(); @@ -109,6 +123,7 @@ public String toString() { "url='" + url + '\'' + ", tableName='" + tableName + '\'' + ", schema='" + schema + '\'' + + ", driverName='" + driverName + '\'' + '}'; return cacheInfo + " , " + connectionInfo; } From 5063d240b8995f0da46c093f296d7838aab0b92d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Thu, 17 Dec 2020 21:26:26 +0800 Subject: [PATCH 21/80] [feat] add dirty manager --- {dirtyDataManager => dirtyData}/console/pom.xml | 0 .../flink/sql/dirty/print/ConsoleDirtyDataConsumer.java | 0 .../flink/sql/dirty/print/TestPrintDirtyDataConsumer.java | 0 {dirtyDataManager => dirtyData}/mysql/pom.xml | 0 .../dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java | 0 {dirtyDataManager => dirtyData}/pom.xml | 0 pom.xml | 3 +++ 7 files changed, 3 insertions(+) rename {dirtyDataManager => dirtyData}/console/pom.xml (100%) rename {dirtyDataManager => dirtyData}/console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java (100%) rename {dirtyDataManager => dirtyData}/console/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java (100%) rename {dirtyDataManager => dirtyData}/mysql/pom.xml (100%) rename {dirtyDataManager => dirtyData}/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java (100%) rename {dirtyDataManager => dirtyData}/pom.xml (100%) diff --git a/dirtyDataManager/console/pom.xml b/dirtyData/console/pom.xml similarity index 100% rename from dirtyDataManager/console/pom.xml rename to dirtyData/console/pom.xml diff --git a/dirtyDataManager/console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java b/dirtyData/console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java similarity index 100% rename from dirtyDataManager/console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java rename to dirtyData/console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java diff --git a/dirtyDataManager/console/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java b/dirtyData/console/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java similarity index 100% rename from dirtyDataManager/console/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java rename to dirtyData/console/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java diff --git a/dirtyDataManager/mysql/pom.xml b/dirtyData/mysql/pom.xml similarity index 100% rename from dirtyDataManager/mysql/pom.xml rename to dirtyData/mysql/pom.xml diff --git a/dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java b/dirtyData/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java similarity index 100% rename from dirtyDataManager/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java rename to dirtyData/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java diff --git a/dirtyDataManager/pom.xml b/dirtyData/pom.xml similarity index 100% rename from dirtyDataManager/pom.xml rename to dirtyData/pom.xml diff --git a/pom.xml b/pom.xml index 1bad3677d..ea3709f77 100644 --- a/pom.xml +++ b/pom.xml @@ -38,6 +38,9 @@ tidb kingbase aws + + + dirtyData From d65c07dc20aa6163927aa40d02ae3325d624809f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Mon, 21 Dec 2020 17:57:23 +0800 Subject: [PATCH 22/80] [feat] dirty manager add to kafka source and hbase sink. --- .../consumer/AbstractDirtyDataConsumer.java | 13 +- .../consumer/DirtyConsumerFactory.java | 66 ++++++++++ .../dirtyManager/entity/DirtyDataEntity.java | 9 +- .../manager/DirtyDataManager.java | 122 +++++++----------- .../sql/dirtyManager/manager/TestMain.java | 15 +-- .../flink/sql/exec/ExecuteProcessHelper.java | 59 +++++++-- .../dtstack/flink/sql/exec/ParamsInfo.java | 43 +++++- .../format/DeserializationMetricWrapper.java | 17 ++- .../com/dtstack/flink/sql/option/Options.java | 9 ++ .../flink/sql/table/AbstractTableInfo.java | 85 ++++++------ .../sql/table/AbstractTableInfoParser.java | 72 ++++++----- .../ConsoleDirtyDataConsumer.java | 26 ++-- .../TestPrintDirtyDataConsumer.java | 2 +- .../dirty/mysql/MysqlDirtyDataConsumer.java | 39 +++--- .../sql/sink/hbase/HbaseOutputFormat.java | 23 ++-- .../flink/sql/sink/hbase/HbaseSink.java | 8 ++ .../kafka/AbstractKafkaConsumerFactory.java | 4 +- .../KafkaDeserializationMetricWrapper.java | 9 +- .../source/kafka/table/KafkaSourceParser.java | 2 +- launcher/pom.xml | 6 - .../flink/sql/launcher/LauncherMain.java | 15 ++- .../sql/launcher/entity/JobParamsInfo.java | 56 ++++++-- .../flink/sql/localTest/LocalTest.java | 20 ++- pom.xml | 5 + .../CustomerSocketTextStreamFunction.java | 3 +- 25 files changed, 482 insertions(+), 246 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/DirtyConsumerFactory.java rename dirtyData/console/src/main/java/com/dtstack/flink/sql/dirty/{print => console}/ConsoleDirtyDataConsumer.java (65%) rename dirtyData/console/src/test/java/com/dtstack/flink/sql/dirty/{print => console}/TestPrintDirtyDataConsumer.java (79%) diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java index 6669a2895..5d6759187 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/AbstractDirtyDataConsumer.java @@ -23,8 +23,9 @@ import org.slf4j.LoggerFactory; import java.io.Serializable; -import java.util.Map; +import java.util.Properties; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -65,7 +66,7 @@ public abstract class AbstractDirtyDataConsumer implements Runnable, Serializabl * @param properties 任务参数 * @throws Exception throw exception */ - public abstract void init(Map properties) throws Exception; + public abstract void init(Properties properties) throws Exception; /** * 检验consumer是否正在执行 @@ -77,15 +78,13 @@ public boolean isRunning() { @Override public void run() { try { - LOG.info("start to consume dirty data"); while (isRunning.get()) { consume(); } - LOG.info("consume dirty data end"); } catch (Exception e) { LOG.error("consume dirtyData error", e); errorCount.incrementAndGet(); - if (errorCount.get() == errorLimit) { + if (errorCount.get() > errorLimit) { throw new RuntimeException("The task failed due to the number of dirty data consume failed reached the limit " + errorLimit); } } @@ -95,4 +94,8 @@ public AbstractDirtyDataConsumer setQueue(LinkedBlockingQueue q this.queue = queue; return this; } + + public void collectDirtyData(DirtyDataEntity dataEntity, long blockingInterval) throws InterruptedException { + queue.offer(dataEntity, blockingInterval, TimeUnit.MILLISECONDS); + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/DirtyConsumerFactory.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/DirtyConsumerFactory.java new file mode 100644 index 000000000..fea07357d --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/consumer/DirtyConsumerFactory.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.dirtyManager.consumer; + +import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.dtstack.flink.sql.util.PluginUtil; + +import java.io.File; +import java.lang.reflect.Constructor; +import java.util.Objects; + +import static com.dtstack.flink.sql.util.PluginUtil.upperCaseFirstChar; + +/** + * @author tiezhu + * Company dtstack + * Date 2020/12/21 星期一 + */ +public class DirtyConsumerFactory { + private static final String DEFAULT_DIRTY_TYPE = "console"; + private static final String DIRTY_CONSUMER_PATH = "dirtyData"; + private static final String CLASS_PRE_STR = "com.dtstack.flink.sql.dirty"; + private static final String CLASS_POST_STR = "DirtyDataConsumer"; + + /** + * 通过动态方式去加载脏数据插件 + * + * @param dirtyType 脏数据插件类型 + * @param pluginPath 脏数据插件直地址 + * @param pluginLoadMode 插件加载方式 + * @return 脏数据消费者 + * @throws Exception exception + */ + public static AbstractDirtyDataConsumer getDirtyConsumer( + String dirtyType + , String pluginPath + , String pluginLoadMode) throws Exception { + if (Objects.isNull(dirtyType)) { + dirtyType = DEFAULT_DIRTY_TYPE; + } + String consumerType = DIRTY_CONSUMER_PATH + File.separator + dirtyType; + String consumerJar = PluginUtil.getJarFileDirPath(consumerType, pluginPath, pluginLoadMode); + String className = CLASS_PRE_STR + "." + dirtyType.toLowerCase() + "." + upperCaseFirstChar(dirtyType + CLASS_POST_STR); + return ClassLoaderManager.newInstance(consumerJar, cl -> { + Class clazz = cl.loadClass(className); + Constructor constructor = clazz.getConstructor(); + return (AbstractDirtyDataConsumer) constructor.newInstance(); + }); + } +} diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java index 3d8d96653..42466970d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/entity/DirtyDataEntity.java @@ -87,13 +87,18 @@ public DirtyDataEntity(String dirtyData, Long processDate, String cause, String this.field = field; } + public DirtyDataEntity(String dirtyData, Long processDate, String cause) { + this.dirtyData = dirtyData; + this.processDate = timeFormat.format(processDate); + this.cause = cause; + } + @Override public String toString() { return "DirtyDataEntity{" + "dirtyData='" + dirtyData + '\'' + ", processDate=" + processDate + ", cause='" + cause + '\'' + - ", field='" + field + '\'' + '}'; } @@ -102,6 +107,6 @@ public String toString() { * @return 脏数据信息字符数组 */ public String[] get() { - return new String[] {dirtyData, String.valueOf(processDate), cause, field}; + return new String[] {dirtyData, String.valueOf(processDate), cause}; } } diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java index 88e2b24cc..b0b6cab6d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java @@ -18,18 +18,16 @@ package com.dtstack.flink.sql.dirtyManager.manager; -import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.alibaba.fastjson.JSONObject; import com.dtstack.flink.sql.dirtyManager.consumer.AbstractDirtyDataConsumer; +import com.dtstack.flink.sql.dirtyManager.consumer.DirtyConsumerFactory; import com.dtstack.flink.sql.dirtyManager.entity.DirtyDataEntity; import com.dtstack.flink.sql.factory.DTThreadFactory; -import com.dtstack.flink.sql.util.PluginUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.Serializable; -import java.lang.reflect.Constructor; -import java.util.Map; +import java.util.Properties; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -42,83 +40,68 @@ */ public class DirtyDataManager implements Serializable { + public final static int MAX_POOL_SIZE_LIMIT = 5; private static final long serialVersionUID = 7190970299538893497L; - private static final Logger LOG = LoggerFactory.getLogger(DirtyDataManager.class); + private static final String DIRTY_BLOCK_STR = "blockingInterval"; + private static final String DIRTY_LIMIT_RATE_STR = "errorLimitRate"; + private final static int MAX_TASK_QUEUE_SIZE = 100; + private final static String DEFAULT_ERROR_LIMIT_RATE = "0.8"; + private final static String DEFAULT_BLOCKING_INTERVAL = "60"; + public static AbstractDirtyDataConsumer consumer; - private static final String CLASS_PRE_STR = "com.dtstack.flink.sql.dirty"; - - private static final String CLASS_POST_STR = "DirtyDataConsumer"; - - private static final String DIRTY_CONSUMER_PATH = "dirtyData"; - - /** - * 写入队列阻塞时间 - */ - private long blockingInterval; - - /** - * 缓存脏数据信息队列 - */ - public final LinkedBlockingQueue queue = new LinkedBlockingQueue<>(); - + private static ThreadPoolExecutor dirtyDataConsumer; /** * 统计manager收集到的脏数据条数 */ private final AtomicLong count = new AtomicLong(0); - /** * 脏数据写入队列失败条数 */ private final AtomicLong errorCount = new AtomicLong(0); - + /** + * 写入队列阻塞时间 + */ + private long blockingInterval; + /** + * 任务失败的脏数据比例 + */ private double errorLimitRate; - public static AbstractDirtyDataConsumer consumer; - - private static ThreadPoolExecutor dirtyDataConsumer; - - public final static int MAX_POOL_SIZE_LIMIT = 5; - - private final static int MAX_TASK_QUEUE_SIZE = 100; - - private final static String DEFAULT_TYPE = "console"; - - private final static String DEFAULT_ERROR_LIMIT_RATE = "0.8"; - - private final static String DEFAULT_BLOCKING_INTERVAL = "60"; - /** * 通过参数生成manager实例，并同时将consumer实例化 */ - public static DirtyDataManager newInstance(Map properties) throws Exception { - DirtyDataManager manager = new DirtyDataManager(); - manager.blockingInterval = Long.parseLong(properties.getOrDefault("blockingInterval", DEFAULT_BLOCKING_INTERVAL)); - manager.errorLimitRate = Double.parseDouble(properties.getOrDefault("errorLimitRate", DEFAULT_ERROR_LIMIT_RATE)); - consumer = createConsumer(properties); - consumer.init(properties); - consumer.setQueue(manager.queue); - dirtyDataConsumer = new ThreadPoolExecutor(MAX_POOL_SIZE_LIMIT, MAX_POOL_SIZE_LIMIT, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(MAX_TASK_QUEUE_SIZE), new DTThreadFactory("dirtyDataConsumer"), new ThreadPoolExecutor.CallerRunsPolicy()); - dirtyDataConsumer.execute(consumer); - - return manager; + public static DirtyDataManager newInstance(Properties properties) { + try { + DirtyDataManager manager = new DirtyDataManager(); + manager.blockingInterval = Long.parseLong(String.valueOf(properties.getOrDefault(DIRTY_BLOCK_STR, DEFAULT_BLOCKING_INTERVAL))); + manager.errorLimitRate = Double.parseDouble(String.valueOf(properties.getOrDefault(DIRTY_LIMIT_RATE_STR, DEFAULT_ERROR_LIMIT_RATE))); + consumer = DirtyConsumerFactory.getDirtyConsumer( + properties.getProperty("type") + , properties.getProperty("pluginPath") + , properties.getProperty("pluginLoadMode") + ); + consumer.init(properties); + consumer.setQueue(new LinkedBlockingQueue<>()); + dirtyDataConsumer = new ThreadPoolExecutor(MAX_POOL_SIZE_LIMIT, MAX_POOL_SIZE_LIMIT, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(MAX_TASK_QUEUE_SIZE), new DTThreadFactory("dirtyDataConsumer"), new ThreadPoolExecutor.CallerRunsPolicy()); + dirtyDataConsumer.execute(consumer); + return manager; + } catch (Exception e) { + throw new RuntimeException("create dirtyManager error!", e); + } } /** - * 通过动态加载的方式加载Consumer + * 设置脏数据插件默认配置 + * + * @return console的默认配置 */ - private static AbstractDirtyDataConsumer createConsumer(Map properties) throws Exception { - String type = properties.getOrDefault("type", DEFAULT_TYPE); - String consumerType = DIRTY_CONSUMER_PATH + File.separator + type; - String consumerJar = PluginUtil.getJarFileDirPath(consumerType, properties.getOrDefault("pluginPath", null), "shipfile"); - String className = CLASS_PRE_STR + "." + type.toLowerCase() + "." + upperCaseFirstChar(type + CLASS_POST_STR); - - return ClassLoaderManager.newInstance(consumerJar, cl -> { - Class clazz = cl.loadClass(className); - Constructor constructor = clazz.getConstructor(); - return (AbstractDirtyDataConsumer) constructor.newInstance(); - }); + public static String buildDefaultDirty() { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("type", "console"); + jsonObject.put("printLimit", "1000"); + return jsonObject.toJSONString(); } /** @@ -136,16 +119,16 @@ public void close() { /** * 收集脏数据放入队列缓存中，记录放入失败的数目和存入队列中的总数目，如果放入失败的数目超过一定比例，那么manager任务失败 */ - public void collectDirtyData(String dataInfo, String cause, String field) { - DirtyDataEntity dirtyDataEntity = new DirtyDataEntity(dataInfo, System.currentTimeMillis(), cause, field); + public void collectDirtyData(String dataInfo, String cause) { + DirtyDataEntity dirtyDataEntity = new DirtyDataEntity(dataInfo, System.currentTimeMillis(), cause); try { - queue.offer(dirtyDataEntity, blockingInterval, TimeUnit.MILLISECONDS); + consumer.collectDirtyData(dirtyDataEntity, blockingInterval); count.incrementAndGet(); } catch (Exception ignored) { LOG.warn("dirty Data insert error ... Failed number: " + errorCount.incrementAndGet()); LOG.warn("error dirty data:" + dirtyDataEntity.toString()); if (errorCount.get() > Math.ceil(count.longValue() * errorLimitRate)) { - throw new RuntimeException("The number of failed number reaches the limit, manager fails"); + throw new RuntimeException(String.format("The number of failed number 【%s】 reaches the limit, manager fails", errorCount.get())); } } } @@ -156,11 +139,4 @@ public void collectDirtyData(String dataInfo, String cause, String field) { public boolean checkConsumer() { return consumer.isRunning(); } - - /** - * 首字母大写 - */ - private static String upperCaseFirstChar(String str) { - return str.substring(0, 1).toUpperCase() + str.substring(1); - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java index 8ec2296c1..5d8f38382 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java @@ -1,7 +1,6 @@ package com.dtstack.flink.sql.dirtyManager.manager; -import java.util.HashMap; -import java.util.Map; +import java.util.Properties; /** * @author tiezhu @@ -12,11 +11,11 @@ public class TestMain { private static final Integer DATA_NUMBER = 1000; public static void main(String[] args) throws Exception { - Map properties = new HashMap<>(8); - properties.put("type", "mysql"); - properties.put("pluginPath", "/Users/wtz/IdeaProjects/flinkStreamSQLTemp/sqlplugins"); - properties.put("url", "jdbc:mysql://kerberos01:3306/tiezhu"); - properties.put("userName", "dtstack"); + Properties properties = new Properties(); + properties.put("type", "console"); + properties.put("pluginPath", "/Users/wtz/IdeaProjects/flinkStreamSQLDemoThree/sqlplugins"); + properties.put("url", "jdbc:mysql://localhost:3306/tiezhu"); + properties.put("userName", "root"); properties.put("password", "abc123"); properties.put("isCreatedTable", "false"); properties.put("batchSize", "1"); @@ -26,7 +25,7 @@ public static void main(String[] args) throws Exception { for (int i = 0; i < DATA_NUMBER; i++) { Thread.sleep(100); manager.collectDirtyData("testDirtyData" + i, - new Exception("testException" + i).getMessage(), "testField"); + new Exception("testException" + i).getMessage()); if (i == 50) { manager.close(); } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 37749956c..523b827eb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.exec; import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.dtstack.flink.sql.dirtyManager.manager.DirtyDataManager; import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.enums.EPluginLoadMode; @@ -79,6 +80,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Properties; import java.util.Set; import java.util.TimeZone; @@ -97,6 +99,8 @@ public class ExecuteProcessHelper { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final String TIME_ZONE = "timezone"; + private static final String PLUGIN_PATH_STR = "pluginPath"; + private static final String PLUGIN_LOAD_STR = "pluginLoadMode"; public static FlinkPlanner flinkPlanner = new FlinkPlanner(); @@ -114,11 +118,23 @@ public static ParamsInfo parseParams(String[] args) throws Exception { String remoteSqlPluginPath = options.getRemoteSqlPluginPath(); String pluginLoadMode = options.getPluginLoadMode(); String deployMode = options.getMode(); + String dirtyStr = options.getDirtyProperties(); Preconditions.checkArgument(checkRemoteSqlPluginPath(remoteSqlPluginPath, deployMode, pluginLoadMode), "Non-local mode or shipfile deployment mode, remoteSqlPluginPath is required"); String confProp = URLDecoder.decode(options.getConfProp(), Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); + Properties dirtyProperties = PluginUtil.jsonStrToObject(Objects.isNull(dirtyStr) ? + DirtyDataManager.buildDefaultDirty() : dirtyStr, Properties.class); + + if (Objects.isNull(dirtyProperties.getProperty(PLUGIN_LOAD_STR))) { + dirtyProperties.put(PLUGIN_LOAD_STR, pluginLoadMode); + } + + if (!pluginLoadMode.equalsIgnoreCase(EPluginLoadMode.LOCALTEST.name()) && Objects.isNull(dirtyProperties.getProperty(PLUGIN_PATH_STR))) { + dirtyProperties.setProperty(PLUGIN_PATH_STR, + Objects.isNull(remoteSqlPluginPath) ? localSqlPluginPath : remoteSqlPluginPath); + } List jarUrlList = getExternalJarUrls(options.getAddjar()); @@ -131,12 +147,14 @@ public static ParamsInfo parseParams(String[] args) throws Exception { .setDeployMode(deployMode) .setConfProp(confProperties) .setJarUrlList(jarUrlList) + .setDirtyProperties(dirtyProperties) .build(); } /** - * 非local模式或者shipfile部署模式，remoteSqlPluginPath必填 + * 非local模式或者shipfile部署模式，remoteSqlPluginPath必填 + * * @param remoteSqlPluginPath * @param deployMode * @param pluginLoadMode @@ -165,12 +183,20 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf //register udf ExecuteProcessHelper.registerUserDefinedFunction(sqlTree, paramsInfo.getJarUrlList(), tableEnv, paramsInfo.isGetPlan()); //register table schema - Set classPathSets = ExecuteProcessHelper.registerTable(sqlTree, env, tableEnv, paramsInfo.getLocalSqlPluginPath(), - paramsInfo.getRemoteSqlPluginPath(), paramsInfo.getPluginLoadMode(), sideTableMap, registerTableCache); + Set classPathSets = ExecuteProcessHelper.registerTable( + sqlTree + , env + , tableEnv + , paramsInfo.getLocalSqlPluginPath() + , paramsInfo.getRemoteSqlPluginPath() + , paramsInfo.getPluginLoadMode() + , paramsInfo.getDirtyProperties() + , sideTableMap + , registerTableCache); // cache classPathSets ExecuteProcessHelper.registerPluginUrlToCachedFile(env, classPathSets); - ExecuteProcessHelper.sqlTranslation(paramsInfo.getLocalSqlPluginPath(), paramsInfo.getPluginLoadMode(),tableEnv, sqlTree, sideTableMap, registerTableCache); + ExecuteProcessHelper.sqlTranslation(paramsInfo.getLocalSqlPluginPath(), paramsInfo.getPluginLoadMode(), tableEnv, sqlTree, sideTableMap, registerTableCache); if (env instanceof MyLocalStreamEnvironment) { ((MyLocalStreamEnvironment) env).setClasspaths(ClassLoaderManager.getClassPath()); @@ -196,7 +222,7 @@ public static List getExternalJarUrls(String addJarListStr) throws java.io. private static void sqlTranslation(String localSqlPluginPath, String pluginLoadMode, StreamTableEnvironment tableEnv, - SqlTree sqlTree,Map sideTableMap, + SqlTree sqlTree, Map sideTableMap, Map registerTableCache) throws Exception { SideSqlExec sideSqlExec = new SideSqlExec(); @@ -270,24 +296,37 @@ public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrl } /** - * 向Flink注册源表和结果表，返回执行时插件包的全路径 + * 向Flink注册源表和结果表，返回执行时插件包的全路径 + * * @param sqlTree * @param env * @param tableEnv * @param localSqlPluginPath * @param remoteSqlPluginPath - * @param pluginLoadMode 插件加载模式 classpath or shipfile + * @param pluginLoadMode 插件加载模式 classpath or shipfile * @param sideTableMap * @param registerTableCache * @return * @throws Exception */ - public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { + public static Set registerTable( + SqlTree sqlTree + , StreamExecutionEnvironment env + , StreamTableEnvironment tableEnv + , String localSqlPluginPath + , String remoteSqlPluginPath + , String pluginLoadMode + , Properties dirtyProperties + , Map sideTableMap + , Map registerTableCache + ) throws Exception { Set pluginClassPathSets = Sets.newHashSet(); WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); for (AbstractTableInfo tableInfo : sqlTree.getTableInfoMap().values()) { + // 配置dirty manager + tableInfo.setDirtyProperties(dirtyProperties); + if (tableInfo instanceof AbstractSourceTableInfo) { AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; @@ -388,7 +427,7 @@ public static StreamTableEnvironment getStreamTableEnv(StreamExecutionEnvironmen private static void timeZoneCheck(String timeZone) { ArrayList zones = Lists.newArrayList(TimeZone.getAvailableIDs()); - if (!zones.contains(timeZone)){ + if (!zones.contains(timeZone)) { throw new IllegalArgumentException(String.format(" timezone of %s is Incorrect!", timeZone)); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java index 9619c75c1..50401cb3e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java @@ -21,7 +21,6 @@ import java.net.URL; import java.util.List; -import java.util.Objects; import java.util.Properties; /** @@ -41,9 +40,17 @@ public class ParamsInfo { private String deployMode; private Properties confProp; private boolean getPlan = false; - - public ParamsInfo(String sql, String name, List jarUrlList, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, String deployMode, Properties confProp) { + private Properties dirtyProperties; + + public ParamsInfo( + String sql + , String name, List jarUrlList + , String localSqlPluginPath + , String remoteSqlPluginPath + , String pluginLoadMode + , String deployMode + , Properties confProp + , Properties dirtyProperties) { this.sql = sql; this.name = name; this.jarUrlList = jarUrlList; @@ -52,6 +59,7 @@ public ParamsInfo(String sql, String name, List jarUrlList, String localSql this.pluginLoadMode = pluginLoadMode; this.deployMode = deployMode; this.confProp = confProp; + this.dirtyProperties = dirtyProperties; } public boolean isGetPlan() { @@ -94,17 +102,23 @@ public Properties getConfProp() { return confProp; } + public Properties getDirtyProperties() { + return dirtyProperties; + } + @Override public String toString() { return "ParamsInfo{" + "sql='" + sql + '\'' + ", name='" + name + '\'' + - ", jarUrlList=" + convertJarUrlListToString(jarUrlList) + + ", jarUrlList=" + jarUrlList + ", localSqlPluginPath='" + localSqlPluginPath + '\'' + ", remoteSqlPluginPath='" + remoteSqlPluginPath + '\'' + ", pluginLoadMode='" + pluginLoadMode + '\'' + ", deployMode='" + deployMode + '\'' + ", confProp=" + confProp + + ", getPlan=" + getPlan + + ", dirtyProperties='" + dirtyProperties + '\'' + '}'; } @@ -115,6 +129,7 @@ public String convertJarUrlListToString(List jarUrlList) { public static ParamsInfo.Builder builder() { return new ParamsInfo.Builder(); } + public static class Builder { private String sql; @@ -125,6 +140,7 @@ public static class Builder { private String pluginLoadMode; private String deployMode; private Properties confProp; + private Properties dirtyProperties; public ParamsInfo.Builder setSql(String sql) { this.sql = sql; @@ -167,9 +183,22 @@ public ParamsInfo.Builder setConfProp(Properties confProp) { return this; } + public ParamsInfo.Builder setDirtyProperties(Properties dirtyProperties) { + this.dirtyProperties = dirtyProperties; + return this; + } + public ParamsInfo build() { - return new ParamsInfo(sql, name, jarUrlList, localSqlPluginPath, - remoteSqlPluginPath, pluginLoadMode, deployMode, confProp); + return new ParamsInfo( + sql + , name + , jarUrlList + , localSqlPluginPath + , remoteSqlPluginPath + , pluginLoadMode + , deployMode + , confProp + , dirtyProperties); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java b/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java index 7c8424371..5696c1f61 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.format; +import com.dtstack.flink.sql.dirtyManager.manager.DirtyDataManager; import com.dtstack.flink.sql.metric.MetricConstant; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.api.common.serialization.AbstractDeserializationSchema; @@ -31,6 +32,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.charset.StandardCharsets; /** * add metric for source @@ -69,9 +71,15 @@ public class DeserializationMetricWrapper extends AbstractDeserializationSchema< protected transient Meter numInBytesRate; - public DeserializationMetricWrapper(TypeInformation typeInfo, DeserializationSchema deserializationSchema) { + protected DirtyDataManager dirtyDataManager; + + public DeserializationMetricWrapper( + TypeInformation typeInfo + , DeserializationSchema deserializationSchema + , DirtyDataManager dirtyDataManager) { super(typeInfo); this.deserializationSchema = deserializationSchema; + this.dirtyDataManager = dirtyDataManager; } public void initMetric() { @@ -91,7 +99,7 @@ public void initMetric() { public Row deserialize(byte[] message) throws IOException { try { if (numInRecord.getCount() % dataPrintFrequency == 0) { - LOG.info("receive source data:" + new String(message, "UTF-8")); + LOG.info("receive source data:" + new String(message, StandardCharsets.UTF_8)); } numInRecord.inc(); numInBytes.inc(message.length); @@ -102,10 +110,7 @@ public Row deserialize(byte[] message) throws IOException { return row; } catch (Exception e) { //add metric of dirty data - if (dirtyDataCounter.getCount() % dataPrintFrequency == 0) { - LOG.info("dirtyData: " + new String(message)); - LOG.error("data parse error", e); - } + dirtyDataManager.collectDirtyData(new String(message), e.getMessage()); dirtyDataCounter.inc(); return null; } diff --git a/core/src/main/java/com/dtstack/flink/sql/option/Options.java b/core/src/main/java/com/dtstack/flink/sql/option/Options.java index 2e2167227..18908b221 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/Options.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/Options.java @@ -72,6 +72,8 @@ public class Options { @OptionRequired(description = "file add to ship file") private String addShipfile; + @OptionRequired(description = "dirty plugin properties") + private String dirtyProperties; public String getMode() { return mode; @@ -185,4 +187,11 @@ public void setAddShipfile(String addShipfile) { this.addShipfile = addShipfile; } + public String getDirtyProperties() { + return dirtyProperties; + } + + public void setDirtyProperties(String dirtyProperties) { + this.dirtyProperties = dirtyProperties; + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java index f8fa9c3ef..5ee8cd53f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java @@ -24,47 +24,49 @@ import java.io.Serializable; import java.util.List; import java.util.Map; +import java.util.Properties; /** * Reason: * Date: 2018/6/22 * Company: www.dtstack.com + * * @author xuchao */ public abstract class AbstractTableInfo implements Serializable { public static final String PARALLELISM_KEY = "parallelism"; - + private final List fieldList = Lists.newArrayList(); + private final List fieldTypeList = Lists.newArrayList(); + private final List fieldClassList = Lists.newArrayList(); + private final List fieldExtraInfoList = Lists.newArrayList(); private String name; - private String type; - private String[] fields; - private String[] fieldTypes; - private Class[] fieldClasses; - - private final List fieldList = Lists.newArrayList(); - - /**key:别名, value: realField */ + /** + * key:别名, value: realField + */ private Map physicalFields = Maps.newLinkedHashMap(); - - private final List fieldTypeList = Lists.newArrayList(); - - private final List fieldClassList = Lists.newArrayList(); - - private final List fieldExtraInfoList = Lists.newArrayList(); - private List primaryKeys; private Integer parallelism = -1; + /** + * 构建脏数据插件的相关信息 + */ + private Properties dirtyProperties; + public String[] getFieldTypes() { return fieldTypes; } + public void setFieldTypes(String[] fieldTypes) { + this.fieldTypes = fieldTypes; + } + public abstract boolean check(); public String getType() { @@ -79,10 +81,18 @@ public String[] getFields() { return fields; } + public void setFields(String[] fields) { + this.fields = fields; + } + public Class[] getFieldClasses() { return fieldClasses; } + public void setFieldClasses(Class[] fieldClasses) { + this.fieldClasses = fieldClasses; + } + public List getPrimaryKeys() { return primaryKeys; } @@ -104,18 +114,18 @@ public Integer getParallelism() { } public void setParallelism(Integer parallelism) { - if(parallelism == null){ + if (parallelism == null) { return; } - if(parallelism <= 0){ + if (parallelism <= 0) { throw new RuntimeException("Abnormal parameter settings: parallelism > 0"); } this.parallelism = parallelism; } - public void addField(String fieldName){ + public void addField(String fieldName) { if (fieldList.contains(fieldName)) { throw new RuntimeException("redundancy field name " + fieldName + " in table " + getName()); } @@ -123,30 +133,18 @@ public void addField(String fieldName){ fieldList.add(fieldName); } - public void addPhysicalMappings(String aliasName, String physicalFieldName){ + public void addPhysicalMappings(String aliasName, String physicalFieldName) { physicalFields.put(aliasName, physicalFieldName); } - public void addFieldClass(Class fieldClass){ + public void addFieldClass(Class fieldClass) { fieldClassList.add(fieldClass); } - public void addFieldType(String fieldType){ + public void addFieldType(String fieldType) { fieldTypeList.add(fieldType); } - public void setFields(String[] fields) { - this.fields = fields; - } - - public void setFieldTypes(String[] fieldTypes) { - this.fieldTypes = fieldTypes; - } - - public void setFieldClasses(Class[] fieldClasses) { - this.fieldClasses = fieldClasses; - } - public void addFieldExtraInfo(FieldExtraInfo extraInfo) { fieldExtraInfoList.add(extraInfo); } @@ -167,15 +165,24 @@ public Map getPhysicalFields() { return physicalFields; } + public void setPhysicalFields(Map physicalFields) { + this.physicalFields = physicalFields; + } + public List getFieldExtraInfoList() { return fieldExtraInfoList; } - public void setPhysicalFields(Map physicalFields) { - this.physicalFields = physicalFields; + public Properties getDirtyProperties() { + dirtyProperties.setProperty("tableName", this.name); + return dirtyProperties; + } + + public void setDirtyProperties(Properties dirtyProperties) { + this.dirtyProperties = dirtyProperties; } - public void finish(){ + public void finish() { this.fields = fieldList.toArray(new String[0]); this.fieldClasses = fieldClassList.toArray(new Class[0]); this.fieldTypes = fieldTypeList.toArray(new String[0]); @@ -183,7 +190,7 @@ public void finish(){ /** * field extra info，used to store `not null` `default 0`...， - * + *

* now, only support not null */ public static class FieldExtraInfo implements Serializable { @@ -193,7 +200,7 @@ public static class FieldExtraInfo implements Serializable { */ boolean notNull = false; /** - * field length,eg.char(4) + * field length,eg.char(4) */ int length; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java index dce4dbfb4..41d0d230e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java @@ -16,7 +16,7 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.table; @@ -50,75 +50,79 @@ public class AbstractTableInfoParser { private final static Pattern SIDE_PATTERN = Pattern.compile(SIDE_TABLE_SIGN); - private Map sourceTableInfoMap = Maps.newConcurrentMap(); + private Map sourceTableInfoMap = Maps.newConcurrentMap(); + + private Map targetTableInfoMap = Maps.newConcurrentMap(); - private Map targetTableInfoMap = Maps.newConcurrentMap(); + private Map sideTableInfoMap = Maps.newConcurrentMap(); - private Map sideTableInfoMap = Maps.newConcurrentMap(); + /** + * judge dim table of PERIOD FOR SYSTEM_TIME + * + * @param tableField + * @return + */ + private static boolean checkIsSideTable(String tableField) { + String[] fieldInfos = StringUtils.split(tableField, ","); + for (String field : fieldInfos) { + Matcher matcher = SIDE_PATTERN.matcher(field.trim()); + if (matcher.find()) { + return true; + } + } + + return false; + } //Parsing loaded plugin - public AbstractTableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserResult parserResult, - String localPluginRoot, String pluginLoadMode) throws Exception { + public AbstractTableInfo parseWithTableType( + int tableType + , CreateTableParser.SqlParserResult parserResult + , String localPluginRoot + , String pluginLoadMode + ) throws Exception { AbstractTableParser absTableParser = null; Map props = parserResult.getPropMap(); String type = MathUtil.getString(props.get(TYPE_KEY)); - if(Strings.isNullOrEmpty(type)){ + if (Strings.isNullOrEmpty(type)) { throw new RuntimeException("create table statement requires property of type"); } - if(tableType == ETableType.SOURCE.getType()){ + if (tableType == ETableType.SOURCE.getType()) { boolean isSideTable = checkIsSideTable(parserResult.getFieldsInfoStr()); - if(!isSideTable){ + if (!isSideTable) { absTableParser = sourceTableInfoMap.get(type); - if(absTableParser == null){ + if (absTableParser == null) { absTableParser = StreamSourceFactory.getSqlParser(type, localPluginRoot, pluginLoadMode); sourceTableInfoMap.put(type, absTableParser); } - }else{ + } else { absTableParser = sideTableInfoMap.get(type); - if(absTableParser == null){ + if (absTableParser == null) { String cacheType = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_KEY)); absTableParser = StreamSideFactory.getSqlParser(type, localPluginRoot, cacheType, pluginLoadMode); sideTableInfoMap.put(type + cacheType, absTableParser); } } - }else if(tableType == ETableType.SINK.getType()){ + } else if (tableType == ETableType.SINK.getType()) { absTableParser = targetTableInfoMap.get(type); - if(absTableParser == null){ + if (absTableParser == null) { absTableParser = StreamSinkFactory.getSqlParser(type, localPluginRoot, pluginLoadMode); targetTableInfoMap.put(type, absTableParser); } } - if(absTableParser == null){ + if (absTableParser == null) { throw new RuntimeException(String.format("not support %s type of table", type)); } Map prop = Maps.newHashMap(); //Shield case - parserResult.getPropMap().forEach((key,val) -> prop.put(key.toLowerCase(), val)); - + parserResult.getPropMap().forEach((key, val) -> prop.put(key.toLowerCase(), val)); return absTableParser.getTableInfo(parserResult.getTableName(), parserResult.getFieldsInfoStr(), prop); } - - /** - * judge dim table of PERIOD FOR SYSTEM_TIME - * @param tableField - * @return - */ - private static boolean checkIsSideTable(String tableField){ - String[] fieldInfos = StringUtils.split(tableField, ","); - for(String field : fieldInfos){ - Matcher matcher = SIDE_PATTERN.matcher(field.trim()); - if(matcher.find()){ - return true; - } - } - - return false; - } } diff --git a/dirtyData/console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java b/dirtyData/console/src/main/java/com/dtstack/flink/sql/dirty/console/ConsoleDirtyDataConsumer.java similarity index 65% rename from dirtyData/console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java rename to dirtyData/console/src/main/java/com/dtstack/flink/sql/dirty/console/ConsoleDirtyDataConsumer.java index 59f6151e2..3e6e2412e 100644 --- a/dirtyData/console/src/main/java/com/dtstack/flink/sql/dirty/print/ConsoleDirtyDataConsumer.java +++ b/dirtyData/console/src/main/java/com/dtstack/flink/sql/dirty/console/ConsoleDirtyDataConsumer.java @@ -16,14 +16,15 @@ * limitations under the License. */ -package com.dtstack.flink.sql.dirty.print; +package com.dtstack.flink.sql.dirty.console; import com.dtstack.flink.sql.dirtyManager.consumer.AbstractDirtyDataConsumer; import com.dtstack.flink.sql.dirtyManager.entity.DirtyDataEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Map; +import java.util.Objects; +import java.util.Properties; /** * @author tiezhu @@ -33,16 +34,22 @@ public class ConsoleDirtyDataConsumer extends AbstractDirtyDataConsumer { private static final long serialVersionUID = 5727194679865135189L; - private final Logger LOG = LoggerFactory.getLogger(ConsoleDirtyDataConsumer.class); + private static final Logger LOG = LoggerFactory.getLogger(ConsoleDirtyDataConsumer.class); + + private static final Long DEFAULT_PRINT_LIMIT = 1000L; + + private Long printLimit; @Override public void consume() throws InterruptedException { DirtyDataEntity dataEntity = queue.take(); count.incrementAndGet(); - LOG.warn("get dirtyData: " + dataEntity.getDirtyData() + "\n" - + "cause: " + dataEntity.getCause() + "\n" - + "processTime: " + dataEntity.getProcessDate() + "\n" - + "error field: " + dataEntity.getField()); + if (count.get() % printLimit == 0) { + LOG.warn("\nget dirtyData: " + dataEntity.getDirtyData() + "\n" + + "cause: " + dataEntity.getCause() + "\n" + + "processTime: " + dataEntity.getProcessDate() + "\n" + + "error field: " + dataEntity.getField()); + } } @Override @@ -52,7 +59,10 @@ public void close() { } @Override - public void init(Map properties) { + public void init(Properties properties) { LOG.info("console dirty consumer init ..."); + Object printLimit = properties.get("printLimit"); + this.printLimit = Objects.isNull(printLimit) ? + DEFAULT_PRINT_LIMIT : Long.parseLong(String.valueOf(printLimit)); } } diff --git a/dirtyData/console/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java b/dirtyData/console/src/test/java/com/dtstack/flink/sql/dirty/console/TestPrintDirtyDataConsumer.java similarity index 79% rename from dirtyData/console/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java rename to dirtyData/console/src/test/java/com/dtstack/flink/sql/dirty/console/TestPrintDirtyDataConsumer.java index de8864114..3d1d64c41 100644 --- a/dirtyData/console/src/test/java/com/dtstack/flink/sql/dirty/print/TestPrintDirtyDataConsumer.java +++ b/dirtyData/console/src/test/java/com/dtstack/flink/sql/dirty/console/TestPrintDirtyDataConsumer.java @@ -1,4 +1,4 @@ -package com.dtstack.flink.sql.dirty.print; +package com.dtstack.flink.sql.dirty.console; /** * @author tiezhu diff --git a/dirtyData/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java b/dirtyData/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java index 00edb7ad3..8d425e7e1 100644 --- a/dirtyData/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java +++ b/dirtyData/mysql/src/main/java/com/dtstack/flink/sql/dirty/mysql/MysqlDirtyDataConsumer.java @@ -31,7 +31,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; +import java.util.Objects; +import java.util.Properties; import java.util.stream.Collectors; /** @@ -47,11 +48,11 @@ public class MysqlDirtyDataConsumer extends AbstractDirtyDataConsumer { private static final int CONN_VALID_TIME = 1000; - private static final Integer FIELD_NUMBER = 5; + private static final Integer FIELD_NUMBER = 4; private final Object LOCK_STR = new Object(); - private final String[] tableField = {"id", "dirtyData", "processTime", "cause", "field"}; + private final String[] tableField = {"id", "dirtyData", "processTime", "cause"}; private PreparedStatement statement; @@ -77,7 +78,7 @@ private void beforeConsume(String url, .map(this::quoteIdentifier) .collect(Collectors.joining(", ")); String insertSql = "INSERT INTO " + quoteIdentifier(tableName) - + "(" + insertField + ") VALUES (?, ?, ?, ?, ?)"; + + "(" + insertField + ") VALUES (?, ?, ?, ?)"; statement = connection.prepareStatement(insertSql); } } @@ -98,11 +99,10 @@ private void createTable(String tableName) throws SQLException { String sql = "CREATE TABLE IF NOT EXISTS \n" + quoteIdentifier(tableName) + " (\n" + - " `id` int(11) not null AUTO_INCREMENT,\n" + - " `dirtyData` varchar(255) DEFAULT NULL,\n" + + " `id` bigint not null AUTO_INCREMENT,\n" + + " `dirtyData` text DEFAULT NULL,\n" + " `processTime` varchar(255) DEFAULT NULL,\n" + - " `cause` varchar(255) DEFAULT NULL,\n" + - " `field` varchar(255) DEFAULT NULL,\n" + + " `cause` text DEFAULT NULL,\n" + " PRIMARY KEY (id)\n" + ") DEFAULT CHARSET=utf8;"; statement = connection.createStatement(); @@ -125,7 +125,7 @@ public void consume() throws Exception { data.add(String.valueOf(count.get())); Collections.addAll(data, entity.get()); for (int i = 0; i < FIELD_NUMBER; i++) { - statement.setString(i + 1, data.get(i)); + statement.setString(i + 1, Objects.isNull(data.get(i)) ? null : data.get(i)); } statement.addBatch(); @@ -153,18 +153,21 @@ public void close() { } @Override - public void init(Map properties) throws Exception { + public void init(Properties properties) throws Exception { SimpleDateFormat timeFormat = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss"); - String tableName = properties.getOrDefault("tableName", - "DirtyDataFromMysql_" + timeFormat.format(System.currentTimeMillis())); - String userName = properties.get("userName"); - String password = properties.get("password"); - String url = properties.get("url"); - batchSize = Long.parseLong(properties.getOrDefault("batchSize", "10000")); - errorLimit = Long.parseLong(properties.getOrDefault("errorLimit", "1000")); + String tableName = (String) properties.getOrDefault("dirtyTableName", + "DirtyData_" + + properties.getProperty("tableName") + "_" + + timeFormat.format(System.currentTimeMillis())); + + String userName = (String) properties.get("userName"); + String password = (String) properties.get("password"); + String url = (String) properties.get("url"); + batchSize = Long.parseLong((String) properties.getOrDefault("batchSize", "10000")); + errorLimit = Long.parseLong((String) properties.getOrDefault("errorLimit", "1000")); boolean isCreatedTable = Boolean.parseBoolean( - properties.getOrDefault("isCreatedTable", "false")); + (String) properties.getOrDefault("isCreatedTable", "false")); beforeConsume(url, userName, password, tableName, isCreatedTable); } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 4fbaa9f39..3ed52be14 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.sink.hbase; +import com.dtstack.flink.sql.dirtyManager.manager.DirtyDataManager; import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.google.common.collect.Maps; @@ -46,7 +47,6 @@ import java.io.File; import java.io.IOException; import java.security.PrivilegedAction; -import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -85,6 +85,8 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private transient ChoreService choreService; + private DirtyDataManager dirtyDataManager; + @Override public void configure(Configuration parameters) { LOG.warn("---configure---"); @@ -174,10 +176,9 @@ protected void dealInsert(Row record) { try { table.put(put); } catch (Exception e) { - if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { - LOG.error("record insert failed ..{}", record.toString()); - LOG.error("", e); - } + dirtyDataManager.collectDirtyData( + record.toString() + , e.getMessage()); outDirtyRecords.inc(); } @@ -194,10 +195,9 @@ protected void dealDelete(Row record) { try { table.delete(delete); } catch (IOException e) { - if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { - LOG.error("record insert failed ..{}", record.toString()); - LOG.error("", e); - } + dirtyDataManager.collectDirtyData( + record.toString() + , e.getMessage()); outDirtyRecords.inc(); } if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0) { @@ -345,6 +345,11 @@ public HbaseOutputFormatBuilder setClientKeytabFile(String clientKeytabFile) { return this; } + public HbaseOutputFormatBuilder setDirtyManager(DirtyDataManager dirtyDataManager) { + format.dirtyDataManager = dirtyDataManager; + return this; + } + public HbaseOutputFormat finish() { Preconditions.checkNotNull(format.host, "zookeeperQuorum should be specified"); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index c19dfa5ad..63df6c5e8 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.sink.hbase; +import com.dtstack.flink.sql.dirtyManager.manager.DirtyDataManager; import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.hbase.table.HbaseTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; @@ -36,6 +37,7 @@ import org.apache.flink.types.Row; import java.util.Map; +import java.util.Properties; /** * Date: 2018/09/14 @@ -65,6 +67,8 @@ public class HbaseSink implements RetractStreamTableSink, IStreamSinkGener< private String clientKeytabFile; private int parallelism = 1; + private Properties dirtyProperties; + public HbaseSink() { // TO DO NOTHING @@ -90,6 +94,8 @@ public HbaseSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.clientKeytabFile = hbaseTableInfo.getClientKeytabFile(); this.clientPrincipal = hbaseTableInfo.getClientPrincipal(); + this.dirtyProperties = hbaseTableInfo.getDirtyProperties(); + Integer tmpSinkParallelism = hbaseTableInfo.getParallelism(); if (tmpSinkParallelism != null) { this.parallelism = tmpSinkParallelism; @@ -119,6 +125,8 @@ public DataStreamSink> consumeDataStream(DataStream createDeserializationSchema(KafkaSourceTableInfo kafkaSourceTableInfo, TypeInformation typeInformation) { diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java index bf0cee7dd..102791ec2 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.source.kafka; +import com.dtstack.flink.sql.dirtyManager.manager.DirtyDataManager; import com.dtstack.flink.sql.format.DeserializationMetricWrapper; import org.apache.flink.api.common.serialization.DeserializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; @@ -58,8 +59,12 @@ public class KafkaDeserializationMetricWrapper extends DeserializationMetricWrap private Calculate calculate; - public KafkaDeserializationMetricWrapper(TypeInformation typeInfo, DeserializationSchema deserializationSchema, Calculate calculate) { - super(typeInfo, deserializationSchema); + public KafkaDeserializationMetricWrapper( + TypeInformation typeInfo + , DeserializationSchema deserializationSchema + , Calculate calculate + , DirtyDataManager dirtyDataManager) { + super(typeInfo, deserializationSchema, dirtyDataManager); this.calculate = calculate; } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 490aa2e6d..52db5b156 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -41,7 +41,6 @@ public class KafkaSourceParser extends AbstractSourceParser { public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { KafkaSourceTableInfo kafkaSourceTableInfo = new KafkaSourceTableInfo(); - parseFieldsInfo(fieldsInfo, kafkaSourceTableInfo); kafkaSourceTableInfo.setName(tableName); kafkaSourceTableInfo.setType(MathUtil.getString(props.get(KafkaSourceTableInfo.TYPE_KEY.toLowerCase()))); @@ -68,6 +67,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Mapslf4j-log4j12 1.6.1 - - - com.alibaba - fastjson - 1.2.70 - diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 1ad7e7d73..07b2fa4a2 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -19,14 +19,15 @@ package com.dtstack.flink.sql.launcher; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.TypeReference; +import com.dtstack.flink.sql.Main; +import com.dtstack.flink.sql.dirtyManager.manager.DirtyDataManager; +import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.launcher.entity.JobParamsInfo; import com.dtstack.flink.sql.launcher.executor.StandaloneExecutor; import com.dtstack.flink.sql.launcher.executor.YarnJobClusterExecutor; import com.dtstack.flink.sql.launcher.executor.YarnSessionClusterExecutor; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.TypeReference; -import com.dtstack.flink.sql.enums.ClusterMode; -import com.dtstack.flink.sql.Main; import com.dtstack.flink.sql.option.OptionParser; import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.util.PluginUtil; @@ -41,6 +42,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Properties; /** @@ -72,12 +74,15 @@ public static JobParamsInfo parseArgs(String[] args) throws Exception { String queue = launcherOptions.getQueue(); String pluginLoadMode = launcherOptions.getPluginLoadMode(); String addShipfile = launcherOptions.getAddShipfile(); + String dirtyStr = launcherOptions.getDirtyProperties(); String yarnSessionConf = URLDecoder.decode(launcherOptions.getYarnSessionConf(), Charsets.UTF_8.toString()); Properties yarnSessionConfProperties = PluginUtil.jsonStrToObject(yarnSessionConf, Properties.class); String confProp = URLDecoder.decode(launcherOptions.getConfProp(), Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); + Properties dirtyProperties = PluginUtil.jsonStrToObject(Objects.isNull(dirtyStr) ? + DirtyDataManager.buildDefaultDirty() : dirtyStr, Properties.class); return JobParamsInfo.builder() .setExecArgs(execArgs) @@ -92,10 +97,12 @@ public static JobParamsInfo parseArgs(String[] args) throws Exception { .setFlinkJarPath(flinkJarPath) .setPluginLoadMode(pluginLoadMode) .setQueue(queue) + .setDirtyProperties(dirtyProperties) .setAddShipfile(addShipfile) .build(); } + private static String[] parseJson(String[] args) { BufferedReader reader = null; StringBuilder lastStr = new StringBuilder(); diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/entity/JobParamsInfo.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/entity/JobParamsInfo.java index 9e7529ef0..628b847ff 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/entity/JobParamsInfo.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/entity/JobParamsInfo.java @@ -43,10 +43,23 @@ public class JobParamsInfo { private final Properties confProperties; private final Properties yarnSessionConfProperties; private final String addShipFile; - - private JobParamsInfo(String mode, String name, String queue, String localPluginRoot, String flinkConfDir, String yarnConfDir, - String pluginLoadMode, String[] execArgs, Properties confProperties, Properties yarnSessionConfProperties, - String udfJar, String flinkJarPath, String addShipFile) { + private final Properties dirtyProperties; + + private JobParamsInfo( + String mode + , String name + , String queue + , String localPluginRoot + , String flinkConfDir + , String yarnConfDir + , String pluginLoadMode + , String[] execArgs + , Properties confProperties + , Properties yarnSessionConfProperties + , String udfJar + , String flinkJarPath + , String addShipFile + , Properties dirtyProperties) { this.mode = mode; this.name = name; this.queue = queue; @@ -60,6 +73,11 @@ private JobParamsInfo(String mode, String name, String queue, String localPlugin this.udfJar = udfJar; this.flinkJarPath = flinkJarPath; this.addShipFile = addShipFile; + this.dirtyProperties = dirtyProperties; + } + + public static JobParamsInfo.Builder builder() { + return new JobParamsInfo.Builder(); } public String getMode() { @@ -114,11 +132,10 @@ public String getAddShipFile() { return addShipFile; } - public static JobParamsInfo.Builder builder() { - return new JobParamsInfo.Builder(); + public Properties getDirtyProperties() { + return dirtyProperties; } - public static class Builder { private String mode; private String name; @@ -133,6 +150,7 @@ public static class Builder { private Properties confProperties; private Properties yarnSessionConfProperties; private String addShipfile; + private Properties dirtyProperties; public JobParamsInfo.Builder setMode(String mode) { this.mode = mode; @@ -199,10 +217,27 @@ public JobParamsInfo.Builder setAddShipfile(String addShipfile) { return this; } + public JobParamsInfo.Builder setDirtyProperties(Properties dirtyProperties) { + this.dirtyProperties = dirtyProperties; + return this; + } + public JobParamsInfo build() { - return new JobParamsInfo(mode, name, queue, localPluginRoot, flinkConfDir, - yarnConfDir, pluginLoadMode, execArgs, confProperties, - yarnSessionConfProperties, udfJar, flinkJarPath, addShipfile); + return new JobParamsInfo( + mode + , name + , queue + , localPluginRoot + , flinkConfDir + , yarnConfDir + , pluginLoadMode + , execArgs + , confProperties + , yarnSessionConfProperties + , udfJar + , flinkJarPath + , addShipfile + , dirtyProperties); } } @@ -222,6 +257,7 @@ public String toString() { ", confProperties=" + confProperties + ", yarnSessionConfProperties=" + yarnSessionConfProperties + ", addShipFile='" + addShipFile + '\'' + + ", dirtyProperties=" + dirtyProperties + '}'; } } diff --git a/localTest/src/main/java/com/dtstack/flink/sql/localTest/LocalTest.java b/localTest/src/main/java/com/dtstack/flink/sql/localTest/LocalTest.java index c0e435ac0..9b198d276 100644 --- a/localTest/src/main/java/com/dtstack/flink/sql/localTest/LocalTest.java +++ b/localTest/src/main/java/com/dtstack/flink/sql/localTest/LocalTest.java @@ -50,7 +50,7 @@ public static void main(String[] args) throws Exception { setLogLevel("INFO"); List propertiesList = new ArrayList<>(); - String sqlPath = "/Users/chuixue/Desktop/tmp/sqlFile.sql"; + String sqlPath = "/Users/wtz/dtstack/sql/test/JoinDemoOne.sql"; Map conf = new HashMap<>(); JSONObject properties = new JSONObject(); @@ -65,6 +65,7 @@ public static void main(String[] args) throws Exception { conf.put("-name", "flinkStreamSQLLocalTest"); conf.put("-confProp", properties.toString()); conf.put("-pluginLoadMode", "LocalTest"); + conf.put("-dirtyProperties", buildDirtyStr()); for (Map.Entry keyValue : conf.entrySet()) { propertiesList.add(keyValue.getKey()); @@ -74,6 +75,23 @@ public static void main(String[] args) throws Exception { Main.main(propertiesList.toArray(new String[0])); } + private static String buildDirtyStr() { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("type", "console"); + // 多少条数据打印一次 + jsonObject.put("printLimit", "100"); + jsonObject.put("url", "jdbc:mysql://localhost:3306/tiezhu"); + jsonObject.put("userName", "root"); + jsonObject.put("password", "abc123"); + jsonObject.put("isCreateTable", "false"); + // 多少条数据写入一次 + jsonObject.put("batchSize", "1"); + jsonObject.put("tableName", "dirtyData"); + + return jsonObject.toJSONString(); + + } + private static String readSQL(String sqlPath) { try { byte[] array = Files.readAllBytes(Paths.get(sqlPath)); diff --git a/pom.xml b/pom.xml index ea3709f77..54270a19b 100644 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,11 @@ test ${jacoco.version} + + com.alibaba + fastjson + 1.2.70 + diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java index 5ea2d31a4..ca2450c40 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java @@ -17,6 +17,7 @@ */ package com.dtstack.flink.sql.source.serversocket; +import com.dtstack.flink.sql.dirtyManager.manager.DirtyDataManager; import com.dtstack.flink.sql.format.DeserializationMetricWrapper; import com.dtstack.flink.sql.format.dtnest.DtNestRowDeserializationSchema; import com.dtstack.flink.sql.source.serversocket.table.ServersocketSourceTableInfo; @@ -74,7 +75,7 @@ public CustomerSocketTextStreamFunction(ServersocketSourceTableInfo tableInfo, T Map rowAndFieldMapping, List fieldExtraInfos) { this.tableInfo = tableInfo; this.deserializationSchema = new DtNestRowDeserializationSchema(typeInfo, rowAndFieldMapping, fieldExtraInfos, CHARSET_NAME); - this.deserializationMetricWrapper = new DeserializationMetricWrapper(typeInfo, deserializationSchema); + this.deserializationMetricWrapper = new DeserializationMetricWrapper(typeInfo, deserializationSchema, DirtyDataManager.newInstance(tableInfo.getDirtyProperties())); } @Override From c38da770a1f5387f004d0d776dccf60c5b2cb150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Tue, 22 Dec 2020 15:04:51 +0800 Subject: [PATCH 23/80] [fix-33836][core] support POSITION func in SQL. --- .../flink/sql/side/JoinNodeDealer.java | 40 +++++++------------ .../flink/sql/util/FieldReplaceUtil.java | 14 +++---- 2 files changed, 20 insertions(+), 34 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index 99dcee0df..7fd24cabd 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -551,16 +551,13 @@ private void extractJoinField(SqlNode condition, Set> joi if ( AGGREGATE.contains(condition.getKind()) || AVG_AGG_FUNCTIONS.contains(joinKind) || COMPARISON.contains(joinKind) - || joinKind == OTHER_FUNCTION - || joinKind == DIVIDE - || joinKind == CAST - || joinKind == TRIM - || joinKind == TIMES - || joinKind == PLUS + || FUNCTION.contains(joinKind) + || COVAR_AVG_AGG_FUNCTIONS.contains(joinKind) + || BINARY_ARITHMETIC.contains(joinKind) + || BINARY_COMPARISON.contains(joinKind) || joinKind == NOT_IN || joinKind == OR || joinKind == AND - || joinKind == MINUS || joinKind == TUMBLE || joinKind == TUMBLE_START || joinKind == TUMBLE_END @@ -578,7 +575,7 @@ private void extractJoinField(SqlNode condition, Set> joi || joinKind == TIMESTAMP_DIFF || joinKind == LIKE || joinKind == COALESCE - || joinKind == EQUALS ){ + ){ SqlBasicCall sqlBasicCall = (SqlBasicCall) condition; for(int i=0; i tableMap){ SqlKind joinKind = node.getKind(); - if( AGGREGATE.contains(joinKind) + if(AGGREGATE.contains(joinKind) || AVG_AGG_FUNCTIONS.contains(joinKind) || COMPARISON.contains(joinKind) - || joinKind == OTHER_FUNCTION - || joinKind == DIVIDE - || joinKind == CAST - || joinKind == TRIM - || joinKind == TIMES - || joinKind == PLUS + || FUNCTION.contains(joinKind) + || COVAR_AVG_AGG_FUNCTIONS.contains(joinKind) + || BINARY_ARITHMETIC.contains(joinKind) + || BINARY_COMPARISON.contains(joinKind) || joinKind == NOT_IN || joinKind == OR || joinKind == AND - || joinKind == MINUS || joinKind == TUMBLE || joinKind == TUMBLE_START || joinKind == TUMBLE_END @@ -935,8 +926,7 @@ private SqlIdentifier checkAndReplaceJoinCondition(SqlNode node, Map Date: Tue, 22 Dec 2020 15:08:25 +0800 Subject: [PATCH 24/80] =?UTF-8?q?[doc]taskmanager=20slot=E5=8F=82=E6=95=B0?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/config.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/config.md b/docs/config.md index 465893d16..aa1860800 100644 --- a/docs/config.md +++ b/docs/config.md @@ -75,8 +75,7 @@ sh submit.sh -key1 val1 -key2 val2 * flinkCheckpointDataURI: 设置checkpoint的外部存储路径,根据实际的需求设定文件路径,hdfs://, file:// * jobmanager.memory.mb: per_job模式下指定jobmanager的内存大小(单位MB, 默认值:768) * taskmanager.memory.mb: per_job模式下指定taskmanager的内存大小(单位MB, 默认值:768) - * taskmanager.num: per_job模式下指定taskmanager的实例数(默认1) - * taskmanager.slots：per_job模式下指定每个taskmanager对应的slot数量(默认1) + * taskmanager.numberOfTaskSlots：per_job模式下指定每个taskmanager对应的slot数量(默认1),通过该参数和sql.env.parallelism可控制tm的个数,即sql.env.parallelism/taskmanager.numberOfTaskSlots 向上取整。 * savePointPath：任务恢复点的路径（默认无） * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） * logLevel: 日志级别动态配置（默认info） From 215acfc223255db4a5cc767e497fdb51155c9193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Tue, 22 Dec 2020 20:10:12 +0800 Subject: [PATCH 25/80] =?UTF-8?q?[opt-33307][core]=20=E8=B0=83=E6=95=B4?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E4=BF=A1=E6=81=AF=E5=92=8C=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/exec/ExecuteProcessHelper.java | 18 ++- .../sql/parser/CreateTmpTableParser.java | 147 +++++++++--------- .../flink/sql/parser/FlinkPlanner.java | 4 +- .../dtstack/flink/sql/parser/SqlParser.java | 81 +++++----- .../flink/sql/table/AbstractTableParser.java | 32 ++-- .../source/kafka/table/KafkaSourceParser.java | 3 +- 6 files changed, 154 insertions(+), 131 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 37749956c..02bfb020f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -170,7 +170,13 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf // cache classPathSets ExecuteProcessHelper.registerPluginUrlToCachedFile(env, classPathSets); - ExecuteProcessHelper.sqlTranslation(paramsInfo.getLocalSqlPluginPath(), paramsInfo.getPluginLoadMode(),tableEnv, sqlTree, sideTableMap, registerTableCache); + ExecuteProcessHelper.sqlTranslation( + paramsInfo.getLocalSqlPluginPath(), + paramsInfo.getPluginLoadMode(), + tableEnv, + sqlTree, + sideTableMap, + registerTableCache); if (env instanceof MyLocalStreamEnvironment) { ((MyLocalStreamEnvironment) env).setClasspaths(ClassLoaderManager.getClassPath()); @@ -282,8 +288,14 @@ public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrl * @return * @throws Exception */ - public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { + public static Set registerTable(SqlTree sqlTree, + StreamExecutionEnvironment env, + StreamTableEnvironment tableEnv, + String localSqlPluginPath, + String remoteSqlPluginPath, + String pluginLoadMode, + Map sideTableMap, + Map registerTableCache) throws Exception { Set pluginClassPathSets = Sets.newHashSet(); WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); for (AbstractTableInfo tableInfo : sqlTree.getTableInfoMap().values()) { diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java index 92a90ba51..bd475240f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java @@ -17,17 +17,17 @@ */ - package com.dtstack.flink.sql.parser; import com.dtstack.flink.sql.util.DtStringUtil; +import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlJoin; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlMatchRecognize; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlSelect; -import org.apache.calcite.sql.SqlMatchRecognize; -import com.google.common.collect.Lists; + import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -38,6 +38,7 @@ * parser create tmp table sql * Date: 2018/6/26 * Company: www.dtstack.com + * * @author yanxi */ public class CreateTmpTableParser implements IParser { @@ -51,15 +52,75 @@ public class CreateTmpTableParser implements IParser { private static final Pattern EMPTYVIEW = Pattern.compile(EMPTY_STR); - private FlinkPlanner flinkPlanner = new FlinkPlanner(); + private final FlinkPlanner flinkPlanner = new FlinkPlanner(); - public static CreateTmpTableParser newInstance(){ + public static CreateTmpTableParser newInstance() { return new CreateTmpTableParser(); } + private static void parseNode(SqlNode sqlNode, CreateTmpTableParser.SqlParserResult sqlParseResult) { + SqlKind sqlKind = sqlNode.getKind(); + switch (sqlKind) { + case SELECT: + SqlNode sqlFrom = ((SqlSelect) sqlNode).getFrom(); + if (sqlFrom.getKind() == IDENTIFIER) { + sqlParseResult.addSourceTable(sqlFrom.toString()); + } else { + parseNode(sqlFrom, sqlParseResult); + } + break; + case JOIN: + SqlNode leftNode = ((SqlJoin) sqlNode).getLeft(); + SqlNode rightNode = ((SqlJoin) sqlNode).getRight(); + + if (leftNode.getKind() == IDENTIFIER) { + sqlParseResult.addSourceTable(leftNode.toString()); + } else { + parseNode(leftNode, sqlParseResult); + } + + if (rightNode.getKind() == IDENTIFIER) { + sqlParseResult.addSourceTable(rightNode.toString()); + } else { + parseNode(rightNode, sqlParseResult); + } + break; + case AS: + //不解析column,所以 as 相关的都是表 + SqlNode identifierNode = ((SqlBasicCall) sqlNode).getOperands()[0]; + if (identifierNode.getKind() != IDENTIFIER) { + parseNode(identifierNode, sqlParseResult); + } else { + sqlParseResult.addSourceTable(identifierNode.toString()); + } + break; + case UNION: + SqlNode unionLeft = ((SqlBasicCall) sqlNode).getOperands()[0]; + SqlNode unionRight = ((SqlBasicCall) sqlNode).getOperands()[1]; + if (unionLeft.getKind() == IDENTIFIER) { + sqlParseResult.addSourceTable(unionLeft.toString()); + } else { + parseNode(unionLeft, sqlParseResult); + } + if (unionRight.getKind() == IDENTIFIER) { + sqlParseResult.addSourceTable(unionRight.toString()); + } else { + parseNode(unionRight, sqlParseResult); + } + break; + case MATCH_RECOGNIZE: + SqlMatchRecognize node = (SqlMatchRecognize) sqlNode; + sqlParseResult.addSourceTable(node.getTableRef().toString()); + break; + default: + //do nothing + break; + } + } + @Override public boolean verify(String sql) { - if (Pattern.compile(EMPTY_STR).matcher(sql).find()){ + if (Pattern.compile(EMPTY_STR).matcher(sql).find()) { return true; } return NONEMPTYVIEW.matcher(sql).find(); @@ -67,11 +128,11 @@ public boolean verify(String sql) { @Override public void parseSql(String sql, SqlTree sqlTree) { - if (NONEMPTYVIEW.matcher(sql).find()){ + if (NONEMPTYVIEW.matcher(sql).find()) { Matcher matcher = NONEMPTYVIEW.matcher(sql); String tableName = null; String selectSql = null; - if(matcher.find()) { + if (matcher.find()) { tableName = matcher.group(1); selectSql = "select " + matcher.group(2); } @@ -92,12 +153,11 @@ public void parseSql(String sql, SqlTree sqlTree) { sqlTree.addTmpSql(sqlParseResult); sqlTree.addTmplTableInfo(tableName, sqlParseResult); } else { - if (EMPTYVIEW.matcher(sql).find()) - { + if (EMPTYVIEW.matcher(sql).find()) { Matcher matcher = EMPTYVIEW.matcher(sql); String tableName = null; String fieldsInfoStr = null; - if (matcher.find()){ + if (matcher.find()) { tableName = matcher.group(1); fieldsInfoStr = matcher.group(2); } @@ -106,68 +166,6 @@ public void parseSql(String sql, SqlTree sqlTree) { sqlParseResult.setTableName(tableName); sqlTree.addTmplTableInfo(tableName, sqlParseResult); } - - } - - } - - private static void parseNode(SqlNode sqlNode, CreateTmpTableParser.SqlParserResult sqlParseResult){ - SqlKind sqlKind = sqlNode.getKind(); - switch (sqlKind){ - case SELECT: - SqlNode sqlFrom = ((SqlSelect)sqlNode).getFrom(); - if(sqlFrom.getKind() == IDENTIFIER){ - sqlParseResult.addSourceTable(sqlFrom.toString()); - }else{ - parseNode(sqlFrom, sqlParseResult); - } - break; - case JOIN: - SqlNode leftNode = ((SqlJoin)sqlNode).getLeft(); - SqlNode rightNode = ((SqlJoin)sqlNode).getRight(); - - if(leftNode.getKind() == IDENTIFIER){ - sqlParseResult.addSourceTable(leftNode.toString()); - }else{ - parseNode(leftNode, sqlParseResult); - } - - if(rightNode.getKind() == IDENTIFIER){ - sqlParseResult.addSourceTable(rightNode.toString()); - }else{ - parseNode(rightNode, sqlParseResult); - } - break; - case AS: - //不解析column,所以 as 相关的都是表 - SqlNode identifierNode = ((SqlBasicCall)sqlNode).getOperands()[0]; - if(identifierNode.getKind() != IDENTIFIER){ - parseNode(identifierNode, sqlParseResult); - }else { - sqlParseResult.addSourceTable(identifierNode.toString()); - } - break; - case UNION: - SqlNode unionLeft = ((SqlBasicCall)sqlNode).getOperands()[0]; - SqlNode unionRight = ((SqlBasicCall)sqlNode).getOperands()[1]; - if(unionLeft.getKind() == IDENTIFIER){ - sqlParseResult.addSourceTable(unionLeft.toString()); - }else{ - parseNode(unionLeft, sqlParseResult); - } - if(unionRight.getKind() == IDENTIFIER){ - sqlParseResult.addSourceTable(unionRight.toString()); - }else{ - parseNode(unionRight, sqlParseResult); - } - break; - case MATCH_RECOGNIZE: - SqlMatchRecognize node = (SqlMatchRecognize) sqlNode; - sqlParseResult.addSourceTable(node.getTableRef().toString()); - break; - default: - //do nothing - break; } } @@ -204,13 +202,12 @@ public void setFieldsInfoStr(String fieldsInfoStr) { this.fieldsInfoStr = fieldsInfoStr; } - public void addSourceTable(String sourceTable){ + public void addSourceTable(String sourceTable) { sourceTableList.add(sourceTable); } public List getSourceTableList() { return sourceTableList; } - } } \ No newline at end of file diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java b/core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java index 771fd46ce..d600a320f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java @@ -40,6 +40,7 @@ * Company: www.dtstack.com * @author maqi */ +@Deprecated public class FlinkPlanner { private final TableConfig tableConfig = new TableConfig(); @@ -54,7 +55,8 @@ public class FlinkPlanner { catalogManager, moduleManager); private final PlannerContext plannerContext = - new PlannerContext(tableConfig, + new PlannerContext( + tableConfig, functionCatalog, catalogManager, asRootSchema(new CatalogManagerCalciteSchema(catalogManager, false)), diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index 098703726..36dcaab67 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -17,16 +17,15 @@ */ - package com.dtstack.flink.sql.parser; import com.dtstack.flink.sql.enums.ETableType; import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.table.AbstractTableInfoParser; import com.dtstack.flink.sql.util.DtStringUtil; -import org.apache.commons.lang3.StringUtils; -import com.google.common.collect.Lists; import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +38,7 @@ * Reason: * Date: 2018/6/22 * Company: www.dtstack.com + * * @author xuchao */ @@ -46,29 +46,27 @@ public class SqlParser { private static final Logger LOG = LoggerFactory.getLogger(SqlParser.class); private static final char SQL_DELIMITER = ';'; - + private static final Pattern ADD_FILE_AND_JAR_PATTERN = Pattern.compile("(?i).*add\\s+file\\s+.+|(?i).*add\\s+jar\\s+.+"); private static String LOCAL_SQL_PLUGIN_ROOT; - private static List sqlParserList = Lists.newArrayList(CreateFuncParser.newInstance(), CreateTableParser.newInstance(), InsertSqlParser.newInstance(), CreateTmpTableParser.newInstance()); - public static void setLocalSqlPluginRoot(String localSqlPluginRoot){ + public static void setLocalSqlPluginRoot(String localSqlPluginRoot) { LOCAL_SQL_PLUGIN_ROOT = localSqlPluginRoot; } - private static final Pattern ADD_FILE_AND_JAR_PATTERN = Pattern.compile("(?i).*add\\s+file\\s+.+|(?i).*add\\s+jar\\s+.+"); - /** * flink support sql syntax * CREATE TABLE sls_stream() with (); * CREATE (TABLE|SCALA) FUNCTION fcnName WITH com.dtstack.com; * insert into tb1 select * from tb2; + * * @param sql */ public static SqlTree parseSql(String sql, String pluginLoadMode) throws Exception { - if(StringUtils.isBlank(sql)){ - throw new RuntimeException("sql is not null"); + if (StringUtils.isBlank(sql)) { + throw new IllegalArgumentException("SQL must be not empty!"); } sql = DtStringUtil.dealSqlComment(sql) @@ -80,12 +78,12 @@ public static SqlTree parseSql(String sql, String pluginLoadMode) throws Excepti sqlArr = removeAddFileAndJarStmt(sqlArr); SqlTree sqlTree = new SqlTree(); AbstractTableInfoParser tableInfoParser = new AbstractTableInfoParser(); - for(String childSql : sqlArr){ - if(Strings.isNullOrEmpty(childSql)){ + for (String childSql : sqlArr) { + if (Strings.isNullOrEmpty(childSql)) { continue; } boolean result = false; - for(IParser sqlParser : sqlParserList){ + for (IParser sqlParser : sqlParserList) { try { if (!sqlParser.verify(childSql)) { continue; @@ -100,67 +98,78 @@ public static SqlTree parseSql(String sql, String pluginLoadMode) throws Excepti } } - if(!result){ - throw new RuntimeException(String.format("%s:Syntax does not support,the format of SQL like insert into tb1 select * from tb2.", childSql)); + if (!result) { + throw new RuntimeException(String.format("%s \nSyntax like this does not support, the format of SQL like 'insert into tb1 select field from tb2'.", childSql)); } } //解析exec-sql - if(sqlTree.getExecSqlList().size() == 0){ + if (sqlTree.getExecSqlList().size() == 0) { throw new RuntimeException("sql no executable statement"); } - for(InsertSqlParser.SqlParseResult result : sqlTree.getExecSqlList()){ + for (InsertSqlParser.SqlParseResult result : sqlTree.getExecSqlList()) { List sourceTableList = result.getSourceTableList(); List targetTableList = result.getTargetTableList(); Set tmpTableList = sqlTree.getTmpTableMap().keySet(); - for(String tableName : sourceTableList){ - if (!tmpTableList.contains(tableName)){ + for (String tableName : sourceTableList) { + if (!tmpTableList.contains(tableName)) { CreateTableParser.SqlParserResult createTableResult = sqlTree.getPreDealTableMap().get(tableName); - if(createTableResult == null){ + if (createTableResult == null) { throw new RuntimeException("can't find table " + tableName); } - AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), - createTableResult, LOCAL_SQL_PLUGIN_ROOT, pluginLoadMode); + AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType( + ETableType.SOURCE.getType(), + createTableResult, + LOCAL_SQL_PLUGIN_ROOT, + pluginLoadMode + ); sqlTree.addTableInfo(tableName, tableInfo); } } - for(String tableName : targetTableList){ - if (!tmpTableList.contains(tableName)){ + for (String tableName : targetTableList) { + if (!tmpTableList.contains(tableName)) { CreateTableParser.SqlParserResult createTableResult = sqlTree.getPreDealTableMap().get(tableName); - if(createTableResult == null){ + if (createTableResult == null) { throw new RuntimeException("can't find table " + tableName); } - AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SINK.getType(), - createTableResult, LOCAL_SQL_PLUGIN_ROOT, pluginLoadMode); + AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType( + ETableType.SINK.getType(), + createTableResult, + LOCAL_SQL_PLUGIN_ROOT, + pluginLoadMode + ); sqlTree.addTableInfo(tableName, tableInfo); } } } - for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()){ + for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { List sourceTableList = result.getSourceTableList(); - for(String tableName : sourceTableList){ - if (!sqlTree.getTableInfoMap().keySet().contains(tableName)){ + for (String tableName : sourceTableList) { + if (!sqlTree.getTableInfoMap().containsKey(tableName)) { CreateTableParser.SqlParserResult createTableResult = sqlTree.getPreDealTableMap().get(tableName); - if(createTableResult == null){ + if (createTableResult == null) { CreateTmpTableParser.SqlParserResult tmpTableResult = sqlTree.getTmpTableMap().get(tableName); - if (tmpTableResult == null){ + if (tmpTableResult == null) { throw new RuntimeException("can't find table " + tableName); } } else { - AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), - createTableResult, LOCAL_SQL_PLUGIN_ROOT, pluginLoadMode); + AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType( + ETableType.SOURCE.getType(), + createTableResult, + LOCAL_SQL_PLUGIN_ROOT, + pluginLoadMode + ); sqlTree.addTableInfo(tableName, tableInfo); } } } } - return sqlTree; } @@ -171,7 +180,7 @@ private static List removeAddFileAndJarStmt(List stmts) { List cleanedStmts = Lists.newArrayList(); for (String stmt : stmts) { Matcher matcher = ADD_FILE_AND_JAR_PATTERN.matcher(stmt); - if(!matcher.matches()) { + if (!matcher.matches()) { cleanedStmts.add(stmt); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 1e7f72de6..e5ad0bf8d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -17,7 +17,6 @@ */ - package com.dtstack.flink.sql.table; import com.dtstack.flink.sql.util.ClassUtil; @@ -38,6 +37,7 @@ * Reason: * Date: 2018/7/4 * Company: www.dtstack.com + * * @author xuchao */ @@ -47,11 +47,11 @@ public abstract class AbstractTableParser { private static final String NEST_JSON_FIELD_KEY = "nestFieldKey"; private static final String CHAR_TYPE_NO_LENGTH = "CHAR"; - private static Pattern primaryKeyPattern = Pattern.compile("(?i)(^\\s*)PRIMARY\\s+KEY\\s*\\((.*)\\)"); - private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+?)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); - private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); - private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); - private static Pattern typePattern = Pattern.compile("(\\S+)\\s+(\\w+.*)"); + private static final Pattern primaryKeyPattern = Pattern.compile("(?i)(^\\s*)PRIMARY\\s+KEY\\s*\\((.*)\\)"); + private static final Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+?)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); + private static final Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); + private static final Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); + private static final Pattern typePattern = Pattern.compile("(\\S+)\\s+(\\w+.*)"); private Map patternMap = Maps.newHashMap(); @@ -69,14 +69,14 @@ protected boolean fieldNameNeedsUpperCase() { public abstract AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception; - public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ - for(Map.Entry keyPattern : patternMap.entrySet()){ + public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo) { + for (Map.Entry keyPattern : patternMap.entrySet()) { Pattern pattern = keyPattern.getValue(); String key = keyPattern.getKey(); Matcher matcher = pattern.matcher(fieldRow); - if(matcher.find()){ + if (matcher.find()) { ITableFieldDealHandler handler = handlerMap.get(key); - if(handler == null){ + if (handler == null) { throw new RuntimeException("parse field [" + fieldRow + "] error."); } @@ -92,11 +92,12 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { List fieldRows = DtStringUtil.splitField(fieldsInfo); - for (String fieldRow : fieldRows) { - fieldRow = fieldRow.trim(); + for (int i = 0; i < fieldRows.size(); i++) { + String fieldRow = fieldRows.get(i).trim(); if (StringUtils.isBlank(fieldRow)) { - throw new RuntimeException(String.format("table [%s],exists field empty.", tableInfo.getName())); + throw new RuntimeException(String.format("Empty field appears in position [%s] in table [%s]", + i + 1, tableInfo.getName())); } boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo); @@ -115,7 +116,7 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { if (matcher.find()) { fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); - fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); + fieldExtraInfo.setLength(Integer.parseInt(matcher.group(1))); } else { fieldClass = dbTypeConvertToJavaType(fieldType); } @@ -147,12 +148,13 @@ public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo) { List primaryKeys = Arrays .stream(primaryFields.split(",")) .map(String::trim) - .collect(Collectors.toList());; + .collect(Collectors.toList()); tableInfo.setPrimaryKeys(primaryKeys); } /** * add parser for alias field + * * @param matcher * @param tableInfo */ diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 490aa2e6d..085ef7613 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -41,9 +41,10 @@ public class KafkaSourceParser extends AbstractSourceParser { public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { KafkaSourceTableInfo kafkaSourceTableInfo = new KafkaSourceTableInfo(); - parseFieldsInfo(fieldsInfo, kafkaSourceTableInfo); kafkaSourceTableInfo.setName(tableName); + parseFieldsInfo(fieldsInfo, kafkaSourceTableInfo); + kafkaSourceTableInfo.setType(MathUtil.getString(props.get(KafkaSourceTableInfo.TYPE_KEY.toLowerCase()))); kafkaSourceTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(KafkaSourceTableInfo.PARALLELISM_KEY.toLowerCase()))); kafkaSourceTableInfo.setBootstrapServers(MathUtil.getString(props.get(KafkaSourceTableInfo.BOOTSTRAPSERVERS_KEY.toLowerCase()))); From 40a199263ec2fc12168dcb6ef16f73329240c238 Mon Sep 17 00:00:00 2001 From: chuixue Date: Wed, 23 Dec 2020 18:54:40 +0800 Subject: [PATCH 26/80] =?UTF-8?q?[fix-31342][core][rdb]=E5=AF=B9=E7=BB=B4?= =?UTF-8?q?=E8=A1=A8=E3=80=81=E7=BB=93=E6=9E=9C=E8=A1=A8=E8=BF=9E=E9=80=9A?= =?UTF-8?q?=E6=80=A7=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/resource/ResourceCheck.java | 6 +- .../flink/sql/side/AbstractSideTableInfo.java | 14 ++ .../sql/table/AbstractTargetTableInfo.java | 13 ++ .../flink/sql/sink/mysql/MysqlSink.java | 1 + .../sql/sink/mysql/table/MysqlSinkParser.java | 3 + .../side/rdb/all/AbstractRdbAllReqRow.java | 9 ++ .../sql/side/rdb/async/RdbAsyncReqRow.java | 13 +- .../side/rdb/resource/JdbcResourceCheck.java | 21 +-- .../sql/side/rdb/table/RdbSideParser.java | 3 +- .../sql/side/rdb/table/RdbSideTableInfo.java | 5 +- .../flink/sql/sink/rdb/AbstractRdbSink.java | 3 + .../rdb/format/AbstractJDBCOutputFormat.java | 2 +- .../rdb/format/JDBCUpsertOutputFormat.java | 31 +++- .../sink/rdb/resource/JdbcResourceCheck.java | 133 ++++++++++++++++++ .../sql/sink/rdb/table/RdbSinkParser.java | 2 + .../sql/sink/rdb/table/RdbTableInfo.java | 18 +++ .../sink/rdb/writer/AbstractUpsertWriter.java | 2 +- 17 files changed, 254 insertions(+), 25 deletions(-) create mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/resource/JdbcResourceCheck.java diff --git a/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java b/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java index d002a246e..5fee88fd3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java +++ b/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.resource; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; /** * @author: chuixue @@ -34,7 +34,7 @@ public abstract class ResourceCheck { /** * 资源可用性检测 * - * @param abstractSideTableInfo 连接信息 + * @param abstractTableInfo 连接信息 */ - public abstract void checkResourceStatus(AbstractSideTableInfo abstractSideTableInfo); + public abstract void checkResourceStatus(AbstractTableInfo abstractTableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index 635d171f4..203ff7443 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -46,6 +46,8 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements Serializable { + public static final String FAST_CHECK = "fastCheck"; + public static final String TARGET_SUFFIX = "Side"; public static final String CACHE_KEY = "cache"; @@ -93,6 +95,8 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements private List predicateInfoes = Lists.newArrayList(); + private boolean fastCheck; + public RowTypeInfo getRowTypeInfo(){ Class[] fieldClass = getFieldClasses(); TypeInformation[] types = new TypeInformation[fieldClass.length]; @@ -214,6 +218,15 @@ public Integer getConnectRetryMaxNum(Integer defaultValue) { public void setConnectRetryMaxNum(Integer connectRetryMaxNum) { this.connectRetryMaxNum = connectRetryMaxNum; } + + public boolean getFastCheck() { + return fastCheck; + } + + public void setFastCheck(boolean fastCheck) { + this.fastCheck = fastCheck; + } + @Override public String toString() { return "Cache Info{" + @@ -225,6 +238,7 @@ public String toString() { ", asyncPoolSize=" + asyncPoolSize + ", asyncFailMaxNum=" + asyncFailMaxNum + ", partitionedJoin=" + partitionedJoin + + ", fastCheck='" + fastCheck + ", cacheMode='" + cacheMode + '\'' + '}'; } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java index dbd99f993..48de59227 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java @@ -31,12 +31,16 @@ public abstract class AbstractTargetTableInfo extends AbstractTableInfo { + public static final String FAST_CHECK = "fastCheck"; + public static final String TARGET_SUFFIX = "Sink"; public static final String SINK_DATA_TYPE = "sinkdatatype"; private String sinkDataType = FormatType.JSON.name(); + private boolean fastCheck; + public String getSinkDataType() { return sinkDataType; } @@ -44,4 +48,13 @@ public String getSinkDataType() { public void setSinkDataType(String sinkDataType) { this.sinkDataType = sinkDataType; } + + public boolean getFastCheck() { + return fastCheck; + } + + public void setFastCheck(boolean fastCheck) { + this.fastCheck = fastCheck; + } + } diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java index 0a1749f04..3b1d260d7 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java @@ -58,6 +58,7 @@ public JDBCUpsertOutputFormat getOutputFormat() { .setKeyFields(primaryKeys) .setAllReplace(allReplace) .setUpdateMode(updateMode) + .setName(name) .build(); } } diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java index 49105a7a8..e6daa24b0 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java @@ -24,6 +24,8 @@ import java.util.Map; +import static com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo.DRIVER_NAME; + /** * Reason: * Date: 2018/7/4 @@ -37,6 +39,7 @@ public class MysqlSinkParser extends RdbSinkParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(DRIVER_NAME, "com.mysql.jdbc.Driver"); AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index d0aa93927..e782dec4b 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -20,6 +20,7 @@ import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.rdb.resource.JdbcResourceCheck; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import com.dtstack.flink.sql.util.JDBCUtils; @@ -72,12 +73,20 @@ public abstract class AbstractRdbAllReqRow extends BaseAllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); + private static volatile boolean resourceCheck = true; + public AbstractRdbAllReqRow(BaseSideInfo sideInfo) { super(sideInfo); } @Override public void open(Configuration parameters) throws Exception { + synchronized (AbstractRdbAllReqRow.class) { + if (resourceCheck) { + resourceCheck = false; + JdbcResourceCheck.getInstance().checkResourceStatus(sideInfo.getSideTableInfo()); + } + } super.open(parameters); RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); LOG.info("rdb dim table config info: {} ", tableInfo.toString()); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index bb5a9ea75..3ec316c30 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -25,6 +25,7 @@ import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.cache.CacheObj; +import com.dtstack.flink.sql.side.rdb.resource.JdbcResourceCheck; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import com.dtstack.flink.sql.util.DateUtil; @@ -40,17 +41,14 @@ import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.dataformat.BaseRow; import org.apache.flink.types.Row; -import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.math.BigDecimal; -import java.security.PrivilegedAction; import java.sql.Timestamp; import java.time.Instant; import java.util.List; import java.util.Map; -import java.util.TimeZone; import java.util.concurrent.CountDownLatch; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -91,13 +89,20 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private AtomicBoolean connectionStatus = new AtomicBoolean(true); + private static volatile boolean resourceCheck = true; + private transient ThreadPoolExecutor executor; private final static int MAX_TASK_QUEUE_SIZE = 100000; - @Override public void open(Configuration parameters) throws Exception { + synchronized (RdbAsyncReqRow.class) { + if (resourceCheck) { + resourceCheck = false; + JdbcResourceCheck.getInstance().checkResourceStatus(sideInfo.getSideTableInfo()); + } + } super.open(parameters); executor = new ThreadPoolExecutor(MAX_DB_CONN_POOL_SIZE_LIMIT, MAX_DB_CONN_POOL_SIZE_LIMIT, 0, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(MAX_TASK_QUEUE_SIZE), new DTThreadFactory("rdbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy()); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/resource/JdbcResourceCheck.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/resource/JdbcResourceCheck.java index cfdd7e420..357f31c42 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/resource/JdbcResourceCheck.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/resource/JdbcResourceCheck.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.side.rdb.resource; import com.dtstack.flink.sql.resource.ResourceCheck; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import org.apache.flink.runtime.execution.SuppressRestartsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,24 +47,25 @@ public static JdbcResourceCheck getInstance() { } @Override - public void checkResourceStatus(AbstractSideTableInfo abstractSideTableInfo) { - RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) abstractSideTableInfo; + public void checkResourceStatus(AbstractTableInfo abstractTableInfo) { + RdbSideTableInfo rdbTableInfo = (RdbSideTableInfo) abstractTableInfo; + LOG.info("Start check if table [{}] resource is valid !!!",rdbTableInfo.getName()); Connection connection = null; - forName(rdbSideTableInfo.getDriverName()); + forName(rdbTableInfo.getDriverName()); for (int i = 0; i < MAX_RETRY_TIMES; i++) { try { - if (rdbSideTableInfo.getUserName() == null) { - connection = DriverManager.getConnection(rdbSideTableInfo.getUrl()); + if (rdbTableInfo.getUserName() == null) { + connection = DriverManager.getConnection(rdbTableInfo.getUrl()); } else { - connection = DriverManager.getConnection(rdbSideTableInfo.getUrl(), rdbSideTableInfo.getUserName(), rdbSideTableInfo.getPassword()); + connection = DriverManager.getConnection(rdbTableInfo.getUrl(), rdbTableInfo.getUserName(), rdbTableInfo.getPassword()); } if (null != connection) { // 这里不能抛出RuntimeException，用户可能只有某个table的读取权限，而没有读取meta表权限 try { // 如果没有异常，说明能有权限读取meta表，如果找不到，则抛出RuntimeException - if (!connection.getMetaData().getTables(null, rdbSideTableInfo.getSchema(), rdbSideTableInfo.getTableName(), null).next()) { - LOG.error("Table " + rdbSideTableInfo.getTableName() + " doesn't exist"); - throw new SuppressRestartsException(new Throwable("Table " + rdbSideTableInfo.getTableName() + " doesn't exist")); + if (!connection.getMetaData().getTables(null, rdbTableInfo.getSchema(), rdbTableInfo.getTableName(), null).next()) { + LOG.error("Table " + rdbTableInfo.getTableName() + " doesn't exist"); + throw new SuppressRestartsException(new Throwable("Table " + rdbTableInfo.getTableName() + " doesn't exist")); } } catch (SQLException e) { LOG.error(e.getMessage()); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java index 3d25a3599..eeaad16fa 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java @@ -50,7 +50,8 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map, Serializable, IStreamSinkGener { + protected String name; + protected String dbUrl; protected String userName; @@ -93,6 +95,7 @@ public AbstractRdbSink(JDBCDialect jdbcDialect) { @Override public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { RdbTableInfo rdbTableInfo = (RdbTableInfo) targetTableInfo; + this.name = rdbTableInfo.getName(); this.batchNum = rdbTableInfo.getBatchSize() == null ? batchNum : rdbTableInfo.getBatchSize(); this.batchWaitInterval = rdbTableInfo.getBatchWaitInterval() == null ? batchWaitInterval : rdbTableInfo.getBatchWaitInterval(); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java index 2a3ce5e90..4b5270a12 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java @@ -47,7 +47,7 @@ public abstract class AbstractJDBCOutputFormat extends AbstractDtRichOutputFo protected final String username; protected final String password; - private final String driverName; + protected final String driverName; protected final String dbURL; protected transient Connection connection; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index 783c59e11..55453e939 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -23,18 +23,17 @@ import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import com.dtstack.flink.sql.sink.rdb.resource.JdbcResourceCheck; import com.dtstack.flink.sql.sink.rdb.writer.AppendOnlyWriter; import com.dtstack.flink.sql.sink.rdb.writer.JDBCWriter; import com.dtstack.flink.sql.sink.rdb.writer.AbstractUpsertWriter; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; -import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.security.PrivilegedAction; import java.sql.SQLException; import java.util.List; import java.util.concurrent.ScheduledExecutorService; @@ -57,6 +56,7 @@ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat driverClass = Class.forName(clazz); + driverClass.newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java index aca7acb6c..340f1a3ea 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java @@ -49,6 +49,8 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map fieldTypes[f]).toArray(); String deleteSql = dialect.getDeleteStatement(schema, tableName, keyFields); - LOG.info("deleteSQL is :{}", deleteSql); + // LOG.info("deleteSQL is :{}", deleteSql); Optional upsertSql = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); LOG.info("execute UpsertStatement: {}", upsertSql.orElse("use UsingInsertUpdateStatement")); From 2c0c5a2f8a3ac8aa9688abc3745f934eda5a516b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Sat, 26 Dec 2020 17:00:58 +0800 Subject: [PATCH 27/80] =?UTF-8?q?[feat][core][rdb]=20=E5=A2=9E=E5=8A=A0rdb?= =?UTF-8?q?=E6=A8=A1=E5=9D=97=E8=B5=84=E6=BA=90=E6=A3=80=E6=B5=8B=EF=BC=8C?= =?UTF-8?q?=E6=A0=A1=E9=AA=8C=E5=86=85=E5=AE=B9=E6=9C=89=EF=BC=9A=201.?= =?UTF-8?q?=E5=88=A4=E6=96=AD=E6=95=B0=E6=8D=AE=E6=BA=90=E6=98=AF=E5=90=A6?= =?UTF-8?q?=E5=8F=AF=E7=94=A8;=202.=E6=A0=B9=E6=8D=AE=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E5=92=8C=E7=BB=93=E6=9E=9C=E8=A1=A8=E5=88=A4=E6=96=AD=E5=BD=93?= =?UTF-8?q?=E5=89=8D=E7=94=A8=E6=88=B7=E6=98=AF=E5=90=A6=E6=9C=89=E5=AF=B9?= =?UTF-8?q?=E5=BA=94=E7=9A=84=E6=9D=83=E9=99=90=E3=80=90=E6=9C=AA=E5=AE=8C?= =?UTF-8?q?=E6=88=90=E9=83=A8=E5=88=86=EF=BC=9A=E7=BB=93=E6=9E=9C=E8=A1=A8?= =?UTF-8?q?update=E5=92=8Creplace=E7=9A=84=E6=9D=83=E9=99=90=E6=9A=82?= =?UTF-8?q?=E6=9C=AA=E6=A0=A1=E9=AA=8C=E3=80=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../side/clickhouse/ClickhouseAllReqRow.java | 4 +- .../table/ClickhouseSideParser.java | 2 + .../table/ClickhouseSinkParser.java | 2 + .../sql/classloader/ClassLoaderManager.java | 30 +++- .../flink/sql/resource/ResourceCheck.java | 15 +- .../flink/sql/table/AbstractTableInfo.java | 84 ++++++----- .../dtstack/flink/sql/util/ThreadUtil.java | 53 +++++++ .../sql/side/db2/table/Db2SideParser.java | 2 + .../flink/sql/sink/db/table/DbSinkParser.java | 2 + .../sql/side/impala/ImpalaAllReqRow.java | 6 +- .../side/impala/table/ImpalaSideParser.java | 6 +- .../sql/sink/impala/ImpalaOutputFormat.java | 58 ++++---- .../sink/impala/table/ImpalaSinkParser.java | 1 + .../sink/impala/table/ImpalaTableInfo.java | 10 ++ .../kingbase/table/KingbaseSideParser.java | 2 + .../kingbase/table/KingbaseSinkParser.java | 2 + .../sql/side/mysql/table/MysqlSideParser.java | 5 +- .../sql/sink/mysql/table/MysqlSinkParser.java | 5 +- .../oceanbase/table/OceanbaseSideParser.java | 2 + .../oceanbase/table/OceanbaseSinkParser.java | 2 + .../side/oracle/table/OracleSideParser.java | 2 + .../sink/oracle/table/OracleSinkParser.java | 2 + .../side/polardb/table/PolardbSideParser.java | 3 + .../sink/polardb/table/PolardbSinkParser.java | 3 + .../table/PostgresqlSideParser.java | 2 + .../table/PostgresqlSinkParser.java | 2 + rdb/pom.xml | 1 + rdb/rdb-core/pom.xml | 20 +++ .../flink/sql/core/rdb/JdbcCheckKeys.java | 39 +++++ .../flink/sql/core/rdb/JdbcResourceCheck.java | 119 ++++++++++++++++ .../sql/core/rdb/util/JdbcConnectUtil.java | 90 ++++++++---- rdb/rdb-side/pom.xml | 6 + .../side/rdb/all/AbstractRdbAllReqRow.java | 22 ++- .../sql/side/rdb/async/RdbAsyncReqRow.java | 86 +++++------ .../side/rdb/resource/JdbcResourceCheck.java | 108 -------------- .../sql/side/rdb/table/RdbSideParser.java | 1 + .../sql/side/rdb/table/RdbSideTableInfo.java | 47 ++++--- rdb/rdb-sink/pom.xml | 8 ++ .../flink/sql/sink/rdb/JDBCOptions.java | 16 +++ .../rdb/format/AbstractJDBCOutputFormat.java | 4 +- .../rdb/format/JDBCUpsertOutputFormat.java | 16 +-- .../sink/rdb/resource/JdbcResourceCheck.java | 133 ------------------ .../sql/sink/rdb/table/RdbTableInfo.java | 25 +++- .../sqlserver/table/SqlserverSideParser.java | 2 + .../sqlserver/table/SqlserverSinkParser.java | 2 + .../sql/side/tidb/table/TidbSideParser.java | 2 + .../sql/sink/tidb/table/TidbSinkParser.java | 2 + 47 files changed, 588 insertions(+), 468 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/util/ThreadUtil.java create mode 100644 rdb/rdb-core/pom.xml create mode 100644 rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcCheckKeys.java create mode 100644 rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java rename core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java => rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java (52%) delete mode 100644 rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/resource/JdbcResourceCheck.java delete mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/resource/JdbcResourceCheck.java diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java index c9a0c447b..d770356a8 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java @@ -18,11 +18,11 @@ package com.dtstack.flink.sql.side.clickhouse; +import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; -import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +45,7 @@ public ClickhouseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "ru.yandex.clickhouse.ClickHouseDriver"); AbstractTableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); clickhouseTableInfo.setType(CURR_TYPE); return clickhouseTableInfo; diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java index 5b0f2598f..75478061e 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.sink.clickhouse.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; import com.dtstack.flink.sql.table.AbstractTableInfo; import ru.yandex.clickhouse.domain.ClickHouseDataType; @@ -31,6 +32,7 @@ public class ClickhouseSinkParser extends RdbSinkParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "ru.yandex.clickhouse.ClickHouseDriver"); AbstractTableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); clickhouseTableInfo.setType(CURR_TYPE); return clickhouseTableInfo; diff --git a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java index 4281a9f09..daef7b600 100644 --- a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java @@ -28,6 +28,7 @@ import java.lang.reflect.Method; import java.net.URL; import java.net.URLClassLoader; +import java.sql.DriverManager; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -43,8 +44,28 @@ public class ClassLoaderManager { private static final Logger LOG = LoggerFactory.getLogger(ClassLoaderManager.class); + private static final Map pluginClassLoader = new ConcurrentHashMap<>(); + private static final Object LOCK = new Object(); - private static Map pluginClassLoader = new ConcurrentHashMap<>(); + public static void forName(String clazz, ClassLoader classLoader) { + synchronized (LOCK) { + try { + Class.forName(clazz, true, classLoader); + DriverManager.setLoginTimeout(10); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + public synchronized static void forName(String clazz) { + try { + Class driverClass = Class.forName(clazz); + driverClass.newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } public static R newInstance(String pluginJarPath, ClassLoaderSupplier supplier) throws Exception { ClassLoader classLoader = retrieveClassLoad(pluginJarPath); @@ -97,11 +118,10 @@ public static List getClassPath() { } - public static URLClassLoader loadExtraJar(List jarUrlList, URLClassLoader classLoader) - throws IllegalAccessException, InvocationTargetException { - for(URL url : jarUrlList){ - if(url.toString().endsWith(".jar")){ + throws IllegalAccessException, InvocationTargetException { + for (URL url : jarUrlList) { + if (url.toString().endsWith(".jar")) { urlClassLoaderAddUrl(classLoader, url); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java b/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java index 5fee88fd3..0a5ba3d21 100644 --- a/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java +++ b/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java @@ -18,23 +18,22 @@ package com.dtstack.flink.sql.resource; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import java.util.Map; +import java.util.Properties; /** * @author: chuixue * @create: 2020-12-08 17:21 * @description:资源检测 **/ -public abstract class ResourceCheck { - /** - * 数据库连接的最大重试次数 - */ - protected static final int MAX_RETRY_TIMES = 3; +public interface ResourceCheck { + String SINK_STR = "sink"; + String SIDE_STR = "side"; /** * 资源可用性检测 * - * @param abstractTableInfo 连接信息 + * @param checkProperties 校验资源可用性的参数配置 */ - public abstract void checkResourceStatus(AbstractTableInfo abstractTableInfo); + void checkResourceStatus(Map checkProperties); } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java index f8fa9c3ef..9494d2a5e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java @@ -24,47 +24,45 @@ import java.io.Serializable; import java.util.List; import java.util.Map; +import java.util.Properties; /** * Reason: * Date: 2018/6/22 * Company: www.dtstack.com + * * @author xuchao */ public abstract class AbstractTableInfo implements Serializable { public static final String PARALLELISM_KEY = "parallelism"; - + private final List fieldList = Lists.newArrayList(); + private final List fieldTypeList = Lists.newArrayList(); + private final List fieldClassList = Lists.newArrayList(); + private final List fieldExtraInfoList = Lists.newArrayList(); private String name; - private String type; - private String[] fields; - private String[] fieldTypes; - private Class[] fieldClasses; - - private final List fieldList = Lists.newArrayList(); - - /**key:别名, value: realField */ + /** + * key:别名, value: realField + */ private Map physicalFields = Maps.newLinkedHashMap(); - - private final List fieldTypeList = Lists.newArrayList(); - - private final List fieldClassList = Lists.newArrayList(); - - private final List fieldExtraInfoList = Lists.newArrayList(); - private List primaryKeys; private Integer parallelism = -1; + private Map checkProperties; public String[] getFieldTypes() { return fieldTypes; } + public void setFieldTypes(String[] fieldTypes) { + this.fieldTypes = fieldTypes; + } + public abstract boolean check(); public String getType() { @@ -79,10 +77,18 @@ public String[] getFields() { return fields; } + public void setFields(String[] fields) { + this.fields = fields; + } + public Class[] getFieldClasses() { return fieldClasses; } + public void setFieldClasses(Class[] fieldClasses) { + this.fieldClasses = fieldClasses; + } + public List getPrimaryKeys() { return primaryKeys; } @@ -104,18 +110,18 @@ public Integer getParallelism() { } public void setParallelism(Integer parallelism) { - if(parallelism == null){ + if (parallelism == null) { return; } - if(parallelism <= 0){ + if (parallelism <= 0) { throw new RuntimeException("Abnormal parameter settings: parallelism > 0"); } this.parallelism = parallelism; } - public void addField(String fieldName){ + public void addField(String fieldName) { if (fieldList.contains(fieldName)) { throw new RuntimeException("redundancy field name " + fieldName + " in table " + getName()); } @@ -123,30 +129,18 @@ public void addField(String fieldName){ fieldList.add(fieldName); } - public void addPhysicalMappings(String aliasName, String physicalFieldName){ + public void addPhysicalMappings(String aliasName, String physicalFieldName) { physicalFields.put(aliasName, physicalFieldName); } - public void addFieldClass(Class fieldClass){ + public void addFieldClass(Class fieldClass) { fieldClassList.add(fieldClass); } - public void addFieldType(String fieldType){ + public void addFieldType(String fieldType) { fieldTypeList.add(fieldType); } - public void setFields(String[] fields) { - this.fields = fields; - } - - public void setFieldTypes(String[] fieldTypes) { - this.fieldTypes = fieldTypes; - } - - public void setFieldClasses(Class[] fieldClasses) { - this.fieldClasses = fieldClasses; - } - public void addFieldExtraInfo(FieldExtraInfo extraInfo) { fieldExtraInfoList.add(extraInfo); } @@ -167,15 +161,27 @@ public Map getPhysicalFields() { return physicalFields; } + public void setPhysicalFields(Map physicalFields) { + this.physicalFields = physicalFields; + } + public List getFieldExtraInfoList() { return fieldExtraInfoList; } - public void setPhysicalFields(Map physicalFields) { - this.physicalFields = physicalFields; + public Map getCheckProperties() { + return checkProperties; + } + + public void setCheckProperties() { + this.checkProperties = buildCheckProperties(); + } + + public Map buildCheckProperties() { + return Maps.newHashMap(); } - public void finish(){ + public void finish() { this.fields = fieldList.toArray(new String[0]); this.fieldClasses = fieldClassList.toArray(new Class[0]); this.fieldTypes = fieldTypeList.toArray(new String[0]); @@ -183,7 +189,7 @@ public void finish(){ /** * field extra info，used to store `not null` `default 0`...， - * + *

* now, only support not null */ public static class FieldExtraInfo implements Serializable { @@ -193,7 +199,7 @@ public static class FieldExtraInfo implements Serializable { */ boolean notNull = false; /** - * field length,eg.char(4) + * field length,eg.char(4) */ int length; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ThreadUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ThreadUtil.java new file mode 100644 index 000000000..5e73f7c3b --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/util/ThreadUtil.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.util; + +import java.util.concurrent.TimeUnit; + +/** + * @author tiezhu + * Date 2020-12-25 + * Company dtstack + */ +public class ThreadUtil { + public static final Long DEFAULT_SLEEP_TIME = 10L; + public static void sleepMilliseconds(long timeout) { + try { + TimeUnit.MILLISECONDS.sleep(timeout); + } catch (InterruptedException ie) { + throw new RuntimeException(ie); + } + } + + public static void sleepMinutes(long timeout) { + try { + TimeUnit.MINUTES.sleep(timeout); + } catch (InterruptedException ie) { + throw new RuntimeException(ie); + } + } + + public static void sleepMicroseconds(long timeout) { + try { + TimeUnit.MICROSECONDS.sleep(timeout); + } catch (InterruptedException ie) { + throw new RuntimeException(ie); + } + } +} diff --git a/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java b/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java index 96be4ec15..bc12e9a38 100644 --- a/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java +++ b/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.db2.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -37,6 +38,7 @@ public class Db2SideParser extends RdbSideParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "com.ibm.db2.jcc.DB2Driver"); AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURR_TYPE); diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java index 3e823ab20..8a9ad62ab 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java @@ -1,5 +1,6 @@ package com.dtstack.flink.sql.sink.db.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -11,6 +12,7 @@ public class DbSinkParser extends RdbSinkParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "com.ibm.db2.jcc.DB2Driver"); AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURR_TYPE); return tableInfo; diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java index 6c5021068..2a99eef99 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java @@ -18,21 +18,19 @@ package com.dtstack.flink.sql.side.impala; +import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; -import com.dtstack.flink.sql.util.JDBCUtils; import com.dtstack.flink.sql.util.KrbUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; import java.sql.Connection; import java.sql.DriverManager; @@ -67,7 +65,7 @@ public Connection getConn(String dbUrl, String userName, String password) { try { Connection connection; String url = getUrl(); - JDBCUtils.forName(IMPALA_DRIVER, getClass().getClassLoader()); + ClassLoaderManager.forName(IMPALA_DRIVER, getClass().getClassLoader()); // Kerberos if (impalaSideTableInfo.getAuthMech() == 1) { String keyTabFilePath = impalaSideTableInfo.getKeyTabFilePath(); diff --git a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java index 2f1d73be7..4a35162d5 100644 --- a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java +++ b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java @@ -29,7 +29,6 @@ import java.math.BigDecimal; import java.sql.Timestamp; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -61,6 +60,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map fieldValues = new HashMap(); + Map fieldValues; try { ObjectMapper objectMapper = new ObjectMapper(); fieldValues = objectMapper.readValue(partitionfieldValuesStr, Map.class); diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index 0e8288eca..e0b9eafce 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -18,11 +18,11 @@ package com.dtstack.flink.sql.sink.impala; +import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.sink.rdb.JDBCTypeConvertUtils; import com.dtstack.flink.sql.table.AbstractTableInfo; -import com.dtstack.flink.sql.util.JDBCUtils; import com.dtstack.flink.sql.util.KrbUtils; import com.google.common.collect.Maps; import org.apache.commons.collections.CollectionUtils; @@ -34,7 +34,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.rmi.RemoteException; import java.security.PrivilegedExceptionAction; import java.sql.Connection; import java.sql.DriverManager; @@ -52,7 +51,6 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -92,27 +90,14 @@ public class ImpalaOutputFormat extends AbstractDtRichOutputFormat staticPartitionFields = new ArrayList<>(); + public List fieldNames; + public List fieldTypes; + public List fieldExtraInfoList; protected transient Connection connection; protected transient Statement statement; protected transient PreparedStatement updateStatement; - - private transient volatile boolean closed = false; - private int batchCount = 0; - - // |------------------------------------------------| - // | partitionCondition |Array of valueCondition| - // |------------------------------------------------| - // | ptOne, ptTwo, ptThree | [(v1, v2, v3, v4, v5)]| DP - // |------------------------------------------------| - // | ptOne = v1, ptTwo = v2 | [(v3, v4, v5)] | SP - // |------------------------------------------------| - // | ptOne, ptTwo = v2 | [(v1, v3, v4, v5)] | DP and SP - // |------------------------------------------------| - // | noPartition | [(v1, v2, v3, v4, v5)]| kudu or disablePartition - // |------------------------------------------------| - private transient Map> rowDataMap; - protected String keytabPath; protected String krb5confPath; protected String principal; @@ -129,13 +114,20 @@ public class ImpalaOutputFormat extends AbstractDtRichOutputFormat fieldNames; - public List fieldTypes; - public List fieldExtraInfoList; - - // partition field of static partition which matched by ${field} - private final List staticPartitionFields = new ArrayList<>(); - + private transient volatile boolean closed = false; + private int batchCount = 0; + // |------------------------------------------------| + // | partitionCondition |Array of valueCondition| + // |------------------------------------------------| + // | ptOne, ptTwo, ptThree | [(v1, v2, v3, v4, v5)]| DP + // |------------------------------------------------| + // | ptOne = v1, ptTwo = v2 | [(v3, v4, v5)] | SP + // |------------------------------------------------| + // | ptOne, ptTwo = v2 | [(v1, v3, v4, v5)] | DP and SP + // |------------------------------------------------| + // | noPartition | [(v1, v2, v3, v4, v5)]| kudu or disablePartition + // |------------------------------------------------| + private transient Map> rowDataMap; // valueFieldsName -> 重组之后的fieldNames，为了重组row data字段值对应 // 需要对partition字段做特殊处理，比如原来的字段顺序为(age, name, id)，但是因为partition，写入的SQL为 // INSERT INTO tableName(name, id) PARTITION(age) VALUES(?, ?, ?) @@ -147,6 +139,10 @@ public class ImpalaOutputFormat extends AbstractDtRichOutputFormat scheduledFuture; + public static Builder getImpalaBuilder() { + return new Builder(); + } + @Override public void configure(Configuration parameters) { } @@ -229,7 +225,7 @@ private void openConnect() throws IOException { * get jdbc connection */ private void openJdbc() { - JDBCUtils.forName(DRIVER_NAME, getClass().getClassLoader()); + ClassLoaderManager.forName(DRIVER_NAME, getClass().getClassLoader()); try { connection = DriverManager.getConnection(dbUrl, userName, password); statement = connection.createStatement(); @@ -627,10 +623,6 @@ private String quoteIdentifier(String identifier) { return "`" + identifier + "`"; } - public static Builder getImpalaBuilder() { - return new Builder(); - } - public static class Builder { private final ImpalaOutputFormat format = new ImpalaOutputFormat(); diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java index 003b5a6ab..e7e28d1bd 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java @@ -101,6 +101,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "com.kingbase8.Driver"); AbstractTableInfo kingbaseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); kingbaseTableInfo.setType(CURRENT_TYPE); return kingbaseTableInfo; diff --git a/kingbase/kingbase-sink/src/main/java/com/dtstack/flink/sql/sink/kingbase/table/KingbaseSinkParser.java b/kingbase/kingbase-sink/src/main/java/com/dtstack/flink/sql/sink/kingbase/table/KingbaseSinkParser.java index 1a82d49ac..a22d09004 100644 --- a/kingbase/kingbase-sink/src/main/java/com/dtstack/flink/sql/sink/kingbase/table/KingbaseSinkParser.java +++ b/kingbase/kingbase-sink/src/main/java/com/dtstack/flink/sql/sink/kingbase/table/KingbaseSinkParser.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.kingbase.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -34,6 +35,7 @@ public class KingbaseSinkParser extends RdbSinkParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "com.kingbase8.Driver"); AbstractTableInfo kingbaseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); kingbaseTableInfo.setType(CURRENT_TYPE); return kingbaseTableInfo; diff --git a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java index 6f0ad0c32..4488ca7d7 100644 --- a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java +++ b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java @@ -19,14 +19,13 @@ package com.dtstack.flink.sql.side.mysql.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; -import static com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo.DRIVER_NAME; - /** * Reason: * Date: 2018/7/25 @@ -41,7 +40,7 @@ public class MysqlSideParser extends RdbSideParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - props.put(DRIVER_NAME, "com.mysql.jdbc.Driver"); + props.put(JdbcCheckKeys.DRIVER_NAME, "com.mysql.jdbc.Driver"); RdbSideTableInfo mysqlTableInfo = (RdbSideTableInfo) super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java index e6daa24b0..8359fa0ff 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java @@ -19,13 +19,12 @@ package com.dtstack.flink.sql.sink.mysql.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; -import static com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo.DRIVER_NAME; - /** * Reason: * Date: 2018/7/4 @@ -39,7 +38,7 @@ public class MysqlSinkParser extends RdbSinkParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - props.put(DRIVER_NAME, "com.mysql.jdbc.Driver"); + props.put(JdbcCheckKeys.DRIVER_NAME, "com.mysql.jdbc.Driver"); AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; diff --git a/oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java b/oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java index a5c6324a9..aad37ae2f 100644 --- a/oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java +++ b/oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java @@ -17,6 +17,7 @@ */ package com.dtstack.flink.sql.side.oceanbase.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -32,6 +33,7 @@ public class OceanbaseSideParser extends RdbSideParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "com.mysql.jdbc.Driver"); AbstractTableInfo oceanbaseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); oceanbaseTableInfo.setType(CURRENT_TYPE); return oceanbaseTableInfo; diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/table/OceanbaseSinkParser.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/table/OceanbaseSinkParser.java index af1a41d6b..96b783a6f 100644 --- a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/table/OceanbaseSinkParser.java +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/oceanbase/table/OceanbaseSinkParser.java @@ -17,6 +17,7 @@ */ package com.dtstack.flink.sql.sink.oceanbase.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -32,6 +33,7 @@ public class OceanbaseSinkParser extends RdbSinkParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "com.mysql.jdbc.Driver"); AbstractTableInfo oceanbaseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); diff --git a/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java b/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java index 897b77d30..59fdd795d 100644 --- a/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java +++ b/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java @@ -17,6 +17,7 @@ */ package com.dtstack.flink.sql.side.oracle.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -28,6 +29,7 @@ public class OracleSideParser extends RdbSideParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "oracle.jdbc.driver.OracleDriver"); AbstractTableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); oracleTableInfo.setType(CURR_TYPE); return oracleTableInfo; diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java index 1732c8cc2..2eeee3fb2 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java @@ -17,6 +17,7 @@ */ package com.dtstack.flink.sql.sink.oracle.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -34,6 +35,7 @@ public class OracleSinkParser extends RdbSinkParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "oracle.jdbc.driver.OracleDriver"); AbstractTableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); oracleTableInfo.setType(CURR_TYPE); return oracleTableInfo; diff --git a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java index 71aad7cbb..4e2683b14 100644 --- a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java +++ b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java @@ -18,6 +18,7 @@ */ package com.dtstack.flink.sql.side.polardb.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -26,6 +27,7 @@ /** * Date: 2019/12/20 * Company: www.dtstack.com + * * @author yinxi */ public class PolardbSideParser extends RdbSideParser { @@ -33,6 +35,7 @@ public class PolardbSideParser extends RdbSideParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "com.mysql.cj.jdbc.Driver"); AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java index b4f02665e..80d62f3d0 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java @@ -17,6 +17,7 @@ */ package com.dtstack.flink.sql.sink.polardb.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -25,6 +26,7 @@ /** * Date: 2019/12/20 * Company: www.dtstack.com + * * @author yinxi */ public class PolardbSinkParser extends RdbSinkParser { @@ -32,6 +34,7 @@ public class PolardbSinkParser extends RdbSinkParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "com.mysql.cj.jdbc.Driver"); AbstractTableInfo polardbTableInfo = super.getTableInfo(tableName, fieldsInfo, props); polardbTableInfo.setType(CURR_TYPE); return polardbTableInfo; diff --git a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java b/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java index be0c5ea8b..fe1afb578 100644 --- a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java +++ b/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.postgresql.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -38,6 +39,7 @@ public class PostgresqlSideParser extends RdbSideParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "org.postgresql.Driver"); AbstractTableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); pgTableInfo.setType(CURR_TYPE); return pgTableInfo; diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java index e94cfc8a0..9e9db93ee 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.sink.postgresql.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -36,6 +37,7 @@ public class PostgresqlSinkParser extends RdbSinkParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "org.postgresql.Driver"); AbstractTableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); pgTableInfo.setType(CURR_TYPE); return pgTableInfo; diff --git a/rdb/pom.xml b/rdb/pom.xml index 5cd5d4187..3756d91d3 100644 --- a/rdb/pom.xml +++ b/rdb/pom.xml @@ -15,6 +15,7 @@ rdb-side rdb-sink + rdb-core pom diff --git a/rdb/rdb-core/pom.xml b/rdb/rdb-core/pom.xml new file mode 100644 index 000000000..32ad8b6c0 --- /dev/null +++ b/rdb/rdb-core/pom.xml @@ -0,0 +1,20 @@ + + + + sql.rdb + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + rdb-core + sql.core.rdb + + + 8 + 8 + + + \ No newline at end of file diff --git a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcCheckKeys.java b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcCheckKeys.java new file mode 100644 index 000000000..3b1dfcb27 --- /dev/null +++ b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcCheckKeys.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.core.rdb; + +/** + * @author tiezhu + * Date 2020-12-25 + * Company dtstack + */ +public class JdbcCheckKeys { + public static final String DRIVER_NAME = "driverName"; + public static final String URL_KEY = "url"; + public static final String USER_NAME_KEY = "userName"; + public static final String PASSWORD_KEY = "password"; + public static final String TABLE_TYPE_KEY = "tableType"; + public static final String SCHEMA_KEY = "schema"; + public static final String TABLE_NAME_KEY = "tableName"; + // create 语句中的name + public static final String OPERATION_NAME_KEY = "operationName"; + // 用来检查update、replace等操作的column + public static final String COLUMN_KEY = "column"; + public static final String TABLE_INFO_KEY = "tableInfo"; +} diff --git a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java new file mode 100644 index 000000000..0c7f88be8 --- /dev/null +++ b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.core.rdb; + +import com.dtstack.flink.sql.core.rdb.util.JdbcConnectUtil; +import com.dtstack.flink.sql.resource.ResourceCheck; +import org.apache.flink.runtime.execution.SuppressRestartsException; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; + +/** + * @author: chuixue + * @create: 2020-12-08 17:24 + * @description:jdbc资源检测 + **/ +public class JdbcResourceCheck implements ResourceCheck { + private static final String DELETE_STR = "delete"; + private static final String SELECT_STR = "select"; + private static final String INSERT_STR = "insert"; + private static final String UPDATE_STR = "update"; + private static final String REPLACE_STR = "replace"; + + private static final String CHECK_SELECT_SQL = "select 1 from %s where 1=1;"; + private static final String CHECK_DELETE_SQL = "delete from %s where 1 = 3;"; + private static final String CHECK_INSERT_SQL = "insert into %s (select * from %s where 1 = 2);"; + + private static final Map PRIVILEGE_SQL_MAP = new HashMap<>(); + private static final JdbcResourceCheck Instance = new JdbcResourceCheck(); + + static { + PRIVILEGE_SQL_MAP.put(SELECT_STR, CHECK_SELECT_SQL); + PRIVILEGE_SQL_MAP.put(DELETE_STR, CHECK_DELETE_SQL); + PRIVILEGE_SQL_MAP.put(INSERT_STR, CHECK_INSERT_SQL); + } + + private JdbcResourceCheck() { + } + + public static JdbcResourceCheck getInstance() { + return Instance; + } + + @Override + public void checkResourceStatus(Map checkProperties) { + List privilegeList = new ArrayList<>(); + if (checkProperties.get(JdbcCheckKeys.TABLE_TYPE_KEY).equalsIgnoreCase(SIDE_STR)) { + privilegeList.add(SELECT_STR); + } + if (checkProperties.get(JdbcCheckKeys.TABLE_TYPE_KEY).equalsIgnoreCase(SINK_STR)) { + privilegeList.add(INSERT_STR); + privilegeList.add(DELETE_STR); + } + checkPrivilege( + checkProperties.get(JdbcCheckKeys.DRIVER_NAME) + , checkProperties.get(JdbcCheckKeys.URL_KEY) + , checkProperties.get(JdbcCheckKeys.USER_NAME_KEY) + , checkProperties.get(JdbcCheckKeys.PASSWORD_KEY) + , checkProperties.get(JdbcCheckKeys.TABLE_NAME_KEY) + , checkProperties.get(JdbcCheckKeys.SCHEMA_KEY) + , privilegeList + ); + } + + public void checkPrivilege( + String driverName + , String url + , String userName + , String password + , String tableName + , String schema + , List privilegeList) { + Connection connection = + JdbcConnectUtil.getConnectWithRetry(driverName, url, userName, password); + Statement statement = null; + String tableInfo = Objects.isNull(schema) ? tableName : schema + "." + tableName; + String privilege = null; + try { + statement = connection.createStatement(); + for (String s : privilegeList) { + privilege = s; + statement.executeQuery( + String.format(PRIVILEGE_SQL_MAP.get(privilege.toLowerCase()), tableInfo)); + } + } catch (SQLException sqlException) { + if (sqlException.getMessage().contains("command denied")) { + throw new SuppressRestartsException(new Throwable( + String.format("user [%s] don't have [%s] privilege of table [%s]", userName, privilege, tableInfo))); + } + + throw new SuppressRestartsException(new Throwable(sqlException.getMessage())); + } finally { + JdbcConnectUtil.closeConnectionResource(null, statement, connection, false); + } + } +} diff --git a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java similarity index 52% rename from core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java rename to rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java index 1d87a1c75..2d4a49d3e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java +++ b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java @@ -16,9 +16,11 @@ * limitations under the License. */ +package com.dtstack.flink.sql.core.rdb.util; -package com.dtstack.flink.sql.util; - +import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.dtstack.flink.sql.util.ThreadUtil; +import com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,31 +31,16 @@ import java.sql.Statement; import java.util.Objects; -public class JDBCUtils { - private static final Logger LOG = LoggerFactory.getLogger(JDBCUtils.class); - - private static final Object LOCK = new Object(); - - public static void forName(String clazz, ClassLoader classLoader) { - synchronized (LOCK) { - try { - Class.forName(clazz, true, classLoader); - DriverManager.setLoginTimeout(10); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - - - public synchronized static void forName(String clazz) { - try { - Class driverClass = Class.forName(clazz); - driverClass.newInstance(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } +/** + * @author tiezhu + * Date 2020-12-25 + * Company dtstack + */ +public class JdbcConnectUtil { + private static final int DEFAULT_RETRY_NUM = 5; + private static final long DEFAULT_RETRY_TIME_WAIT = 5L; + private static final int DEFAULT_VALID_NUM = 10; + private static final Logger LOG = LoggerFactory.getLogger(JdbcConnectUtil.class); /** * 关闭连接资源 @@ -61,9 +48,13 @@ public synchronized static void forName(String clazz) { * @param rs ResultSet * @param stmt Statement * @param conn Connection - * @param commit + * @param commit 手动提交事务 */ - public static void closeConnectionResource(ResultSet rs, Statement stmt, Connection conn, boolean commit) { + public static void closeConnectionResource( + ResultSet rs + , Statement stmt + , Connection conn + , boolean commit) { if (Objects.nonNull(rs)) { try { rs.close(); @@ -102,7 +93,7 @@ public static void closeConnectionResource(ResultSet rs, Statement stmt, Connect */ public static void commit(Connection conn) { try { - if (!conn.isClosed() && !conn.getAutoCommit()) { + if (!conn.isClosed() && conn.isValid(DEFAULT_VALID_NUM) && !conn.getAutoCommit()) { conn.commit(); } } catch (SQLException e) { @@ -117,7 +108,7 @@ public static void commit(Connection conn) { */ public static void rollBack(Connection conn) { try { - if (!conn.isClosed() && !conn.getAutoCommit()) { + if (!conn.isClosed() && conn.isValid(DEFAULT_VALID_NUM) && !conn.getAutoCommit()) { conn.rollback(); } } catch (SQLException e) { @@ -125,4 +116,41 @@ public static void rollBack(Connection conn) { } } + /** + * get connect from datasource and retry when failed. + * + * @param driverName driver name for rdb datasource + * @param url connect url + * @param userName connect user name + * @param password password for user name + * @return a valid connection + */ + public static Connection getConnectWithRetry( + String driverName + , String url + , String userName + , String password) { + String errorMessage = "Get connect failed with properties: \nurl: " + url + + (Objects.isNull(userName) ? "" : "\nuserName: " + userName); + + ClassLoaderManager.forName(driverName); + Preconditions.checkNotNull(url, "url can't be null!"); + + Connection connection = null; + for (int i = 0; i < DEFAULT_RETRY_NUM; i++) { + try { + connection = Objects.isNull(userName) ? + DriverManager.getConnection(url) : DriverManager.getConnection(url, userName, password); + // 校验connection是否可用 + connection.isValid(DEFAULT_VALID_NUM); + return connection; + } catch (Exception e) { + LOG.warn(errorMessage, e); + LOG.warn("Connect will retry after [{}] s......", DEFAULT_RETRY_TIME_WAIT); + ThreadUtil.sleepMilliseconds(DEFAULT_RETRY_TIME_WAIT); + closeConnectionResource(null, null, connection, false); + } + } + throw new IllegalArgumentException(errorMessage); + } } diff --git a/rdb/rdb-side/pom.xml b/rdb/rdb-side/pom.xml index c7350877f..3665aa189 100644 --- a/rdb/rdb-side/pom.xml +++ b/rdb/rdb-side/pom.xml @@ -35,6 +35,12 @@ ${vertx.version} + + com.dtstack.flink + sql.core.rdb + 1.0-SNAPSHOT + + diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index e782dec4b..0df36cf46 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -18,12 +18,12 @@ package com.dtstack.flink.sql.side.rdb.all; +import com.dtstack.flink.sql.core.rdb.JdbcResourceCheck; +import com.dtstack.flink.sql.core.rdb.util.JdbcConnectUtil; import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.rdb.resource.JdbcResourceCheck; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; -import com.dtstack.flink.sql.util.JDBCUtils; import com.dtstack.flink.sql.util.RowDataComplete; import com.dtstack.flink.sql.util.RowDataConvert; import com.google.common.collect.Lists; @@ -70,10 +70,8 @@ public abstract class AbstractRdbAllReqRow extends BaseAllReqRow { private static final int CONN_RETRY_NUM = 3; private static final int DEFAULT_FETCH_SIZE = 1000; - - private AtomicReference>>> cacheRef = new AtomicReference<>(); - private static volatile boolean resourceCheck = true; + private final AtomicReference>>> cacheRef = new AtomicReference<>(); public AbstractRdbAllReqRow(BaseSideInfo sideInfo) { super(sideInfo); @@ -81,14 +79,14 @@ public AbstractRdbAllReqRow(BaseSideInfo sideInfo) { @Override public void open(Configuration parameters) throws Exception { + super.open(parameters); + RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); synchronized (AbstractRdbAllReqRow.class) { if (resourceCheck) { resourceCheck = false; - JdbcResourceCheck.getInstance().checkResourceStatus(sideInfo.getSideTableInfo()); + JdbcResourceCheck.getInstance().checkResourceStatus(tableInfo.getCheckProperties()); } } - super.open(parameters); - RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); LOG.info("rdb dim table config info: {} ", tableInfo.toString()); } @@ -126,7 +124,7 @@ public void flatMap(Row value, Collector out) throws Exception { } String cacheKey = inputParams.stream() - .map(e -> String.valueOf(e)) + .map(String::valueOf) .collect(Collectors.joining("_")); List> cacheList = cacheRef.get().get(cacheKey); @@ -151,7 +149,7 @@ protected Object dealTimeAttributeType(Class entry, O boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(entry); if (obj instanceof LocalDateTime && isTimeIndicatorTypeInfo) { //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 - obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); + obj = ((Timestamp) obj).getTime() + (long) LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } return obj; } @@ -204,13 +202,13 @@ private void queryAndFillData(Map>> tmpCache, C String cacheKey = sideInfo.getEqualFieldList().stream() .map(oneRow::get) - .map(e -> String.valueOf(e)) + .map(String::valueOf) .collect(Collectors.joining("_")); tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()) .add(oneRow); } - JDBCUtils.closeConnectionResource(resultSet, statement, connection, false); + JdbcConnectUtil.closeConnectionResource(resultSet, statement, connection, false); } public int getFetchSize() { diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 3ec316c30..1982de1da 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -19,17 +19,18 @@ package com.dtstack.flink.sql.side.rdb.async; +import com.dtstack.flink.sql.core.rdb.JdbcResourceCheck; import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.cache.CacheObj; -import com.dtstack.flink.sql.side.rdb.resource.JdbcResourceCheck; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import com.dtstack.flink.sql.util.DateUtil; import com.dtstack.flink.sql.util.RowDataComplete; +import com.dtstack.flink.sql.util.ThreadUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import io.vertx.core.json.JsonArray; @@ -65,42 +66,34 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { - private static final long serialVersionUID = 2098635244857937720L; - - private static final Logger LOG = LoggerFactory.getLogger(RdbAsyncReqRow.class); - public final static int DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE = 1; - public final static int DEFAULT_VERTX_WORKER_POOL_SIZE = Runtime.getRuntime().availableProcessors() * 2; - public final static int DEFAULT_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; - public final static int MAX_DB_CONN_POOL_SIZE_LIMIT = 20; - public final static int DEFAULT_IDLE_CONNECTION_TEST_PEROID = 60; - public final static boolean DEFAULT_TEST_CONNECTION_ON_CHECKIN = true; - public final static String DT_PROVIDER_CLASS = "com.dtstack.flink.sql.side.rdb.provider.DTC3P0DataSourceProvider"; - public final static String PREFERRED_TEST_QUERY_SQL = "SELECT 1 FROM DUAL"; - - private transient SQLClient rdbSqlClient; - - private AtomicBoolean connectionStatus = new AtomicBoolean(true); - + private static final long serialVersionUID = 2098635244857937720L; + private static final Logger LOG = LoggerFactory.getLogger(RdbAsyncReqRow.class); + private final static int MAX_TASK_QUEUE_SIZE = 100000; private static volatile boolean resourceCheck = true; - + private transient SQLClient rdbSqlClient; + private final AtomicBoolean connectionStatus = new AtomicBoolean(true); private transient ThreadPoolExecutor executor; - private final static int MAX_TASK_QUEUE_SIZE = 100000; + public RdbAsyncReqRow(BaseSideInfo sideInfo) { + super(sideInfo); + init(sideInfo); + } @Override public void open(Configuration parameters) throws Exception { + RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); synchronized (RdbAsyncReqRow.class) { if (resourceCheck) { resourceCheck = false; - JdbcResourceCheck.getInstance().checkResourceStatus(sideInfo.getSideTableInfo()); + JdbcResourceCheck.getInstance().checkResourceStatus(rdbSideTableInfo.getCheckProperties()); } } super.open(parameters); @@ -108,11 +101,6 @@ public void open(Configuration parameters) throws Exception { new LinkedBlockingQueue<>(MAX_TASK_QUEUE_SIZE), new DTThreadFactory("rdbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy()); } - public RdbAsyncReqRow(BaseSideInfo sideInfo) { - super(sideInfo); - init(sideInfo); - } - protected void init(BaseSideInfo sideInfo) { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); int defaultAsyncPoolSize = Math.min(MAX_DB_CONN_POOL_SIZE_LIMIT, DEFAULT_DB_CONN_POOL_SIZE); @@ -121,7 +109,8 @@ protected void init(BaseSideInfo sideInfo) { } @Override - protected void preInvoke(Row input, ResultFuture resultFuture) { } + protected void preInvoke(Row input, ResultFuture resultFuture) { + } @Override public void handleAsyncInvoke(Map inputParams, Row input, ResultFuture resultFuture) throws Exception { @@ -137,7 +126,7 @@ public void handleAsyncInvoke(Map inputParams, Row input, Result executor.execute(() -> connectWithRetry(params, input, resultFuture, rdbSqlClient)); } - protected void asyncQueryData( Map inputParams, + protected void asyncQueryData(Map inputParams, Row input, ResultFuture resultFuture, SQLClient rdbSqlClient, @@ -145,21 +134,21 @@ protected void asyncQueryData( Map inputParams, AtomicBoolean finishFlag, CountDownLatch latch) { doAsyncQueryData(inputParams, - input, resultFuture, - rdbSqlClient, - failCounter, - finishFlag, - latch); + input, resultFuture, + rdbSqlClient, + failCounter, + finishFlag, + latch); } final protected void doAsyncQueryData( - Map inputParams, - Row input, - ResultFuture resultFuture, - SQLClient rdbSqlClient, - AtomicLong failCounter, - AtomicBoolean finishFlag, - CountDownLatch latch) { + Map inputParams, + Row input, + ResultFuture resultFuture, + SQLClient rdbSqlClient, + AtomicLong failCounter, + AtomicBoolean finishFlag, + CountDownLatch latch) { rdbSqlClient.getConnection(conn -> { try { if (conn.failed()) { @@ -192,12 +181,13 @@ private void connectWithRetry(Map inputParams, Row input, Result while (!finishFlag.get()) { try { CountDownLatch latch = new CountDownLatch(1); - asyncQueryData(inputParams, - input, resultFuture, - rdbSqlClient, - failCounter, - finishFlag, - latch); + asyncQueryData( + inputParams, + input, resultFuture, + rdbSqlClient, + failCounter, + finishFlag, + latch); try { latch.await(); } catch (InterruptedException e) { @@ -209,11 +199,7 @@ private void connectWithRetry(Map inputParams, Row input, Result connectionStatus.set(false); } if (!finishFlag.get()) { - try { - Thread.sleep(3000); - } catch (Exception e) { - LOG.error("", e); - } + ThreadUtil.sleepMilliseconds(ThreadUtil.DEFAULT_SLEEP_TIME); } } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/resource/JdbcResourceCheck.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/resource/JdbcResourceCheck.java deleted file mode 100644 index 357f31c42..000000000 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/resource/JdbcResourceCheck.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.side.rdb.resource; - -import com.dtstack.flink.sql.resource.ResourceCheck; -import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; -import com.dtstack.flink.sql.table.AbstractTableInfo; -import org.apache.flink.runtime.execution.SuppressRestartsException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.concurrent.TimeUnit; - -/** - * @author: chuixue - * @create: 2020-12-08 17:24 - * @description:jdbc资源检测 - **/ -public class JdbcResourceCheck extends ResourceCheck { - private static final Logger LOG = LoggerFactory.getLogger(JdbcResourceCheck.class); - private static final JdbcResourceCheck Instance = new JdbcResourceCheck(); - - private JdbcResourceCheck() { - } - - public static JdbcResourceCheck getInstance() { - return Instance; - } - - @Override - public void checkResourceStatus(AbstractTableInfo abstractTableInfo) { - RdbSideTableInfo rdbTableInfo = (RdbSideTableInfo) abstractTableInfo; - LOG.info("Start check if table [{}] resource is valid !!!",rdbTableInfo.getName()); - Connection connection = null; - forName(rdbTableInfo.getDriverName()); - for (int i = 0; i < MAX_RETRY_TIMES; i++) { - try { - if (rdbTableInfo.getUserName() == null) { - connection = DriverManager.getConnection(rdbTableInfo.getUrl()); - } else { - connection = DriverManager.getConnection(rdbTableInfo.getUrl(), rdbTableInfo.getUserName(), rdbTableInfo.getPassword()); - } - if (null != connection) { - // 这里不能抛出RuntimeException，用户可能只有某个table的读取权限，而没有读取meta表权限 - try { - // 如果没有异常，说明能有权限读取meta表，如果找不到，则抛出RuntimeException - if (!connection.getMetaData().getTables(null, rdbTableInfo.getSchema(), rdbTableInfo.getTableName(), null).next()) { - LOG.error("Table " + rdbTableInfo.getTableName() + " doesn't exist"); - throw new SuppressRestartsException(new Throwable("Table " + rdbTableInfo.getTableName() + " doesn't exist")); - } - } catch (SQLException e) { - LOG.error(e.getMessage()); - } - break; - } - } catch (SQLException sqe) { - if (i == MAX_RETRY_TIMES - 1) { - throw new SuppressRestartsException(new Throwable(sqe)); - } - LOG.error(sqe.getMessage() + " retry " + (i + 1) + " time"); - try { - TimeUnit.SECONDS.sleep(3); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } finally { - if (null != connection) { - try { - connection.close(); - } catch (SQLException e) { - throw new SuppressRestartsException(new Throwable(e)); - } - } - } - } - } - - /** - * @param clazz - */ - public synchronized static void forName(String clazz) { - try { - Class driverClass = Class.forName(clazz); - driverClass.newInstance(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } -} diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java index eeaad16fa..e18dba5f9 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java @@ -52,6 +52,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map buildCheckProperties() { + Map properties = Maps.newHashMap(); + properties.put(JdbcCheckKeys.DRIVER_NAME, getDriverName()); + properties.put(JdbcCheckKeys.URL_KEY, getUrl()); + properties.put(JdbcCheckKeys.USER_NAME_KEY, getUserName()); + properties.put(JdbcCheckKeys.PASSWORD_KEY, getPassword()); + properties.put(JdbcCheckKeys.SCHEMA_KEY, getSchema()); + properties.put(JdbcCheckKeys.TABLE_NAME_KEY, getTableName()); + properties.put(JdbcCheckKeys.OPERATION_NAME_KEY, getName()); + properties.put(JdbcCheckKeys.TABLE_TYPE_KEY, "side"); + return properties; + } } diff --git a/rdb/rdb-sink/pom.xml b/rdb/rdb-sink/pom.xml index 176615769..e0a0d1078 100644 --- a/rdb/rdb-sink/pom.xml +++ b/rdb/rdb-sink/pom.xml @@ -15,4 +15,12 @@ rdb-sink jar + + + com.dtstack.flink + sql.core.rdb + 1.0-SNAPSHOT + + + \ No newline at end of file diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java index f476bfadf..f89cba731 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java @@ -19,8 +19,11 @@ package com.dtstack.flink.sql.sink.rdb; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import com.google.common.collect.Maps; +import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -76,6 +79,19 @@ public String getSchema() { return schema; } + public Map buildCheckProperties() { + Map properties = Maps.newHashMap(); + properties.put(JdbcCheckKeys.DRIVER_NAME, getDriverName()); + properties.put(JdbcCheckKeys.URL_KEY, getDbUrl()); + properties.put(JdbcCheckKeys.USER_NAME_KEY, getUsername()); + properties.put(JdbcCheckKeys.PASSWORD_KEY, getPassword()); + properties.put(JdbcCheckKeys.SCHEMA_KEY, getSchema()); + properties.put(JdbcCheckKeys.TABLE_NAME_KEY, getTableName()); + properties.put(JdbcCheckKeys.OPERATION_NAME_KEY, "jdbcOutputFormat"); + properties.put(JdbcCheckKeys.TABLE_TYPE_KEY, "sink"); + return properties; + } + public static Builder builder() { return new Builder(); } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java index 4b5270a12..f8556c0ab 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.sink.rdb.format; +import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; -import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -64,7 +64,7 @@ public void configure(Configuration parameters) { } protected void establishConnection() throws SQLException, ClassNotFoundException, IOException { - JDBCUtils.forName(driverName, getClass().getClassLoader()); + ClassLoaderManager.forName(driverName, getClass().getClassLoader()); if (username == null) { connection = DriverManager.getConnection(dbURL); } else { diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index 55453e939..55d9a0ca7 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -19,14 +19,14 @@ package com.dtstack.flink.sql.sink.rdb.format; +import com.dtstack.flink.sql.core.rdb.JdbcResourceCheck; import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; -import com.dtstack.flink.sql.sink.rdb.resource.JdbcResourceCheck; +import com.dtstack.flink.sql.sink.rdb.writer.AbstractUpsertWriter; import com.dtstack.flink.sql.sink.rdb.writer.AppendOnlyWriter; import com.dtstack.flink.sql.sink.rdb.writer.JDBCWriter; -import com.dtstack.flink.sql.sink.rdb.writer.AbstractUpsertWriter; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; @@ -36,6 +36,7 @@ import java.io.IOException; import java.sql.SQLException; import java.util.List; +import java.util.Map; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -65,6 +66,8 @@ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat checkProperties; + private final int flushMaxSize; private final long flushIntervalMills; private final boolean allReplace; @@ -104,6 +107,7 @@ public JDBCUpsertOutputFormat( this.allReplace = allReplace; this.updateMode = updateMode; this.jdbcWriter = jdbcWriter; + this.checkProperties = options.buildCheckProperties(); } /** @@ -118,13 +122,7 @@ public void open(int taskNumber, int numTasks) throws IOException { synchronized (JDBCUpsertOutputFormat.class) { if (resourceCheck) { resourceCheck = false; - JdbcResourceCheck.getInstance().checkResourceStatus(name - , driverName - , username - , dbURL - , password - , schema - , tableName); + JdbcResourceCheck.getInstance().checkResourceStatus(this.checkProperties); } } openJdbc(); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/resource/JdbcResourceCheck.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/resource/JdbcResourceCheck.java deleted file mode 100644 index d7a3fe01e..000000000 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/resource/JdbcResourceCheck.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.rdb.resource; - -import com.dtstack.flink.sql.resource.ResourceCheck; -import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; -import com.dtstack.flink.sql.table.AbstractTableInfo; -import org.apache.flink.runtime.execution.SuppressRestartsException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.concurrent.TimeUnit; - -/** - * @author: chuixue - * @create: 2020-12-08 17:24 - * @description:jdbc资源检测 - **/ -public class JdbcResourceCheck extends ResourceCheck { - private static final Logger LOG = LoggerFactory.getLogger(JdbcResourceCheck.class); - private static final JdbcResourceCheck INSTANCE = new JdbcResourceCheck(); - - private JdbcResourceCheck() { - } - - public static JdbcResourceCheck getInstance() { - return INSTANCE; - } - - @Override - public void checkResourceStatus(AbstractTableInfo abstractTableInfo) { - RdbTableInfo rdbTableInfo = (RdbTableInfo) abstractTableInfo; - checkResourceStatus(rdbTableInfo.getName() - , rdbTableInfo.getDriverName() - , rdbTableInfo.getUserName() - , rdbTableInfo.getUrl() - , rdbTableInfo.getPassword() - , rdbTableInfo.getSchema() - , rdbTableInfo.getTableName()); - } - - /** - * @param name flinksql中的表名 - * @param driverName 驱动 - * @param username 用户名 - * @param url 地址 - * @param passWord 密码 - * @param schema schema - * @param tableName 数据库中的表名 - */ - public void checkResourceStatus(String name - , String driverName - , String username - , String url - , String passWord - , String schema - , String tableName) { - LOG.info("Start check if table [{}] resource is valid !!!", name); - Connection connection = null; - forName(driverName); - for (int i = 0; i < MAX_RETRY_TIMES; i++) { - try { - if (username == null) { - connection = DriverManager.getConnection(url); - } else { - connection = DriverManager.getConnection(url, username, passWord); - } - if (null != connection) { - // 这里不能抛出RuntimeException，用户可能只有某个table的读取权限，而没有读取meta表权限 - try { - // 如果没有异常，说明能有权限读取meta表，如果找不到，则抛出RuntimeException - if (!connection.getMetaData().getTables(null, schema, tableName, null).next()) { - LOG.error("Table " + tableName + " doesn't exist"); - throw new SuppressRestartsException(new Throwable("Table " + tableName + " doesn't exist")); - } - } catch (SQLException e) { - LOG.error(e.getMessage()); - } - break; - } - } catch (SQLException sqe) { - if (i == MAX_RETRY_TIMES - 1) { - throw new SuppressRestartsException(new Throwable(sqe)); - } - LOG.error(sqe.getMessage() + " retry " + (i + 1) + " time"); - try { - TimeUnit.SECONDS.sleep(3); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } finally { - if (null != connection) { - try { - connection.close(); - } catch (SQLException e) { - throw new SuppressRestartsException(new Throwable(e)); - } - } - } - } - } - - /** - * @param clazz - */ - public synchronized static void forName(String clazz) { - try { - Class driverClass = Class.forName(clazz); - driverClass.newInstance(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } -} diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java index 9afeab483..10c2742ad 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java @@ -17,12 +17,17 @@ */ package com.dtstack.flink.sql.sink.rdb.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; +import com.dtstack.flink.sql.core.rdb.JdbcResourceCheck; import com.dtstack.flink.sql.enums.EUpdateMode; -import com.dtstack.flink.sql.sink.rdb.resource.JdbcResourceCheck; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; +import java.util.Map; +import java.util.Properties; + /** * Reason: * Date: 2018/11/27 @@ -205,7 +210,7 @@ public boolean check() { // 是否在client端快速检测表资源是否可用,这样在client能访问资源的情况下快速失败,不用提交到集群检测 if (getFastCheck()) { - JdbcResourceCheck.getInstance().checkResourceStatus(this); + JdbcResourceCheck.getInstance().checkResourceStatus(getCheckProperties()); } return true; } @@ -215,4 +220,20 @@ public String getType() { // return super.getType().toLowerCase() + TARGET_SUFFIX; return super.getType().toLowerCase(); } + + @Override + public Map buildCheckProperties() { + Map properties = Maps.newHashMap(); + + properties.put(JdbcCheckKeys.DRIVER_NAME, getDriverName()); + properties.put(JdbcCheckKeys.URL_KEY, getUrl()); + properties.put(JdbcCheckKeys.USER_NAME_KEY, getUserName()); + properties.put(JdbcCheckKeys.PASSWORD_KEY, getPassword()); + properties.put(JdbcCheckKeys.SCHEMA_KEY, getSchema()); + properties.put(JdbcCheckKeys.TABLE_NAME_KEY, getTableName()); + properties.put(JdbcCheckKeys.OPERATION_NAME_KEY, getName()); + properties.put(JdbcCheckKeys.TABLE_TYPE_KEY, "sink"); + + return properties; + } } diff --git a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java index e06c13898..7b87aa09b 100644 --- a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java +++ b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java @@ -17,6 +17,7 @@ */ package com.dtstack.flink.sql.side.sqlserver.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -31,6 +32,7 @@ public class SqlserverSideParser extends RdbSideParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "net.sourceforge.jtds.jdbc.Driver"); AbstractTableInfo sqlServerTableInfo = super.getTableInfo(tableName, fieldsInfo, props); sqlServerTableInfo.setType(CURR_TYPE); return sqlServerTableInfo; diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java index 5300884bc..0e6522446 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java @@ -17,6 +17,7 @@ */ package com.dtstack.flink.sql.sink.sqlserver.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -34,6 +35,7 @@ public class SqlserverSinkParser extends RdbSinkParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "net.sourceforge.jtds.jdbc.Driver"); AbstractTableInfo sqlserverTableInfo = super.getTableInfo(tableName, fieldsInfo, props); sqlserverTableInfo.setType(CURR_TYPE); return sqlserverTableInfo; diff --git a/tidb/tidb-side/tidb-side-core/src/main/java/com/dtstack/flink/sql/side/tidb/table/TidbSideParser.java b/tidb/tidb-side/tidb-side-core/src/main/java/com/dtstack/flink/sql/side/tidb/table/TidbSideParser.java index ae8f59686..4d5088738 100644 --- a/tidb/tidb-side/tidb-side-core/src/main/java/com/dtstack/flink/sql/side/tidb/table/TidbSideParser.java +++ b/tidb/tidb-side/tidb-side-core/src/main/java/com/dtstack/flink/sql/side/tidb/table/TidbSideParser.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.tidb.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -34,6 +35,7 @@ public class TidbSideParser extends RdbSideParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "com.mysql.jdbc.Driver"); AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURRENT_TYPE); return tableInfo; diff --git a/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/table/TidbSinkParser.java b/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/table/TidbSinkParser.java index d00206cc2..4f7001e85 100644 --- a/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/table/TidbSinkParser.java +++ b/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/table/TidbSinkParser.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.tidb.table; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -34,6 +35,7 @@ public class TidbSinkParser extends RdbSinkParser { @Override public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + props.put(JdbcCheckKeys.DRIVER_NAME, "com.mysql.jdbc.Driver"); AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURRENT_TYPE); return tableInfo; From 42f99ab31191a92196fd2794a3f4000227885a4d Mon Sep 17 00:00:00 2001 From: wuren Date: Mon, 28 Dec 2020 11:02:05 +0800 Subject: [PATCH 28/80] [fix-34015][rdb] reduce default connection count to 5. --- .../com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index bb5a9ea75..4fed5f296 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -77,7 +77,7 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { public final static int DEFAULT_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; - public final static int MAX_DB_CONN_POOL_SIZE_LIMIT = 20; + public final static int MAX_DB_CONN_POOL_SIZE_LIMIT = 5; public final static int DEFAULT_IDLE_CONNECTION_TEST_PEROID = 60; From 089c88b7a16566616735d3391e7604bd133074b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Mon, 28 Dec 2020 14:19:09 +0800 Subject: [PATCH 29/80] resolve conflict --- .../flink/sql/exec/ExecuteProcessHelper.java | 32 +++++++++++++++---- .../sql/parser/CreateTmpTableParser.java | 22 +++++++------ 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 01f22782d..0d447a016 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -180,7 +180,8 @@ public static StreamTableEnvironment getStreamExecution(ParamsInfo paramsInfo) t //register udf ExecuteProcessHelper.registerUserDefinedFunction(sqlTree, paramsInfo.getJarUrlList(), tableEnv, paramsInfo.isGetPlan()); //register table schema - Set classPathSets = ExecuteProcessHelper.registerTable(sqlTree + Set classPathSets = ExecuteProcessHelper.registerTable( + sqlTree , env , tableEnv , paramsInfo.getLocalSqlPluginPath() @@ -192,7 +193,8 @@ public static StreamTableEnvironment getStreamExecution(ParamsInfo paramsInfo) t // cache classPathSets ExecuteProcessHelper.registerPluginUrlToCachedFile(env, classPathSets); - ExecuteProcessHelper.sqlTranslation(paramsInfo.getLocalSqlPluginPath() + ExecuteProcessHelper.sqlTranslation( + paramsInfo.getLocalSqlPluginPath() , paramsInfo.getPluginLoadMode() , tableEnv , sqlTree @@ -285,7 +287,6 @@ private static void sqlTranslation(String localSqlPluginPath, } } } - scope++; } } @@ -327,7 +328,8 @@ public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrl * @return * @throws Exception */ - public static Set registerTable(SqlTree sqlTree + public static Set registerTable( + SqlTree sqlTree , StreamExecutionEnvironment env , StreamTableEnvironment tableEnv , String localSqlPluginPath @@ -372,7 +374,12 @@ public static Set registerTable(SqlTree sqlTree } registerTableCache.put(tableInfo.getName(), regTable); - URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractSourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode( + tableInfo.getType() + , AbstractSourceTableInfo.SOURCE_SUFFIX + , localSqlPluginPath + , remoteSqlPluginPath + , pluginLoadMode); pluginClassPathSets.add(sourceTablePathUrl); } else if (tableInfo instanceof AbstractTargetTableInfo) { @@ -380,7 +387,12 @@ public static Set registerTable(SqlTree sqlTree TypeInformation[] flinkTypes = DataTypeUtils.transformTypes(tableInfo.getFieldClasses()); tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); - URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode( + tableInfo.getType() + , AbstractTargetTableInfo.TARGET_SUFFIX + , localSqlPluginPath + , remoteSqlPluginPath + , pluginLoadMode); pluginClassPathSets.add(sinkTablePathUrl); } else if (tableInfo instanceof AbstractSideTableInfo) { String sideOperator = ECacheType.ALL.name().equalsIgnoreCase(((AbstractSideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; @@ -391,7 +403,13 @@ public static Set registerTable(SqlTree sqlTree tableEnv.registerTableSource(tableInfo.getName(), tableSource); } - URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, AbstractSideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode( + tableInfo.getType() + , sideOperator + , AbstractSideTableInfo.TARGET_SUFFIX + , localSqlPluginPath + , remoteSqlPluginPath + , pluginLoadMode); pluginClassPathSets.add(sideTablePathUrl); } else { throw new RuntimeException("not support table type:" + tableInfo.getType()); diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java index a4ab02a6f..1c13946af 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java @@ -27,8 +27,8 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlMatchRecognize; import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlSnapshot; import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlSnapshot; import java.util.List; import java.util.regex.Matcher; @@ -129,18 +129,18 @@ public boolean verify(String sql) { if (Pattern.compile(EMPTY_STR).matcher(sql).find()) { return true; } - return NONEMPTYVIEW.matcher(sql).find(); + return TEMPORARYVIEW.matcher(sql).find(); } @Override - public void parseSql(String sql, SqlTree sqlTree) { - if (NONEMPTYVIEW.matcher(sql).find()) { - Matcher matcher = NONEMPTYVIEW.matcher(sql); + public void parseSql(String sql, SqlTree sqlTree, String planner) { + if (TEMPORARYVIEW.matcher(sql).find()) { + Matcher matcher = TEMPORARYVIEW.matcher(sql); String tableName = null; String selectSql = null; if (matcher.find()) { - tableName = matcher.group(1); - selectSql = "select " + matcher.group(2); + tableName = matcher.group(3); + selectSql = "select " + matcher.group(4); } SqlNode sqlNode = null; @@ -154,8 +154,11 @@ public void parseSql(String sql, SqlTree sqlTree) { parseNode(sqlNode, sqlParseResult); sqlParseResult.setTableName(tableName); - String transformSelectSql = DtStringUtil.replaceIgnoreQuota(sqlNode.toString(), "`", ""); - sqlParseResult.setExecSql(transformSelectSql); + if (planner.equalsIgnoreCase(PlannerType.FLINK.name())) { + sqlParseResult.setExecSql(sql); + } else { + sqlParseResult.setExecSql(DtStringUtil.replaceIgnoreQuota(sqlNode.toString(), "`", "")); + } sqlTree.addTmpSql(sqlParseResult); sqlTree.addTmplTableInfo(tableName, sqlParseResult); } else { @@ -172,6 +175,7 @@ public void parseSql(String sql, SqlTree sqlTree) { sqlParseResult.setTableName(tableName); sqlTree.addTmplTableInfo(tableName, sqlParseResult); } + } } From 2046fed4e9a989193ea50075ab792b41b3906a30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Mon, 28 Dec 2020 19:53:12 +0800 Subject: [PATCH 30/80] =?UTF-8?q?[feat]=E5=A2=9E=E5=8A=A0=E5=85=A8?= =?UTF-8?q?=E5=B1=80=E8=B5=84=E6=BA=90=E6=A0=A1=E9=AA=8C=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/exec/ExecuteProcessHelper.java | 22 +++++++---- .../dtstack/flink/sql/exec/ParamsInfo.java | 31 +++++++++++---- .../com/dtstack/flink/sql/option/Options.java | 39 ++++++++++++------- .../flink/sql/resource/ResourceCheck.java | 17 +++++--- .../flink/sql/launcher/LauncherMain.java | 2 + .../sql/launcher/entity/JobParamsInfo.java | 18 ++++++++- .../flink/sql/core/rdb/JdbcResourceCheck.java | 14 +++++-- 7 files changed, 102 insertions(+), 41 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 02bfb020f..f2eb1bd56 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -33,6 +33,7 @@ import com.dtstack.flink.sql.parser.InsertSqlParser; import com.dtstack.flink.sql.parser.SqlParser; import com.dtstack.flink.sql.parser.SqlTree; +import com.dtstack.flink.sql.resource.ResourceCheck; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.SideSqlExec; import com.dtstack.flink.sql.sink.StreamSinkFactory; @@ -85,9 +86,10 @@ import java.util.stream.Stream; /** - * 任务执行时的流程方法 + * 任务执行时的流程方法 * Date: 2020/2/17 * Company: www.dtstack.com + * * @author maqi */ public class ExecuteProcessHelper { @@ -114,6 +116,7 @@ public static ParamsInfo parseParams(String[] args) throws Exception { String remoteSqlPluginPath = options.getRemoteSqlPluginPath(); String pluginLoadMode = options.getPluginLoadMode(); String deployMode = options.getMode(); + String checkResource = options.getCheckResource(); Preconditions.checkArgument(checkRemoteSqlPluginPath(remoteSqlPluginPath, deployMode, pluginLoadMode), "Non-local mode or shipfile deployment mode, remoteSqlPluginPath is required"); @@ -131,12 +134,14 @@ public static ParamsInfo parseParams(String[] args) throws Exception { .setDeployMode(deployMode) .setConfProp(confProperties) .setJarUrlList(jarUrlList) + .setCheckResource(checkResource) .build(); } /** - * 非local模式或者shipfile部署模式，remoteSqlPluginPath必填 + * 非local模式或者shipfile部署模式，remoteSqlPluginPath必填 + * * @param remoteSqlPluginPath * @param deployMode * @param pluginLoadMode @@ -155,6 +160,7 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExeEnv(paramsInfo.getConfProp(), paramsInfo.getDeployMode()); StreamTableEnvironment tableEnv = getStreamTableEnv(env, paramsInfo.getConfProp()); + ResourceCheck.NEED_CHECK = Boolean.parseBoolean(paramsInfo.getCheckResource()); SqlParser.setLocalSqlPluginRoot(paramsInfo.getLocalSqlPluginPath()); SqlTree sqlTree = SqlParser.parseSql(paramsInfo.getSql(), paramsInfo.getPluginLoadMode()); @@ -202,7 +208,7 @@ public static List getExternalJarUrls(String addJarListStr) throws java.io. private static void sqlTranslation(String localSqlPluginPath, String pluginLoadMode, StreamTableEnvironment tableEnv, - SqlTree sqlTree,Map sideTableMap, + SqlTree sqlTree, Map sideTableMap, Map registerTableCache) throws Exception { SideSqlExec sideSqlExec = new SideSqlExec(); @@ -276,13 +282,14 @@ public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrl } /** - * 向Flink注册源表和结果表，返回执行时插件包的全路径 + * 向Flink注册源表和结果表，返回执行时插件包的全路径 + * * @param sqlTree * @param env * @param tableEnv * @param localSqlPluginPath * @param remoteSqlPluginPath - * @param pluginLoadMode 插件加载模式 classpath or shipfile + * @param pluginLoadMode 插件加载模式 classpath or shipfile * @param sideTableMap * @param registerTableCache * @return @@ -356,7 +363,8 @@ public static Set registerTable(SqlTree sqlTree, } /** - * perjob模式将job依赖的插件包路径存储到cacheFile，在外围将插件包路径传递给jobgraph + * perjob模式将job依赖的插件包路径存储到cacheFile，在外围将插件包路径传递给jobgraph + * * @param env * @param classPathSet */ @@ -400,7 +408,7 @@ public static StreamTableEnvironment getStreamTableEnv(StreamExecutionEnvironmen private static void timeZoneCheck(String timeZone) { ArrayList zones = Lists.newArrayList(TimeZone.getAvailableIDs()); - if (!zones.contains(timeZone)){ + if (!zones.contains(timeZone)) { throw new IllegalArgumentException(String.format(" timezone of %s is Incorrect!", timeZone)); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java index 9619c75c1..cb73061a0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java @@ -21,13 +21,13 @@ import java.net.URL; import java.util.List; -import java.util.Objects; import java.util.Properties; /** * 解析传递的参数信息 * Date: 2020/2/24 * Company: www.dtstack.com + * * @author maqi */ public class ParamsInfo { @@ -41,9 +41,11 @@ public class ParamsInfo { private String deployMode; private Properties confProp; private boolean getPlan = false; + private String checkResource; public ParamsInfo(String sql, String name, List jarUrlList, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, String deployMode, Properties confProp) { + String remoteSqlPluginPath, String pluginLoadMode, String deployMode, + Properties confProp, String checkResource) { this.sql = sql; this.name = name; this.jarUrlList = jarUrlList; @@ -52,6 +54,11 @@ public ParamsInfo(String sql, String name, List jarUrlList, String localSql this.pluginLoadMode = pluginLoadMode; this.deployMode = deployMode; this.confProp = confProp; + this.checkResource = checkResource; + } + + public static ParamsInfo.Builder builder() { + return new ParamsInfo.Builder(); } public boolean isGetPlan() { @@ -94,17 +101,23 @@ public Properties getConfProp() { return confProp; } + public String getCheckResource() { + return checkResource; + } + @Override public String toString() { return "ParamsInfo{" + "sql='" + sql + '\'' + ", name='" + name + '\'' + - ", jarUrlList=" + convertJarUrlListToString(jarUrlList) + + ", jarUrlList=" + jarUrlList + ", localSqlPluginPath='" + localSqlPluginPath + '\'' + ", remoteSqlPluginPath='" + remoteSqlPluginPath + '\'' + ", pluginLoadMode='" + pluginLoadMode + '\'' + ", deployMode='" + deployMode + '\'' + ", confProp=" + confProp + + ", getPlan=" + getPlan + + ", checkResource='" + checkResource + '\'' + '}'; } @@ -112,9 +125,6 @@ public String convertJarUrlListToString(List jarUrlList) { return jarUrlList.stream().map(URL::toString).reduce((pre, last) -> pre + last).orElse(""); } - public static ParamsInfo.Builder builder() { - return new ParamsInfo.Builder(); - } public static class Builder { private String sql; @@ -125,6 +135,7 @@ public static class Builder { private String pluginLoadMode; private String deployMode; private Properties confProp; + private String checkResource; public ParamsInfo.Builder setSql(String sql) { this.sql = sql; @@ -167,9 +178,15 @@ public ParamsInfo.Builder setConfProp(Properties confProp) { return this; } + public ParamsInfo.Builder setCheckResource(String checkResource) { + this.checkResource = checkResource; + return this; + } + public ParamsInfo build() { return new ParamsInfo(sql, name, jarUrlList, localSqlPluginPath, - remoteSqlPluginPath, pluginLoadMode, deployMode, confProp); + remoteSqlPluginPath, pluginLoadMode, deployMode, confProp, + checkResource); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/option/Options.java b/core/src/main/java/com/dtstack/flink/sql/option/Options.java index 2e2167227..14b63cd17 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/Options.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/Options.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,31 +31,31 @@ public class Options { @OptionRequired(description = "Running mode") - private String mode = ClusterMode.local.name(); + private String mode = ClusterMode.local.name(); - @OptionRequired(required = true,description = "Job name") - private String name; + @OptionRequired(required = true, description = "Job name") + private String name; - @OptionRequired(required = true,description = "Job sql file") - private String sql; + @OptionRequired(required = true, description = "Job sql file") + private String sql; @OptionRequired(description = "Flink configuration directory") - private String flinkconf; + private String flinkconf; @OptionRequired(description = "Yarn and Hadoop configuration directory") - private String yarnconf; + private String yarnconf; @OptionRequired(description = "Sql local plugin root") - private String localSqlPluginPath; + private String localSqlPluginPath; @OptionRequired(description = "Sql remote plugin root") - private String remoteSqlPluginPath ; + private String remoteSqlPluginPath; @OptionRequired(description = "sql ext jar,eg udf jar") - private String addjar; + private String addjar; @OptionRequired(description = "sql ref prop,eg specify event time") - private String confProp = "{}"; + private String confProp = "{}"; @OptionRequired(description = "flink jar path for submit of perjob mode") private String flinkJarPath; @@ -68,10 +68,12 @@ public class Options { @OptionRequired(description = "plugin load mode, by classpath or shipfile") private String pluginLoadMode = EPluginLoadMode.CLASSPATH.name(); - + @OptionRequired(description = "file add to ship file") - private String addShipfile; + private String addShipfile; + @OptionRequired(description = "check resource or not") + private String checkResource; public String getMode() { return mode; @@ -185,4 +187,11 @@ public void setAddShipfile(String addShipfile) { this.addShipfile = addShipfile; } + public String getCheckResource() { + return checkResource; + } + + public void setCheckResource(String checkResource) { + this.checkResource = checkResource; + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java b/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java index 0a5ba3d21..a44b8dde7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java +++ b/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java @@ -18,22 +18,27 @@ package com.dtstack.flink.sql.resource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.Map; -import java.util.Properties; /** * @author: chuixue * @create: 2020-12-08 17:21 * @description:资源检测 **/ -public interface ResourceCheck { - String SINK_STR = "sink"; - String SIDE_STR = "side"; +public abstract class ResourceCheck { + public static Boolean NEED_CHECK = true; + public String TABLE_TYPE_KEY = "tableType"; + public String SINK_STR = "sink"; + public String SIDE_STR = "side"; + protected Logger LOG = LoggerFactory.getLogger(ResourceCheck.class); /** * 资源可用性检测 * - * @param checkProperties 校验资源可用性的参数配置 + * @param checkProperties 校验资源可用性的参数配置 */ - void checkResourceStatus(Map checkProperties); + public abstract void checkResourceStatus(Map checkProperties); } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 1ad7e7d73..0f55ff9cd 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -72,6 +72,7 @@ public static JobParamsInfo parseArgs(String[] args) throws Exception { String queue = launcherOptions.getQueue(); String pluginLoadMode = launcherOptions.getPluginLoadMode(); String addShipfile = launcherOptions.getAddShipfile(); + String checkResource = launcherOptions.getCheckResource(); String yarnSessionConf = URLDecoder.decode(launcherOptions.getYarnSessionConf(), Charsets.UTF_8.toString()); Properties yarnSessionConfProperties = PluginUtil.jsonStrToObject(yarnSessionConf, Properties.class); @@ -93,6 +94,7 @@ public static JobParamsInfo parseArgs(String[] args) throws Exception { .setPluginLoadMode(pluginLoadMode) .setQueue(queue) .setAddShipfile(addShipfile) + .setCheckResource(checkResource) .build(); } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/entity/JobParamsInfo.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/entity/JobParamsInfo.java index 9e7529ef0..0d6915ce8 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/entity/JobParamsInfo.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/entity/JobParamsInfo.java @@ -43,10 +43,11 @@ public class JobParamsInfo { private final Properties confProperties; private final Properties yarnSessionConfProperties; private final String addShipFile; + private final String checkResource; private JobParamsInfo(String mode, String name, String queue, String localPluginRoot, String flinkConfDir, String yarnConfDir, String pluginLoadMode, String[] execArgs, Properties confProperties, Properties yarnSessionConfProperties, - String udfJar, String flinkJarPath, String addShipFile) { + String udfJar, String flinkJarPath, String addShipFile, String checkResource) { this.mode = mode; this.name = name; this.queue = queue; @@ -60,6 +61,7 @@ private JobParamsInfo(String mode, String name, String queue, String localPlugin this.udfJar = udfJar; this.flinkJarPath = flinkJarPath; this.addShipFile = addShipFile; + this.checkResource = checkResource; } public String getMode() { @@ -114,6 +116,10 @@ public String getAddShipFile() { return addShipFile; } + public String getCheckResource() { + return checkResource; + } + public static JobParamsInfo.Builder builder() { return new JobParamsInfo.Builder(); } @@ -133,6 +139,7 @@ public static class Builder { private Properties confProperties; private Properties yarnSessionConfProperties; private String addShipfile; + private String checkResource; public JobParamsInfo.Builder setMode(String mode) { this.mode = mode; @@ -199,10 +206,16 @@ public JobParamsInfo.Builder setAddShipfile(String addShipfile) { return this; } + public JobParamsInfo.Builder setCheckResource(String checkResource) { + this.checkResource = checkResource; + return this; + } + public JobParamsInfo build() { return new JobParamsInfo(mode, name, queue, localPluginRoot, flinkConfDir, yarnConfDir, pluginLoadMode, execArgs, confProperties, - yarnSessionConfProperties, udfJar, flinkJarPath, addShipfile); + yarnSessionConfProperties, udfJar, flinkJarPath, addShipfile, + checkResource); } } @@ -222,6 +235,7 @@ public String toString() { ", confProperties=" + confProperties + ", yarnSessionConfProperties=" + yarnSessionConfProperties + ", addShipFile='" + addShipFile + '\'' + + ", checkResource='" + checkResource + '\'' + '}'; } } diff --git a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java index 0c7f88be8..1d95cb728 100644 --- a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java +++ b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java @@ -30,14 +30,13 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Properties; /** * @author: chuixue * @create: 2020-12-08 17:24 * @description:jdbc资源检测 **/ -public class JdbcResourceCheck implements ResourceCheck { +public class JdbcResourceCheck extends ResourceCheck { private static final String DELETE_STR = "delete"; private static final String SELECT_STR = "select"; private static final String INSERT_STR = "insert"; @@ -66,11 +65,16 @@ public static JdbcResourceCheck getInstance() { @Override public void checkResourceStatus(Map checkProperties) { + if (!NEED_CHECK) { + LOG.info("Ignore checking data source ..."); + return; + } + LOG.info("start checking data source ..."); List privilegeList = new ArrayList<>(); - if (checkProperties.get(JdbcCheckKeys.TABLE_TYPE_KEY).equalsIgnoreCase(SIDE_STR)) { + if (checkProperties.get(TABLE_TYPE_KEY).equalsIgnoreCase(SIDE_STR)) { privilegeList.add(SELECT_STR); } - if (checkProperties.get(JdbcCheckKeys.TABLE_TYPE_KEY).equalsIgnoreCase(SINK_STR)) { + if (checkProperties.get(TABLE_TYPE_KEY).equalsIgnoreCase(SINK_STR)) { privilegeList.add(INSERT_STR); privilegeList.add(DELETE_STR); } @@ -83,6 +87,8 @@ public void checkResourceStatus(Map checkProperties) { , checkProperties.get(JdbcCheckKeys.SCHEMA_KEY) , privilegeList ); + LOG.info(String.format("data source is available and user [%s] has the corresponding permissions %s for [%s] table...", + checkProperties.get(JdbcCheckKeys.USER_NAME_KEY), privilegeList.toString(), checkProperties.get(TABLE_TYPE_KEY))); } public void checkPrivilege( From 6795386b022d8d8ab63c9370bb22c22c128e26d1 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 29 Dec 2020 13:56:01 +0800 Subject: [PATCH 31/80] [fix-33847][core][kafka] add array and map type in the kafka sink. --- .../flink/sql/exec/ExecuteProcessHelper.java | 4 +- .../flink/sql/function/FunctionManager.java | 9 ---- .../com/dtstack/flink/sql/util/ClassUtil.java | 6 +++ .../dtstack/flink/sql/util/DataTypeUtils.java | 45 ++++++++++++------- docs/plugin/kafkaSink.md | 26 +++++++++++ docs/pluginsInfo.md | 2 +- .../sql/sink/kafka/AbstractKafkaSink.java | 16 ++++++- 7 files changed, 77 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 02bfb020f..c18af7bc5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -334,9 +334,7 @@ public static Set registerTable(SqlTree sqlTree, } else if (tableInfo instanceof AbstractTargetTableInfo) { TableSink tableSink = StreamSinkFactory.getTableSink((AbstractTargetTableInfo) tableInfo, localSqlPluginPath, pluginLoadMode); - TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); - tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); - + tableEnv.registerTableSink(tableInfo.getName(), tableSink); URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPathSets.add(sinkTablePathUrl); } else if (tableInfo instanceof AbstractSideTableInfo) { diff --git a/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java b/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java index c1e58701f..a9ceea89c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java @@ -124,13 +124,4 @@ public static void registerAggregateUDF(String classPath, String funcName, Table } } - - public static TypeInformation[] transformTypes(Class[] fieldTypes) { - TypeInformation[] types = new TypeInformation[fieldTypes.length]; - for (int i = 0; i < fieldTypes.length; i++) { - types[i] = TypeInformation.of(fieldTypes[i]); - } - - return types; - } } \ No newline at end of file diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index ec88bdbe9..b7d2f12fc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -25,6 +25,8 @@ import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; +import java.util.HashMap; +import java.util.Map; /** * Reason: TODO ADD REASON(可选) @@ -41,6 +43,10 @@ public static Class stringConvertClass(String str) { if (lowerStr.startsWith("array")) { return Array.newInstance(Integer.class, 0).getClass(); } + if (lowerStr.startsWith("map")) { + Map m = new HashMap(); + return m.getClass(); + } switch (lowerStr) { case "boolean": diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index 72a8b4b66..bb15b9c45 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -20,6 +20,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -30,6 +31,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.apache.commons.lang3.StringUtils.split; + /** * @program: flink.sql * @author: wuren @@ -39,28 +42,13 @@ public class DataTypeUtils { private final static Pattern COMPOSITE_TYPE_PATTERN = Pattern.compile("(.+?)<(.+)>"); private final static String ARRAY = "ARRAY"; + private final static String MAP = "MAP"; private final static String ROW = "ROW"; private final static char FIELD_DELIMITER = ','; private final static char TYPE_DELIMITER = ' '; private DataTypeUtils() {} - /** - * 现在只支持ARRAY类型后续可以加入 MAP等类型 - * @param compositeTypeString - * @return - */ - public static TypeInformation convertToCompositeType(String compositeTypeString) { - Matcher matcher = matchCompositeType(compositeTypeString); - final String errorMsg = "type " + compositeTypeString + "is not support!"; - Preconditions.checkState(matcher.find(), errorMsg); - - String normalizedType = normalizeType(matcher.group(1)); - Preconditions.checkState(ARRAY.equals(normalizedType), errorMsg); - - return convertToArray(compositeTypeString); - } - /** * 目前ARRAY里只支持ROW和其他基本类型 * @param arrayTypeString @@ -86,6 +74,30 @@ public static TypeInformation convertToArray(String arrayTypeString) { return Types.OBJECT_ARRAY(elementType); } + /** + * 目前Map里只支持基本类型 + * @param mapTypeString + * @return + */ + public static TypeInformation convertToMap(String mapTypeString) { + Matcher matcher = matchCompositeType(mapTypeString); + final String errorMsg = mapTypeString + "convert to map type error!"; + Preconditions.checkState(matcher.find(), errorMsg); + + String normalizedType = normalizeType(matcher.group(1)); + Preconditions.checkState(MAP.equals(normalizedType), errorMsg); + + String kvTypeString = matcher.group(2); + String[] kvTypeStringList = StringUtils.split(kvTypeString, ","); + final String mapTypeErrorMsg = "There can only be key and value two types in map declaration."; + Preconditions.checkState(kvTypeStringList.length == 2, mapTypeErrorMsg); + String keyTypeString = normalizeType(kvTypeStringList[0]); + String valueTypeString = normalizeType(kvTypeStringList[1]); + TypeInformation keyType = convertToAtomicType(keyTypeString); + TypeInformation valueType = convertToAtomicType(valueTypeString); + return Types.MAP(keyType, valueType); + } + /** * 目前ROW里只支持基本类型 * @param rowTypeString @@ -104,6 +116,7 @@ public static RowTypeInfo convertToRow(String rowTypeString) { return new RowTypeInfo(info.f0, info.f1); } + private static Tuple2 genFieldInfo(Iterable fieldInfoStrs) { ArrayList types = Lists.newArrayList(); ArrayList fieldNames = Lists.newArrayList(); diff --git a/docs/plugin/kafkaSink.md b/docs/plugin/kafkaSink.md index 86b01f3ee..b3a1614bc 100644 --- a/docs/plugin/kafkaSink.md +++ b/docs/plugin/kafkaSink.md @@ -221,3 +221,29 @@ into from MyTable a ``` +## MAP类型示例 +目前Kafka Sink支持Map类型 +```sql +CREATE TABLE ods( + id INT, + name STRING +) WITH ( + ... +); + +CREATE TABLE dwd ( + id INT, + dids MAP> +) WITH ( + type ='kafka', + bootstrapServers ='localhost:9092', + offsetReset ='latest', + groupId='wuren_foo', + topic ='luna_foo', + parallelism ='1' +); + +INSERT INTO dwd + SELECT ods.id, MAP['foo', 1, 'bar', 2] AS dids + FROM ods; +``` diff --git a/docs/pluginsInfo.md b/docs/pluginsInfo.md index 88fda90c9..e9d23cdbd 100644 --- a/docs/pluginsInfo.md +++ b/docs/pluginsInfo.md @@ -1,9 +1,9 @@ ### 1 插件列表 #### 1.1 源表插件 * [kafka 源表插件](plugin/kafkaSource.md) -* [kafka 结果表插件](plugin/kafkaSink.md) #### 1.2 结果表插件 +* [kafka 结果表插件](plugin/kafkaSink.md) * [elasticsearch 结果表插件](plugin/elasticsearchSink.md) * [hbase 结果表插件](plugin/hbaseSink.md) * [mysql 结果表插件](plugin/mysqlSink.md) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index fb87c4217..560df5c28 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; +import com.dtstack.flink.sql.util.DataTypeUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -38,6 +39,7 @@ import org.apache.flink.util.Preconditions; import org.apache.kafka.clients.consumer.ConsumerConfig; +import java.util.HashMap; import java.util.Optional; import java.util.Properties; import java.util.stream.IntStream; @@ -77,11 +79,21 @@ protected Properties getKafkaProperties(KafkaSinkTableInfo KafkaSinkTableInfo) { } return props; } - + // TODO Source有相同的方法日后可以合并 protected TypeInformation[] getTypeInformations(KafkaSinkTableInfo kafka11SinkTableInfo) { + String[] fieldTypes = kafka11SinkTableInfo.getFieldTypes(); Class[] fieldClasses = kafka11SinkTableInfo.getFieldClasses(); TypeInformation[] types = IntStream.range(0, fieldClasses.length) - .mapToObj(i -> TypeInformation.of(fieldClasses[i])) + .mapToObj( + i -> { + if (fieldClasses[i].isArray()) { + return DataTypeUtils.convertToArray(fieldTypes[i]); + } + if (fieldClasses[i] == new HashMap().getClass()) { + return DataTypeUtils.convertToMap(fieldTypes[i]); + } + return TypeInformation.of(fieldClasses[i]); + }) .toArray(TypeInformation[]::new); return types; } From 8c4a25a0b0833c9d0241507a4b698a680dcfec1c Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 29 Dec 2020 21:11:01 +0800 Subject: [PATCH 32/80] [fix-33847][core] fix other sink except kafka table schema not found problem. --- .../dtstack/flink/sql/exec/ExecuteProcessHelper.java | 10 ++++++++-- .../dtstack/flink/sql/function/FunctionManager.java | 8 ++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index c18af7bc5..3125ca85d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -332,9 +332,15 @@ public static Set registerTable(SqlTree sqlTree, URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractSourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPathSets.add(sourceTablePathUrl); } else if (tableInfo instanceof AbstractTargetTableInfo) { - TableSink tableSink = StreamSinkFactory.getTableSink((AbstractTargetTableInfo) tableInfo, localSqlPluginPath, pluginLoadMode); - tableEnv.registerTableSink(tableInfo.getName(), tableSink); + // TODO Kafka Sink直接注册，其他的Sink要修复才可以。 + if (tableInfo.getType().startsWith("kafka")) { + tableEnv.registerTableSink(tableInfo.getName(), tableSink); + } else { + TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); + tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); + } + URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPathSets.add(sinkTablePathUrl); } else if (tableInfo instanceof AbstractSideTableInfo) { diff --git a/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java b/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java index a9ceea89c..51d914a68 100644 --- a/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java @@ -124,4 +124,12 @@ public static void registerAggregateUDF(String classPath, String funcName, Table } } + public static TypeInformation[] transformTypes(Class[] fieldTypes) { + TypeInformation[] types = new TypeInformation[fieldTypes.length]; + for (int i = 0; i < fieldTypes.length; i++) { + types[i] = TypeInformation.of(fieldTypes[i]); + } + return types; + } + } \ No newline at end of file From 47f7434955722467d67107a6556efc32201b54ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Wed, 30 Dec 2020 10:45:11 +0800 Subject: [PATCH 33/80] =?UTF-8?q?[opt]=20=E8=B0=83=E6=95=B4=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/util/ThreadUtil.java | 4 ++-- .../dtstack/flink/sql/localTest/LocalTest.java | 3 ++- .../flink/sql/core/rdb/util/JdbcConnectUtil.java | 15 +++++++++------ .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ThreadUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ThreadUtil.java index 5e73f7c3b..d6f57788e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ThreadUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ThreadUtil.java @@ -27,9 +27,9 @@ */ public class ThreadUtil { public static final Long DEFAULT_SLEEP_TIME = 10L; - public static void sleepMilliseconds(long timeout) { + public static void sleepSeconds(long timeout) { try { - TimeUnit.MILLISECONDS.sleep(timeout); + TimeUnit.SECONDS.sleep(timeout); } catch (InterruptedException ie) { throw new RuntimeException(ie); } diff --git a/localTest/src/main/java/com/dtstack/flink/sql/localTest/LocalTest.java b/localTest/src/main/java/com/dtstack/flink/sql/localTest/LocalTest.java index c0e435ac0..168064a1e 100644 --- a/localTest/src/main/java/com/dtstack/flink/sql/localTest/LocalTest.java +++ b/localTest/src/main/java/com/dtstack/flink/sql/localTest/LocalTest.java @@ -50,7 +50,7 @@ public static void main(String[] args) throws Exception { setLogLevel("INFO"); List propertiesList = new ArrayList<>(); - String sqlPath = "/Users/chuixue/Desktop/tmp/sqlFile.sql"; + String sqlPath = "/Users/wtz/dtstack/sql/test/JoinDemoThree.sql"; Map conf = new HashMap<>(); JSONObject properties = new JSONObject(); @@ -65,6 +65,7 @@ public static void main(String[] args) throws Exception { conf.put("-name", "flinkStreamSQLLocalTest"); conf.put("-confProp", properties.toString()); conf.put("-pluginLoadMode", "LocalTest"); + conf.put("-checkResource", "true"); for (Map.Entry keyValue : conf.entrySet()) { propertiesList.add(keyValue.getKey()); diff --git a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java index 2d4a49d3e..36893fce3 100644 --- a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java +++ b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java @@ -130,8 +130,10 @@ public static Connection getConnectWithRetry( , String url , String userName , String password) { - String errorMessage = "Get connect failed with properties: \nurl: " + url - + (Objects.isNull(userName) ? "" : "\nuserName: " + userName); + String errorMessage = "\nGet connect failed with properties: \nurl: " + url + + (Objects.isNull(userName) ? "" : "\nuserName: " + userName + + "\nerror message: "); + String errorCause = null; ClassLoaderManager.forName(driverName); Preconditions.checkNotNull(url, "url can't be null!"); @@ -145,12 +147,13 @@ public static Connection getConnectWithRetry( connection.isValid(DEFAULT_VALID_NUM); return connection; } catch (Exception e) { - LOG.warn(errorMessage, e); - LOG.warn("Connect will retry after [{}] s......", DEFAULT_RETRY_TIME_WAIT); - ThreadUtil.sleepMilliseconds(DEFAULT_RETRY_TIME_WAIT); + errorCause = e.getCause().toString(); + LOG.warn(errorMessage + e.getCause()); + LOG.warn("Connect will retry after [{}] s. Retry time [{}] ...", DEFAULT_RETRY_TIME_WAIT, i + 1); + ThreadUtil.sleepSeconds(DEFAULT_RETRY_TIME_WAIT); closeConnectionResource(null, null, connection, false); } } - throw new IllegalArgumentException(errorMessage); + throw new IllegalArgumentException(errorMessage + errorCause); } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 0a568c20c..5b1b9e8f1 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -200,7 +200,7 @@ private void connectWithRetry(Map inputParams, Row input, Result connectionStatus.set(false); } if (!finishFlag.get()) { - ThreadUtil.sleepMilliseconds(ThreadUtil.DEFAULT_SLEEP_TIME); + ThreadUtil.sleepSeconds(ThreadUtil.DEFAULT_SLEEP_TIME); } } } From f637482285c9b64955a669334c90415fc7778349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Wed, 30 Dec 2020 13:54:41 +0800 Subject: [PATCH 34/80] =?UTF-8?q?[opt]=201.=20=E8=B0=83=E6=95=B4checkResou?= =?UTF-8?q?rce=E5=8F=82=E6=95=B0;=202.=20=E4=BC=98=E5=8C=96=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/exec/ExecuteProcessHelper.java | 4 +--- .../com/dtstack/flink/sql/exec/ParamsInfo.java | 17 ++--------------- .../com/dtstack/flink/sql/option/Options.java | 11 ----------- .../flink/sql/resource/ResourceCheck.java | 3 ++- .../flink/sql/launcher/LauncherMain.java | 2 -- .../sql/launcher/entity/JobParamsInfo.java | 18 ++---------------- .../sql/core/rdb/util/JdbcConnectUtil.java | 18 +++++++----------- .../sql/side/rdb/all/AbstractRdbAllReqRow.java | 2 +- 8 files changed, 15 insertions(+), 60 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index f2eb1bd56..bf224872d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -116,7 +116,6 @@ public static ParamsInfo parseParams(String[] args) throws Exception { String remoteSqlPluginPath = options.getRemoteSqlPluginPath(); String pluginLoadMode = options.getPluginLoadMode(); String deployMode = options.getMode(); - String checkResource = options.getCheckResource(); Preconditions.checkArgument(checkRemoteSqlPluginPath(remoteSqlPluginPath, deployMode, pluginLoadMode), "Non-local mode or shipfile deployment mode, remoteSqlPluginPath is required"); @@ -134,7 +133,6 @@ public static ParamsInfo parseParams(String[] args) throws Exception { .setDeployMode(deployMode) .setConfProp(confProperties) .setJarUrlList(jarUrlList) - .setCheckResource(checkResource) .build(); } @@ -160,7 +158,7 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExeEnv(paramsInfo.getConfProp(), paramsInfo.getDeployMode()); StreamTableEnvironment tableEnv = getStreamTableEnv(env, paramsInfo.getConfProp()); - ResourceCheck.NEED_CHECK = Boolean.parseBoolean(paramsInfo.getCheckResource()); + ResourceCheck.NEED_CHECK = Boolean.parseBoolean(paramsInfo.getConfProp().getProperty(ResourceCheck.CHECK_STR, "true")); SqlParser.setLocalSqlPluginRoot(paramsInfo.getLocalSqlPluginPath()); SqlTree sqlTree = SqlParser.parseSql(paramsInfo.getSql(), paramsInfo.getPluginLoadMode()); diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java index cb73061a0..65e560465 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java @@ -41,11 +41,10 @@ public class ParamsInfo { private String deployMode; private Properties confProp; private boolean getPlan = false; - private String checkResource; public ParamsInfo(String sql, String name, List jarUrlList, String localSqlPluginPath, String remoteSqlPluginPath, String pluginLoadMode, String deployMode, - Properties confProp, String checkResource) { + Properties confProp) { this.sql = sql; this.name = name; this.jarUrlList = jarUrlList; @@ -54,7 +53,6 @@ public ParamsInfo(String sql, String name, List jarUrlList, String localSql this.pluginLoadMode = pluginLoadMode; this.deployMode = deployMode; this.confProp = confProp; - this.checkResource = checkResource; } public static ParamsInfo.Builder builder() { @@ -101,10 +99,6 @@ public Properties getConfProp() { return confProp; } - public String getCheckResource() { - return checkResource; - } - @Override public String toString() { return "ParamsInfo{" + @@ -117,7 +111,6 @@ public String toString() { ", deployMode='" + deployMode + '\'' + ", confProp=" + confProp + ", getPlan=" + getPlan + - ", checkResource='" + checkResource + '\'' + '}'; } @@ -178,15 +171,9 @@ public ParamsInfo.Builder setConfProp(Properties confProp) { return this; } - public ParamsInfo.Builder setCheckResource(String checkResource) { - this.checkResource = checkResource; - return this; - } - public ParamsInfo build() { return new ParamsInfo(sql, name, jarUrlList, localSqlPluginPath, - remoteSqlPluginPath, pluginLoadMode, deployMode, confProp, - checkResource); + remoteSqlPluginPath, pluginLoadMode, deployMode, confProp); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/option/Options.java b/core/src/main/java/com/dtstack/flink/sql/option/Options.java index 14b63cd17..315ce5f62 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/Options.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/Options.java @@ -72,9 +72,6 @@ public class Options { @OptionRequired(description = "file add to ship file") private String addShipfile; - @OptionRequired(description = "check resource or not") - private String checkResource; - public String getMode() { return mode; } @@ -186,12 +183,4 @@ public String getAddShipfile() { public void setAddShipfile(String addShipfile) { this.addShipfile = addShipfile; } - - public String getCheckResource() { - return checkResource; - } - - public void setCheckResource(String checkResource) { - this.checkResource = checkResource; - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java b/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java index a44b8dde7..41436c0b3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java +++ b/core/src/main/java/com/dtstack/flink/sql/resource/ResourceCheck.java @@ -30,10 +30,11 @@ **/ public abstract class ResourceCheck { public static Boolean NEED_CHECK = true; + public static String CHECK_STR = "checkResource"; + protected static Logger LOG = LoggerFactory.getLogger(ResourceCheck.class); public String TABLE_TYPE_KEY = "tableType"; public String SINK_STR = "sink"; public String SIDE_STR = "side"; - protected Logger LOG = LoggerFactory.getLogger(ResourceCheck.class); /** * 资源可用性检测 diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 0f55ff9cd..1ad7e7d73 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -72,7 +72,6 @@ public static JobParamsInfo parseArgs(String[] args) throws Exception { String queue = launcherOptions.getQueue(); String pluginLoadMode = launcherOptions.getPluginLoadMode(); String addShipfile = launcherOptions.getAddShipfile(); - String checkResource = launcherOptions.getCheckResource(); String yarnSessionConf = URLDecoder.decode(launcherOptions.getYarnSessionConf(), Charsets.UTF_8.toString()); Properties yarnSessionConfProperties = PluginUtil.jsonStrToObject(yarnSessionConf, Properties.class); @@ -94,7 +93,6 @@ public static JobParamsInfo parseArgs(String[] args) throws Exception { .setPluginLoadMode(pluginLoadMode) .setQueue(queue) .setAddShipfile(addShipfile) - .setCheckResource(checkResource) .build(); } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/entity/JobParamsInfo.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/entity/JobParamsInfo.java index 0d6915ce8..9e7529ef0 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/entity/JobParamsInfo.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/entity/JobParamsInfo.java @@ -43,11 +43,10 @@ public class JobParamsInfo { private final Properties confProperties; private final Properties yarnSessionConfProperties; private final String addShipFile; - private final String checkResource; private JobParamsInfo(String mode, String name, String queue, String localPluginRoot, String flinkConfDir, String yarnConfDir, String pluginLoadMode, String[] execArgs, Properties confProperties, Properties yarnSessionConfProperties, - String udfJar, String flinkJarPath, String addShipFile, String checkResource) { + String udfJar, String flinkJarPath, String addShipFile) { this.mode = mode; this.name = name; this.queue = queue; @@ -61,7 +60,6 @@ private JobParamsInfo(String mode, String name, String queue, String localPlugin this.udfJar = udfJar; this.flinkJarPath = flinkJarPath; this.addShipFile = addShipFile; - this.checkResource = checkResource; } public String getMode() { @@ -116,10 +114,6 @@ public String getAddShipFile() { return addShipFile; } - public String getCheckResource() { - return checkResource; - } - public static JobParamsInfo.Builder builder() { return new JobParamsInfo.Builder(); } @@ -139,7 +133,6 @@ public static class Builder { private Properties confProperties; private Properties yarnSessionConfProperties; private String addShipfile; - private String checkResource; public JobParamsInfo.Builder setMode(String mode) { this.mode = mode; @@ -206,16 +199,10 @@ public JobParamsInfo.Builder setAddShipfile(String addShipfile) { return this; } - public JobParamsInfo.Builder setCheckResource(String checkResource) { - this.checkResource = checkResource; - return this; - } - public JobParamsInfo build() { return new JobParamsInfo(mode, name, queue, localPluginRoot, flinkConfDir, yarnConfDir, pluginLoadMode, execArgs, confProperties, - yarnSessionConfProperties, udfJar, flinkJarPath, addShipfile, - checkResource); + yarnSessionConfProperties, udfJar, flinkJarPath, addShipfile); } } @@ -235,7 +222,6 @@ public String toString() { ", confProperties=" + confProperties + ", yarnSessionConfProperties=" + yarnSessionConfProperties + ", addShipFile='" + addShipFile + '\'' + - ", checkResource='" + checkResource + '\'' + '}'; } } diff --git a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java index 36893fce3..fc7c79bd0 100644 --- a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java +++ b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.util.ThreadUtil; import com.google.common.base.Preconditions; +import org.apache.flink.runtime.execution.SuppressRestartsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,9 +38,9 @@ * Company dtstack */ public class JdbcConnectUtil { - private static final int DEFAULT_RETRY_NUM = 5; + private static final int DEFAULT_RETRY_NUM = 3; private static final long DEFAULT_RETRY_TIME_WAIT = 5L; - private static final int DEFAULT_VALID_NUM = 10; + private static final int DEFAULT_VALID_TIME = 10; private static final Logger LOG = LoggerFactory.getLogger(JdbcConnectUtil.class); /** @@ -93,7 +94,7 @@ public static void closeConnectionResource( */ public static void commit(Connection conn) { try { - if (!conn.isClosed() && conn.isValid(DEFAULT_VALID_NUM) && !conn.getAutoCommit()) { + if (!conn.isClosed() && conn.isValid(DEFAULT_VALID_TIME) && !conn.getAutoCommit()) { conn.commit(); } } catch (SQLException e) { @@ -108,7 +109,7 @@ public static void commit(Connection conn) { */ public static void rollBack(Connection conn) { try { - if (!conn.isClosed() && conn.isValid(DEFAULT_VALID_NUM) && !conn.getAutoCommit()) { + if (!conn.isClosed() && conn.isValid(DEFAULT_VALID_TIME) && !conn.getAutoCommit()) { conn.rollback(); } } catch (SQLException e) { @@ -138,22 +139,17 @@ public static Connection getConnectWithRetry( ClassLoaderManager.forName(driverName); Preconditions.checkNotNull(url, "url can't be null!"); - Connection connection = null; for (int i = 0; i < DEFAULT_RETRY_NUM; i++) { try { - connection = Objects.isNull(userName) ? + return Objects.isNull(userName) ? DriverManager.getConnection(url) : DriverManager.getConnection(url, userName, password); - // 校验connection是否可用 - connection.isValid(DEFAULT_VALID_NUM); - return connection; } catch (Exception e) { errorCause = e.getCause().toString(); LOG.warn(errorMessage + e.getCause()); LOG.warn("Connect will retry after [{}] s. Retry time [{}] ...", DEFAULT_RETRY_TIME_WAIT, i + 1); ThreadUtil.sleepSeconds(DEFAULT_RETRY_TIME_WAIT); - closeConnectionResource(null, null, connection, false); } } - throw new IllegalArgumentException(errorMessage + errorCause); + throw new SuppressRestartsException(new Throwable(errorMessage + errorCause)); } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 0df36cf46..e0fe18fb7 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -79,7 +79,6 @@ public AbstractRdbAllReqRow(BaseSideInfo sideInfo) { @Override public void open(Configuration parameters) throws Exception { - super.open(parameters); RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); synchronized (AbstractRdbAllReqRow.class) { if (resourceCheck) { @@ -88,6 +87,7 @@ public void open(Configuration parameters) throws Exception { } } LOG.info("rdb dim table config info: {} ", tableInfo.toString()); + super.open(parameters); } @Override From d886e6866f288aabdf17d77b21d45eaa1e49a2ec Mon Sep 17 00:00:00 2001 From: chuixue Date: Wed, 30 Dec 2020 14:51:38 +0800 Subject: [PATCH 35/80] =?UTF-8?q?[hotfix-32959][core][launcher]=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E5=8F=8C=E5=B1=82group=20by=E7=BC=BA=E5=B0=91?= =?UTF-8?q?=E6=92=A4=E5=9B=9E=E6=95=B0=E6=8D=AE=EF=BC=8C=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E7=BB=93=E7=AE=97=E7=BB=93=E6=9E=9C=E9=94=99=E8=AF=AF=EF=BC=8C?= =?UTF-8?q?=E4=B8=BB=E8=A6=81=E6=98=AF=E5=B0=86=E7=BB=B4=E8=A1=A8=E4=B8=AD?= =?UTF-8?q?=E7=9A=84row=E7=B1=BB=E5=9E=8B=E6=94=B9=E6=88=90baserow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../side/cassandra/CassandraAllReqRow.java | 18 +- .../side/cassandra/CassandraAsyncReqRow.java | 31 +- core/pom.xml | 15 - .../flink/sql/exec/ExecuteProcessHelper.java | 2 +- .../dtstack/flink/sql/side/BaseAllReqRow.java | 18 +- .../flink/sql/side/BaseAsyncReqRow.java | 47 +- .../dtstack/flink/sql/side/ISideReqRow.java | 4 +- .../dtstack/flink/sql/side/SideSqlExec.java | 35 +- .../flink/sql/side/TupleKeySelector.java | 18 +- .../sql/side/operator/SideAsyncOperator.java | 5 +- .../flink/sql/util/RowDataComplete.java | 37 +- .../flink/sql/util/RowDataConvert.java | 11 +- .../Elasticsearch6AllReqRow.java | 15 +- .../Elasticsearch6AsyncReqRow.java | 45 +- .../flink/sql/side/hbase/HbaseAllReqRow.java | 21 +- .../sql/side/hbase/HbaseAsyncReqRow.java | 12 +- .../AbstractRowKeyModeDealer.java | 18 +- .../PreRowKeyModeDealerDealer.java | 20 +- .../rowkeydealer/RowKeyEqualModeDealer.java | 5 +- .../sql/side/impala/ImpalaAsyncReqRow.java | 6 +- .../flink/sql/side/kudu/KuduAllReqRow.java | 13 +- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 24 +- .../planner/plan/QueryOperationConverter.java | 652 ++++++++++++++++++ .../flink/sql/side/mongo/MongoAllReqRow.java | 15 +- .../sql/side/mongo/MongoAsyncReqRow.java | 17 +- .../side/rdb/all/AbstractRdbAllReqRow.java | 14 +- .../sql/side/rdb/async/RdbAsyncReqRow.java | 31 +- .../flink/sql/side/redis/RedisAllReqRow.java | 15 +- .../sql/side/redis/RedisAsyncReqRow.java | 24 +- .../sql/side/redis/table/RedisSideReqRow.java | 15 +- 30 files changed, 920 insertions(+), 283 deletions(-) create mode 100644 launcher/src/main/java/org/apache/flink/table/planner/plan/QueryOperationConverter.java diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index 1b12165e5..877fa1ec2 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -18,10 +18,6 @@ package com.dtstack.flink.sql.side.cassandra; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; - import com.datastax.driver.core.Cluster; import com.datastax.driver.core.ConsistencyLevel; import com.datastax.driver.core.HostDistance; @@ -43,7 +39,10 @@ import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.table.dataformat.BaseRow; +import org.apache.flink.table.dataformat.GenericRow; +import org.apache.flink.util.Collector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,13 +102,14 @@ protected void reloadCache() { @Override - public void flatMap(Row input, Collector out) throws Exception { + public void flatMap(BaseRow input, Collector out) throws Exception { + GenericRow genericRow = (GenericRow) input; List inputParams = Lists.newArrayList(); for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = input.getField(conValIndex); + Object equalObj = genericRow.getField(conValIndex); if (equalObj == null) { if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(input, null); + BaseRow row = fillData(input, null); RowDataComplete.collectRow(out, row); } return; @@ -122,7 +122,7 @@ public void flatMap(Row input, Collector out) throws Exception { List> cacheList = cacheRef.get().get(key); if (CollectionUtils.isEmpty(cacheList)) { if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(input, null); + BaseRow row = fillData(input, null); RowDataComplete.collectRow(out, row); } else { return; @@ -132,7 +132,7 @@ public void flatMap(Row input, Collector out) throws Exception { } for (Map one : cacheList) { - Row row = fillData(input, one); + BaseRow row = fillData(input, one); RowDataComplete.collectRow(out, row); } diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index 9ce811035..a44bece33 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -19,11 +19,22 @@ package com.dtstack.flink.sql.side.cassandra; -import com.datastax.driver.core.*; +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.ConsistencyLevel; +import com.datastax.driver.core.HostDistance; +import com.datastax.driver.core.PoolingOptions; +import com.datastax.driver.core.QueryOptions; +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.SocketOptions; import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.dtstack.flink.sql.util.RowDataComplete; @@ -38,7 +49,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.types.Row; +import org.apache.flink.table.dataformat.GenericRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -149,7 +160,7 @@ private void connCassandraDB(CassandraSideTableInfo tableInfo) { } @Override - public void handleAsyncInvoke(Map inputParams, Row input, ResultFuture resultFuture) throws Exception { + public void handleAsyncInvoke(Map inputParams, BaseRow input, ResultFuture resultFuture) throws Exception { String key = buildCacheKey(inputParams); //connect Cassandra @@ -180,9 +191,9 @@ public void onSuccess(List rows) { cluster.closeAsync(); if (rows.size() > 0) { List cacheContent = Lists.newArrayList(); - List rowList = Lists.newArrayList(); + List rowList = Lists.newArrayList(); for (com.datastax.driver.core.Row line : rows) { - Row row = fillData(input, line); + BaseRow row = fillData(input, line); if (openCache()) { cacheContent.add(line); } @@ -230,11 +241,13 @@ private String buildWhereCondition(Map inputParams){ } @Override - public Row fillData(Row input, Object line) { + public BaseRow fillData(BaseRow input, Object line) { + GenericRow genericRow = (GenericRow) input; com.datastax.driver.core.Row rowArray = (com.datastax.driver.core.Row) line; - Row row = new Row(sideInfo.getOutFieldInfoList().size()); + GenericRow row = new GenericRow(sideInfo.getOutFieldInfoList().size()); + row.setHeader(genericRow.getHeader()); for (Map.Entry entry : sideInfo.getInFieldIndex().entrySet()) { - Object obj = input.getField(entry.getValue()); + Object obj = genericRow.getField(entry.getValue()); obj = convertTimeIndictorTypeInfo(entry.getValue(), obj); row.setField(entry.getKey(), obj); } diff --git a/core/pom.xml b/core/pom.xml index ce7799f5a..0ae790abb 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -17,7 +17,6 @@ UTF-8 core - 1.16.0 2.7.9 19.0 1.0.0-SNAPSHOT @@ -79,20 +78,6 @@ ${flink.version} - - - org.apache.calcite - calcite-server - - ${calcite.server.version} - - - jackson-databind - com.fasterxml.jackson.core - - - - com.fasterxml.jackson.core jackson-databind diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 02bfb020f..32e8571c1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -323,7 +323,7 @@ public static Set registerTable(SqlTree sqlTree, } Table regTable = tableEnv.fromDataStream(adaptStream, fields); - tableEnv.registerTable(tableInfo.getName(), regTable); + tableEnv.createTemporaryView(tableInfo.getName(), regTable); if (LOG.isInfoEnabled()) { LOG.info("registe table {} success.", tableInfo.getName()); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java index 69d9b693c..d09caaa16 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java @@ -26,8 +26,8 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.configuration.Configuration; import org.apache.flink.table.dataformat.BaseRow; +import org.apache.flink.table.dataformat.GenericRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; import org.apache.flink.util.Collector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,6 @@ import java.time.LocalDateTime; import java.util.Map; import java.util.TimeZone; -import java.util.TimeZone; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -50,7 +49,7 @@ * @author xuchao */ -public abstract class BaseAllReqRow extends RichFlatMapFunction implements ISideReqRow { +public abstract class BaseAllReqRow extends RichFlatMapFunction implements ISideReqRow { private static final Logger LOG = LoggerFactory.getLogger(BaseAllReqRow.class); @@ -94,22 +93,23 @@ protected Object convertTimeIndictorTypeInfo(Integer index, Object obj) { return obj; } - protected void sendOutputRow(Row value, Object sideInput, Collector out) { + protected void sendOutputRow(BaseRow value, Object sideInput, Collector out) { if (sideInput == null && sideInfo.getJoinType() != JoinType.LEFT) { return; } - Row row = fillData(value, sideInput); + BaseRow row = fillData(value, sideInput); RowDataComplete.collectRow(out, row); } @Override - public Row fillData(Row input, Object sideInput) { + public BaseRow fillData(BaseRow input, Object sideInput) { + GenericRow genericRow = (GenericRow) input; Map cacheInfo = (Map) sideInput; - Row row = new Row(sideInfo.getOutFieldInfoList().size()); - + GenericRow row = new GenericRow(sideInfo.getOutFieldInfoList().size()); + row.setHeader(genericRow.getHeader()); for (Map.Entry entry : sideInfo.getInFieldIndex().entrySet()) { // origin value - Object obj = input.getField(entry.getValue()); + Object obj = genericRow.getField(entry.getValue()); obj = dealTimeAttributeType(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass(), obj); row.setField(entry.getKey(), obj); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 276f1fcd2..d2380cfb3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -38,10 +38,9 @@ import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService; -import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.dataformat.BaseRow; +import org.apache.flink.table.dataformat.GenericRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,7 +63,7 @@ * @author xuchao */ -public abstract class BaseAsyncReqRow extends RichAsyncFunction implements ISideReqRow { +public abstract class BaseAsyncReqRow extends RichAsyncFunction implements ISideReqRow { private static final Logger LOG = LoggerFactory.getLogger(BaseAsyncReqRow.class); private static final long serialVersionUID = 2098635244857937717L; private RuntimeContext runtimeContext; @@ -137,11 +136,11 @@ protected boolean openCache() { return sideInfo.getSideCache() != null; } - protected void dealMissKey(Row input, ResultFuture resultFuture) { + protected void dealMissKey(BaseRow input, ResultFuture resultFuture) { if (sideInfo.getJoinType() == JoinType.LEFT) { //Reserved left table data try { - Row row = fillData(input, null); + BaseRow row = fillData(input, null); RowDataComplete.completeRow(resultFuture, row); } catch (Exception e) { dealFillDataError(input, resultFuture, e); @@ -158,7 +157,7 @@ protected void dealCacheData(String key, CacheObj missKeyObj) { } @Override - public void timeout(Row input, ResultFuture resultFuture) throws Exception { + public void timeout(BaseRow input, ResultFuture resultFuture) throws Exception { if (timeOutNum % TIMEOUT_LOG_FLUSH_NUM == 0) { LOG.info("Async function call has timed out. input:{}, timeOutNum:{}", input.toString(), timeOutNum); @@ -175,32 +174,32 @@ public void timeout(Row input, ResultFuture resultFuture) throws Except resultFuture.complete(Collections.EMPTY_LIST); } - protected void preInvoke(Row input, ResultFuture resultFuture) + protected void preInvoke(BaseRow input, ResultFuture resultFuture) throws InvocationTargetException, IllegalAccessException { registerTimerAndAddToHandler(input, resultFuture); } @Override - public void asyncInvoke(Row row, ResultFuture resultFuture) throws Exception { - Row input = Row.copy(row); - preInvoke(input, resultFuture); - Map inputParams = parseInputParam(input); + public void asyncInvoke(BaseRow row, ResultFuture resultFuture) throws Exception { + preInvoke(row, resultFuture); + Map inputParams = parseInputParam(row); if (MapUtils.isEmpty(inputParams)) { - dealMissKey(input, resultFuture); + dealMissKey(row, resultFuture); return; } if (isUseCache(inputParams)) { - invokeWithCache(inputParams, input, resultFuture); + invokeWithCache(inputParams, row, resultFuture); return; } - handleAsyncInvoke(inputParams, input, resultFuture); + handleAsyncInvoke(inputParams, row, resultFuture); } - private Map parseInputParam(Row input) { + private Map parseInputParam(BaseRow input) { + GenericRow genericRow = (GenericRow) input; Map inputParams = Maps.newLinkedHashMap(); for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = input.getField(conValIndex); + Object equalObj = genericRow.getField(conValIndex); if (equalObj == null) { return inputParams; } @@ -214,7 +213,7 @@ protected boolean isUseCache(Map inputParams) { return openCache() && getFromCache(buildCacheKey(inputParams)) != null; } - private void invokeWithCache(Map inputParams, Row input, ResultFuture resultFuture) { + private void invokeWithCache(Map inputParams, BaseRow input, ResultFuture resultFuture) { if (openCache()) { CacheObj val = getFromCache(buildCacheKey(inputParams)); if (val != null) { @@ -223,16 +222,16 @@ private void invokeWithCache(Map inputParams, Row input, ResultF return; } else if (ECacheContentType.SingleLine == val.getType()) { try { - Row row = fillData(input, val.getContent()); + BaseRow row = fillData(input, val.getContent()); RowDataComplete.completeRow(resultFuture, row); } catch (Exception e) { dealFillDataError(input, resultFuture, e); } } else if (ECacheContentType.MultiLine == val.getType()) { try { - List rowList = Lists.newArrayList(); + List rowList = Lists.newArrayList(); for (Object one : (List) val.getContent()) { - Row row = fillData(input, one); + BaseRow row = fillData(input, one); rowList.add(row); } RowDataComplete.completeRow(resultFuture,rowList); @@ -247,7 +246,7 @@ private void invokeWithCache(Map inputParams, Row input, ResultF } } - public abstract void handleAsyncInvoke(Map inputParams, Row input, ResultFuture resultFuture) throws Exception; + public abstract void handleAsyncInvoke(Map inputParams, BaseRow input, ResultFuture resultFuture) throws Exception; public abstract String buildCacheKey(Map inputParams); @@ -255,14 +254,14 @@ private ProcessingTimeService getProcessingTimeService() { return ((StreamingRuntimeContext) this.runtimeContext).getProcessingTimeService(); } - protected ScheduledFuture registerTimer(Row input, ResultFuture resultFuture) { + protected ScheduledFuture registerTimer(BaseRow input, ResultFuture resultFuture) { long timeoutTimestamp = sideInfo.getSideTableInfo().getAsyncTimeout() + getProcessingTimeService().getCurrentProcessingTime(); return getProcessingTimeService().registerTimer( timeoutTimestamp, timestamp -> timeout(input, resultFuture)); } - protected void registerTimerAndAddToHandler(Row input, ResultFuture resultFuture) + protected void registerTimerAndAddToHandler(BaseRow input, ResultFuture resultFuture) throws InvocationTargetException, IllegalAccessException { ScheduledFuture timeFuture = registerTimer(input, resultFuture); // resultFuture 是ResultHandler 的实例 @@ -272,7 +271,7 @@ protected void registerTimerAndAddToHandler(Row input, ResultFuture res } - protected void dealFillDataError(Row input, ResultFuture resultFuture, Throwable e) { + protected void dealFillDataError(BaseRow input, ResultFuture resultFuture, Throwable e) { parseErrorRecords.inc(); if (parseErrorRecords.getCount() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)) { LOG.info("dealFillDataError", e); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/ISideReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/ISideReqRow.java index 88066e37f..11a54d329 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/ISideReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/ISideReqRow.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side; -import org.apache.flink.types.Row; +import org.apache.flink.table.dataformat.BaseRow; /** * @@ -28,6 +28,6 @@ */ public interface ISideReqRow { - Row fillData(Row input, Object sideInput); + BaseRow fillData(BaseRow input, Object sideInput); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 6dfc8734e..789677d35 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -28,6 +28,7 @@ import com.dtstack.flink.sql.side.operator.SideWithAllCacheOperator; import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.ParseUtils; +import com.dtstack.flink.sql.util.RowDataConvert; import com.dtstack.flink.sql.util.TableUtils; import com.google.common.base.Preconditions; import com.google.common.collect.HashBasedTable; @@ -49,11 +50,9 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.table.catalog.ObjectIdentifier; +import org.apache.flink.table.dataformat.BaseRow; import org.apache.flink.table.runtime.typeutils.BaseRowTypeInfo; import org.apache.flink.table.runtime.typeutils.BigDecimalTypeInfo; import org.apache.flink.table.runtime.typeutils.LegacyLocalDateTimeTypeInfo; @@ -62,7 +61,6 @@ import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.TimestampType; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,10 +71,8 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Queue; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.calcite.sql.SqlKind.AS; import static org.apache.calcite.sql.SqlKind.INSERT; @@ -528,19 +524,16 @@ private void joinFun(Object pollObj, RowTypeInfo typeInfo = new RowTypeInfo(fieldDataTypes, targetTable.getSchema().getFieldNames()); - DataStream adaptStream = tableEnv.toRetractStream(targetTable, typeInfo) - .filter(f -> f.f0) - .map(f -> f.f1); + DataStream adaptStream = tableEnv.toRetractStream(targetTable, typeInfo) + .map(f -> RowDataConvert.convertToBaseRow(f)); //join side table before keyby ===> Reducing the size of each dimension table cache of async if (sideTableInfo.isPartitionedJoin()) { - List leftJoinColList = getConditionFields(joinInfo.getCondition(), joinInfo.getLeftTableAlias(), sideTableInfo); - List fieldNames = Arrays.asList(targetTable.getSchema().getFieldNames()); - int[] keyIndex = leftJoinColList.stream().mapToInt(fieldNames::indexOf).toArray(); - adaptStream = adaptStream.keyBy(new TupleKeySelector(keyIndex, projectedTypeInfo(keyIndex, targetTable.getSchema()))); + // TODO need change + // adaptStream = adaptStream.keyBy(new TupleKeySelector(leftBaseTypeInfo)); } - DataStream dsOut = null; + DataStream dsOut = null; if(ECacheType.ALL.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ dsOut = SideWithAllCacheOperator.getSideJoinDataStream(adaptStream, sideTableInfo.getType(), localSqlPluginPath, typeInfo, joinInfo, sideJoinFieldInfo, sideTableInfo, pluginLoadMode); }else{ @@ -559,10 +552,6 @@ private void joinFun(Object pollObj, replaceInfo.setTargetTableName(targetTableName); replaceInfo.setTargetTableAlias(targetTableAlias); - ObjectIdentifier objectIdentifier = ObjectIdentifier.of( - EnvironmentSettings.DEFAULT_BUILTIN_CATALOG, - EnvironmentSettings.DEFAULT_BUILTIN_DATABASE, - targetTableName); boolean tableExists = false; for (String table : tableEnv.listTables()) { if (table.equals(targetTableName)) { @@ -578,16 +567,6 @@ private void joinFun(Object pollObj, } } - private TypeInformation projectedTypeInfo(int[] fields, TableSchema schema) { - String[] fieldNames = schema.getFieldNames(); - TypeInformation[] fieldTypes = schema.getFieldTypes(); - - String[] projectedNames = Arrays.stream(fields).mapToObj(i -> fieldNames[i]).toArray(String[]::new); - TypeInformation[] projectedTypes = Arrays.stream(fields).mapToObj(i -> fieldTypes[i]).toArray(TypeInformation[]::new); - return new RowTypeInfo(projectedTypes, projectedNames); - } - - private boolean checkFieldsInfo(CreateTmpTableParser.SqlParserResult result, Table table) { List fieldNames = new LinkedList<>(); String fieldsInfo = result.getFieldsInfoStr(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/TupleKeySelector.java b/core/src/main/java/com/dtstack/flink/sql/side/TupleKeySelector.java index eeeb3dc51..dd7f617ac 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/TupleKeySelector.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/TupleKeySelector.java @@ -20,32 +20,30 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.functions.KeySelector; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.ResultTypeQueryable; -import org.apache.flink.types.Row; +import org.apache.flink.table.dataformat.BaseRow; +import org.apache.flink.table.dataformat.GenericRow; /** * Date: 2020/3/25 * Company: www.dtstack.com * @author maqi */ -public class TupleKeySelector implements ResultTypeQueryable, KeySelector { +public class TupleKeySelector implements ResultTypeQueryable, KeySelector { - private int[] keyFields; - private TypeInformation returnType; + private TypeInformation returnType; - public TupleKeySelector(int[] keyFields, TypeInformation returnType) { - this.keyFields = keyFields; + public TupleKeySelector(TypeInformation returnType) { this.returnType = returnType; } @Override - public Row getKey(Row value) throws Exception { - return Row.project(value, keyFields); + public BaseRow getKey(BaseRow value) throws Exception { + return GenericRow.copyReference((GenericRow)value); } @Override - public TypeInformation getProducedType() { + public TypeInformation getProducedType() { return returnType; } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java index 5d4c2125d..b1f9264e5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java @@ -28,6 +28,7 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.AsyncDataStream; import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.table.dataformat.BaseRow; import java.util.List; import java.util.concurrent.TimeUnit; @@ -60,8 +61,8 @@ private static BaseAsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } - public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, - List outFieldInfoList, AbstractSideTableInfo sideTableInfo, String pluginLoadMode) throws Exception { + public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, + List outFieldInfoList, AbstractSideTableInfo sideTableInfo, String pluginLoadMode) throws Exception { BaseAsyncReqRow asyncDbReq = loadAsyncReq(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo, pluginLoadMode); //TODO How much should be set for the degree of parallelism? Timeout? capacity settings? diff --git a/core/src/main/java/com/dtstack/flink/sql/util/RowDataComplete.java b/core/src/main/java/com/dtstack/flink/sql/util/RowDataComplete.java index 04cba30c6..d2724a886 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/RowDataComplete.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/RowDataComplete.java @@ -19,13 +19,10 @@ package com.dtstack.flink.sql.util; import org.apache.commons.compress.utils.Lists; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import java.util.Collection; import java.util.Collections; import java.util.List; @@ -37,42 +34,22 @@ */ public class RowDataComplete { - public static void completeRow(ResultFuture resultFuture, Row row) { - BaseRow baseRow = RowDataConvert.convertToBaseRow(row); - resultFuture.complete(Collections.singleton(baseRow)); + public static void completeRow(ResultFuture resultFuture, BaseRow row) { + resultFuture.complete(Collections.singleton(row)); } - public static void completeTupleRow(ResultFuture> resultFuture, Tuple2 tupleRow) { - BaseRow baseRow = RowDataConvert.convertToBaseRow(tupleRow.f1); - resultFuture.complete(Collections.singleton(new Tuple2<>(tupleRow.f0, baseRow))); - } - - public static void completeRow(ResultFuture resultFuture, List rowList) { + public static void completeRow(ResultFuture resultFuture, List rowList) { List baseRowList = Lists.newArrayList(); - for (Row row : rowList) { - baseRowList.add(RowDataConvert.convertToBaseRow(row)); + for (BaseRow baseRow : rowList) { + baseRowList.add(baseRow); } resultFuture.complete(baseRowList); } - public static void completeTupleRow(ResultFuture> resultFuture, Collection> tupleRowList) { - List> baseRowList = Lists.newArrayList(); - for (Tuple2 rowTuple : tupleRowList) { - baseRowList.add(new Tuple2<>(rowTuple.f0, RowDataConvert.convertToBaseRow(rowTuple.f1))); - } - resultFuture.complete(baseRowList); - } - - public static void collectRow(Collector out, Row row) { - BaseRow baseRow = RowDataConvert.convertToBaseRow(row); - out.collect(baseRow); - } - - public static void collectTupleRow(Collector> out, Tuple2 tupleRow) { - BaseRow baseRow = RowDataConvert.convertToBaseRow(tupleRow.f1); - out.collect(Tuple2.of(tupleRow.f0, baseRow)); + public static void collectRow(Collector out, BaseRow row) { + out.collect(row); } } \ No newline at end of file diff --git a/core/src/main/java/com/dtstack/flink/sql/util/RowDataConvert.java b/core/src/main/java/com/dtstack/flink/sql/util/RowDataConvert.java index 23770af9d..66d6e2d75 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/RowDataConvert.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/RowDataConvert.java @@ -18,12 +18,14 @@ package com.dtstack.flink.sql.util; +import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.table.dataformat.BaseRow; import org.apache.flink.table.dataformat.BinaryString; import org.apache.flink.table.dataformat.DataFormatConverters; import org.apache.flink.table.dataformat.Decimal; import org.apache.flink.table.dataformat.GenericRow; import org.apache.flink.table.dataformat.SqlTimestamp; +import org.apache.flink.table.dataformat.util.BaseRowUtil; import org.apache.flink.types.Row; import java.math.BigDecimal; @@ -42,7 +44,8 @@ */ public class RowDataConvert { - public static BaseRow convertToBaseRow(Row row) { + public static BaseRow convertToBaseRow(Tuple2 input) { + Row row = input.f1; int length = row.getArity(); GenericRow genericRow = new GenericRow(length); for (int i = 0; i < length; i++) { @@ -88,6 +91,12 @@ public static BaseRow convertToBaseRow(Row row) { } } + if(input.f0){ + BaseRowUtil.setAccumulate(genericRow); + } else { + BaseRowUtil.setRetract(genericRow); + } + return genericRow; } } \ No newline at end of file diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index 34cad9902..5462f2e07 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -30,8 +30,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.table.dataformat.BaseRow; +import org.apache.flink.table.dataformat.GenericRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; import org.apache.flink.util.Collector; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -71,10 +71,11 @@ public Elasticsearch6AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List< } @Override - public void flatMap(Row value, Collector out) throws Exception { + public void flatMap(BaseRow value, Collector out) throws Exception { + GenericRow genericRow = (GenericRow) value; List inputParams = Lists.newArrayList(); for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = value.getField(conValIndex); + Object equalObj = genericRow.getField(conValIndex); if (equalObj == null) { sendOutputRow(value, null, out); return; @@ -96,11 +97,13 @@ public void flatMap(Row value, Collector out) throws Exception { } @Override - public Row fillData(Row input, Object sideInput) { + public BaseRow fillData(BaseRow input, Object sideInput) { + GenericRow genericRow = (GenericRow) input; Map cacheInfo = (Map) sideInput; - Row row = new Row(sideInfo.getOutFieldInfoList().size()); + GenericRow row = new GenericRow(sideInfo.getOutFieldInfoList().size()); + row.setHeader(input.getHeader()); for (Map.Entry entry : sideInfo.getInFieldIndex().entrySet()) { - Object obj = input.getField(entry.getValue()); + Object obj = genericRow.getField(entry.getValue()); boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index 15c0c0b9a..d08e7a55f 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -18,23 +18,27 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; -import com.dtstack.flink.sql.util.RowDataComplete; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; - import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.PredicateInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; +import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; import com.dtstack.flink.sql.util.ParseUtils; +import com.dtstack.flink.sql.util.RowDataComplete; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.dataformat.BaseRow; +import org.apache.flink.table.dataformat.GenericRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -81,7 +85,7 @@ public void open(Configuration parameters) throws Exception { @Override - public void handleAsyncInvoke(Map inputParams, Row input, ResultFuture resultFuture) throws Exception { + public void handleAsyncInvoke(Map inputParams, BaseRow input, ResultFuture resultFuture) throws Exception { String key = buildCacheKey(inputParams); BoolQueryBuilder boolQueryBuilder = Es6Util.setPredicateclause(sideInfo); boolQueryBuilder = setInputParams(inputParams, boolQueryBuilder); @@ -97,7 +101,7 @@ public void handleAsyncInvoke(Map inputParams, Row input, Result public void onResponse(SearchResponse searchResponse) { List cacheContent = Lists.newArrayList(); - List rowList = Lists.newArrayList(); + List rowList = Lists.newArrayList(); SearchHit[] searchHits = searchResponse.getHits().getHits(); if (searchHits.length > 0) { Elasticsearch6SideTableInfo tableInfo = null; @@ -159,7 +163,7 @@ public String buildCacheKey(Map inputParams) { return sb.toString(); } - private void loadDataToCache(SearchHit[] searchHits, List rowList, List cacheContent, Row copyCrow) { + private void loadDataToCache(SearchHit[] searchHits, List rowList, List cacheContent, BaseRow copyCrow) { List results = Lists.newArrayList(); for (SearchHit searchHit : searchHits) { Map object = searchHit.getSourceAsMap(); @@ -168,10 +172,10 @@ private void loadDataToCache(SearchHit[] searchHits, List rowList, List getRows(Row inputRow, List cacheContent, List results) { - List rowList = Lists.newArrayList(); + protected List getRows(BaseRow inputRow, List cacheContent, List results) { + List rowList = Lists.newArrayList(); for (Object line : results) { - Row row = fillData(inputRow, line); + BaseRow row = fillData(inputRow, line); if (null != cacheContent && openCache()) { cacheContent.add(line); } @@ -181,13 +185,14 @@ protected List getRows(Row inputRow, List cacheContent, List cacheInfo = (Map) line; - Row row = new Row(sideInfo.getOutFieldInfoList().size()); + GenericRow row = new GenericRow(sideInfo.getOutFieldInfoList().size()); + row.setHeader(input.getHeader()); String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); for (Map.Entry entry : sideInfo.getInFieldIndex().entrySet()) { - Object obj = input.getField(entry.getValue()); + Object obj = genericRow.getField(entry.getValue()); boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { obj = ((Timestamp) obj).getTime(); diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 1662c83e4..9cc6f2976 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -33,8 +33,8 @@ import org.apache.commons.collections.map.HashedMap; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.table.dataformat.BaseRow; +import org.apache.flink.table.dataformat.GenericRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; import org.apache.flink.util.Collector; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.AuthUtil; @@ -93,11 +93,13 @@ public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List sideInputList = (Map) sideInput; - Row row = new Row(sideInfo.getOutFieldInfoList().size()); + GenericRow row = new GenericRow(sideInfo.getOutFieldInfoList().size()); + row.setHeader(genericRow.getHeader()); for(Map.Entry entry : sideInfo.getInFieldIndex().entrySet()){ - Object obj = input.getField(entry.getValue()); + Object obj = genericRow.getField(entry.getValue()); boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. @@ -142,14 +144,15 @@ protected void reloadCache() { } @Override - public void flatMap(Row input, Collector out) throws Exception { + public void flatMap(BaseRow input, Collector out) throws Exception { + GenericRow genericRow = (GenericRow) input; Map refData = Maps.newHashMap(); for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = input.getField(conValIndex); + Object equalObj = genericRow.getField(conValIndex); if (equalObj == null) { if (sideInfo.getJoinType() == JoinType.LEFT) { - Row data = fillData(input, null); + BaseRow data = fillData(input, null); RowDataComplete.collectRow(out, data); } return; @@ -167,13 +170,13 @@ public void flatMap(Row input, Collector out) throws Exception { for (Map.Entry> entry : cacheRef.get().entrySet()) { if (entry.getKey().startsWith(rowKeyStr)) { cacheList = cacheRef.get().get(entry.getKey()); - Row row = fillData(input, cacheList); + BaseRow row = fillData(input, cacheList); RowDataComplete.collectRow(out, row); } } } else { cacheList = cacheRef.get().get(rowKeyStr); - Row row = fillData(input, cacheList); + BaseRow row = fillData(input, cacheList); RowDataComplete.collectRow(out, row); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index fc7c13cfa..9a2b75808 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -40,7 +40,7 @@ import org.apache.flink.runtime.security.KerberosUtils; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.types.Row; +import org.apache.flink.table.dataformat.GenericRow; import org.apache.hadoop.security.authentication.util.KerberosName; import org.hbase.async.Config; import org.hbase.async.HBaseClient; @@ -173,7 +173,7 @@ private void appendJaasConf(String name, String keytab, String principal) { } @Override - public void handleAsyncInvoke(Map inputParams, Row input, ResultFuture resultFuture) throws Exception { + public void handleAsyncInvoke(Map inputParams, BaseRow input, ResultFuture resultFuture) throws Exception { rowKeyMode.asyncGetData(tableName, buildCacheKey(inputParams), input, resultFuture, sideInfo.getSideCache()); } @@ -183,11 +183,13 @@ public String buildCacheKey(Map inputParams) { } @Override - public Row fillData(Row input, Object sideInput){ + public BaseRow fillData(BaseRow input, Object sideInput){ + GenericRow genericRow = (GenericRow) input; List sideInputList = (List) sideInput; - Row row = new Row(sideInfo.getOutFieldInfoList().size()); + GenericRow row = new GenericRow(sideInfo.getOutFieldInfoList().size()); + row.setHeader(genericRow.getHeader()); for(Map.Entry entry : sideInfo.getInFieldIndex().entrySet()){ - Object obj = input.getField(entry.getValue()); + Object obj = genericRow.getField(entry.getValue()); obj = convertTimeIndictorTypeInfo(entry.getValue(), obj); row.setField(entry.getKey(), obj); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java index 516a5b873..d82ac9a82 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java @@ -27,10 +27,9 @@ import org.apache.calcite.sql.JoinType; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.types.Row; +import org.apache.flink.table.dataformat.GenericRow; import org.hbase.async.HBaseClient; -import java.sql.Timestamp; import java.util.Collections; import java.util.List; import java.util.Map; @@ -74,11 +73,11 @@ public AbstractRowKeyModeDealer(Map colRefType, String[] colName this.sideFieldIndex = sideFieldIndex; } - protected void dealMissKey(Row input, ResultFuture resultFuture) { + protected void dealMissKey(BaseRow input, ResultFuture resultFuture) { if (joinType == JoinType.LEFT) { try { //保留left 表数据 - Row row = fillData(input, null); + BaseRow row = fillData(input, null); RowDataComplete.completeRow(resultFuture, row); } catch (Exception e) { resultFuture.completeExceptionally(e); @@ -88,12 +87,13 @@ protected void dealMissKey(Row input, ResultFuture resultFuture) { } } - protected Row fillData(Row input, Object sideInput){ - + protected BaseRow fillData(BaseRow input, Object sideInput){ + GenericRow genericRow = (GenericRow) input; List sideInputList = (List) sideInput; - Row row = new Row(outFieldInfoList.size()); + GenericRow row = new GenericRow(outFieldInfoList.size()); + row.setHeader(genericRow.getHeader()); for(Map.Entry entry : inFieldIndex.entrySet()){ - Object obj = input.getField(entry.getValue()); + Object obj = genericRow.getField(entry.getValue()); row.setField(entry.getKey(), obj); } @@ -108,6 +108,6 @@ protected Row fillData(Row input, Object sideInput){ return row; } - public abstract void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFuture resultFuture, + public abstract void asyncGetData(String tableName, String rowKeyStr, BaseRow input, ResultFuture resultFuture, AbstractSideCache sideCache); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java index 6b5336dbb..0ab250129 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java @@ -32,8 +32,14 @@ import org.apache.calcite.sql.JoinType; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.types.Row; -import org.hbase.async.*; +import org.hbase.async.BinaryPrefixComparator; +import org.hbase.async.Bytes; +import org.hbase.async.CompareFilter; +import org.hbase.async.HBaseClient; +import org.hbase.async.KeyValue; +import org.hbase.async.RowFilter; +import org.hbase.async.ScanFilter; +import org.hbase.async.Scanner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +66,7 @@ public PreRowKeyModeDealerDealer(Map colRefType, String[] colNam } @Override - public void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFuture resultFuture, + public void asyncGetData(String tableName, String rowKeyStr, BaseRow input, ResultFuture resultFuture, AbstractSideCache sideCache) { Scanner prefixScanner = hBaseClient.newScanner(tableName); ScanFilter scanFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(Bytes.UTF8(rowKeyStr))); @@ -74,7 +80,7 @@ public void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFu } - private String dealOneRow(ArrayList> args, String rowKeyStr, Row input, + private String dealOneRow(ArrayList> args, String rowKeyStr, BaseRow input, ResultFuture resultFuture, AbstractSideCache sideCache) { if(args == null || args.size() == 0){ dealMissKey(input, resultFuture); @@ -84,7 +90,7 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, } List cacheContent = Lists.newArrayList(); - List rowList = Lists.newArrayList(); + List rowList = Lists.newArrayList(); for(List oneRow : args){ try { @@ -112,7 +118,7 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, sideVal.add(val); } - Row row = fillData(input, sideVal); + BaseRow row = fillData(input, sideVal); if (openCache) { cacheContent.add(sideVal); } @@ -140,7 +146,7 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, return ""; } - private String dealFail(Object arg2, Row input, ResultFuture resultFuture){ + private String dealFail(Object arg2, BaseRow input, ResultFuture resultFuture){ LOG.error("record:" + input); LOG.error("get side record exception:" + arg2); resultFuture.complete(Collections.EMPTY_LIST); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index 9c5f7b8fc..16d480c1e 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -32,7 +32,6 @@ import org.apache.calcite.sql.JoinType; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.types.Row; import org.hbase.async.GetRequest; import org.hbase.async.HBaseClient; import org.hbase.async.KeyValue; @@ -62,7 +61,7 @@ public RowKeyEqualModeDealer(Map colRefType, String[] colNames, @Override - public void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFuture resultFuture, + public void asyncGetData(String tableName, String rowKeyStr, BaseRow input, ResultFuture resultFuture, AbstractSideCache sideCache){ //TODO 是否有查询多个col family 和多个col的方法 GetRequest getRequest = new GetRequest(tableName, rowKeyStr); @@ -93,7 +92,7 @@ public void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFu sideVal.add(val); } - Row row = fillData(input, sideVal); + BaseRow row = fillData(input, sideVal); if(openCache){ sideCache.putCache(rowKeyStr, CacheObj.buildCacheObj(ECacheContentType.SingleLine, sideVal)); } diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 2f266f728..18cf279b6 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -18,9 +18,9 @@ package com.dtstack.flink.sql.side.impala; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.util.KrbUtils; @@ -33,7 +33,6 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.types.Row; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +41,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; - import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -150,7 +148,7 @@ public String getUrl() { @Override protected void asyncQueryData(Map inputParams, - Row input, + BaseRow input, ResultFuture resultFuture, SQLClient rdbSqlClient, AtomicLong failCounter, diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 23cdd224c..42aa8a65d 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -1,10 +1,10 @@ package com.dtstack.flink.sql.side.kudu; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; import com.dtstack.flink.sql.util.KrbUtils; @@ -17,7 +17,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.types.Row; +import org.apache.flink.table.dataformat.GenericRow; import org.apache.flink.util.Collector; import org.apache.hadoop.security.UserGroupInformation; import org.apache.kudu.ColumnSchema; @@ -88,10 +88,11 @@ protected void reloadCache() { @Override - public void flatMap(Row input, Collector out) throws Exception { + public void flatMap(BaseRow input, Collector out) throws Exception { + GenericRow genericRow = (GenericRow) input; List inputParams = Lists.newArrayList(); for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = input.getField(conValIndex); + Object equalObj = genericRow.getField(conValIndex); if (equalObj == null) { out.collect(null); } @@ -102,14 +103,14 @@ public void flatMap(Row input, Collector out) throws Exception { List> cacheList = cacheRef.get().get(key); if (CollectionUtils.isEmpty(cacheList)) { if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(input, null); + BaseRow row = fillData(input, null); RowDataComplete.collectRow(out, row); } return; } for (Map one : cacheList) { - Row row = fillData(input, one); + BaseRow row = fillData(input, one); RowDataComplete.collectRow(out, row); } } diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index df1032cd8..03e6646b0 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -21,6 +21,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.dataformat.BaseRow; +import org.apache.flink.table.dataformat.GenericRow; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; import org.apache.hadoop.security.UserGroupInformation; @@ -150,8 +151,9 @@ public AsyncKuduClient run() { } @Override - public void handleAsyncInvoke(Map inputParams, Row input, ResultFuture resultFuture) throws Exception { - Row inputCopy = Row.copy(input); + public void handleAsyncInvoke(Map inputParams, BaseRow input, ResultFuture resultFuture) throws Exception { + GenericRow genericRow = (GenericRow) input; + GenericRow inputCopy = GenericRow.copyReference(genericRow); //scannerBuilder 设置为null重新加载过滤条件,然后connkudu重新赋值 //todo:代码需要优化 scannerBuilder = null; @@ -176,7 +178,7 @@ public void handleAsyncInvoke(Map inputParams, Row input, Result List> cacheContent = Lists.newArrayList(); AsyncKuduScanner asyncKuduScanner = scannerBuilder.build(); - List rowList = Lists.newArrayList(); + List rowList = Lists.newArrayList(); Deferred data = asyncKuduScanner.nextRows(); //从之前的同步修改为调用异步的Callback data.addCallbackDeferring(new GetListRowCB(inputCopy, cacheContent, rowList, asyncKuduScanner, resultFuture, buildCacheKey(inputParams))); @@ -195,11 +197,13 @@ public String buildCacheKey(Map inputParams) { @Override - public Row fillData(Row input, Object sideInput) { + public BaseRow fillData(BaseRow input, Object sideInput) { + GenericRow genericRow = (GenericRow) input; Map cacheInfo = (Map) sideInput; - Row row = new Row(sideInfo.getOutFieldInfoList().size()); + GenericRow row = new GenericRow(sideInfo.getOutFieldInfoList().size()); + row.setHeader(genericRow.getHeader()); for (Map.Entry entry : sideInfo.getInFieldIndex().entrySet()) { - Object obj = input.getField(entry.getValue()); + Object obj = genericRow.getField(entry.getValue()); obj = convertTimeIndictorTypeInfo(entry.getValue(), obj); row.setField(entry.getKey(), obj); } @@ -228,9 +232,9 @@ public void close() throws Exception { } class GetListRowCB implements Callback>, RowResultIterator> { - private Row input; + private BaseRow input; private List> cacheContent; - private List rowList; + private List rowList; private AsyncKuduScanner asyncKuduScanner; private ResultFuture resultFuture; private String key; @@ -239,7 +243,7 @@ class GetListRowCB implements Callback>, RowResultIterator> { public GetListRowCB() { } - GetListRowCB(Row input, List> cacheContent, List rowList, + GetListRowCB(BaseRow input, List> cacheContent, List rowList, AsyncKuduScanner asyncKuduScanner, ResultFuture resultFuture, String key) { this.input = input; this.cacheContent = cacheContent; @@ -260,7 +264,7 @@ public Deferred> call(RowResultIterator results) throws Exception { KuduUtil.setMapValue(columnSchema.getType(), oneRow, sideFieldName, result); } } - Row row = fillData(input, oneRow); + BaseRow row = fillData(input, oneRow); if (openCache()) { cacheContent.add(oneRow); } diff --git a/launcher/src/main/java/org/apache/flink/table/planner/plan/QueryOperationConverter.java b/launcher/src/main/java/org/apache/flink/table/planner/plan/QueryOperationConverter.java new file mode 100644 index 000000000..e942d81b3 --- /dev/null +++ b/launcher/src/main/java/org/apache/flink/table/planner/plan/QueryOperationConverter.java @@ -0,0 +1,652 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.table.api.TableException; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.catalog.CatalogManager; +import org.apache.flink.table.catalog.ConnectorCatalogTable; +import org.apache.flink.table.catalog.FunctionLookup; +import org.apache.flink.table.catalog.ObjectIdentifier; +import org.apache.flink.table.catalog.UnresolvedIdentifier; +import org.apache.flink.table.expressions.CallExpression; +import org.apache.flink.table.expressions.Expression; +import org.apache.flink.table.expressions.ExpressionDefaultVisitor; +import org.apache.flink.table.expressions.FieldReferenceExpression; +import org.apache.flink.table.expressions.ResolvedExpression; +import org.apache.flink.table.expressions.ValueLiteralExpression; +import org.apache.flink.table.expressions.resolver.LookupCallResolver; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.functions.FunctionDefinition; +import org.apache.flink.table.functions.FunctionIdentifier; +import org.apache.flink.table.functions.TableFunction; +import org.apache.flink.table.operations.AggregateQueryOperation; +import org.apache.flink.table.operations.CalculatedQueryOperation; +import org.apache.flink.table.operations.CatalogQueryOperation; +import org.apache.flink.table.operations.DistinctQueryOperation; +import org.apache.flink.table.operations.FilterQueryOperation; +import org.apache.flink.table.operations.JavaDataStreamQueryOperation; +import org.apache.flink.table.operations.JoinQueryOperation; +import org.apache.flink.table.operations.JoinQueryOperation.JoinType; +import org.apache.flink.table.operations.ProjectQueryOperation; +import org.apache.flink.table.operations.QueryOperation; +import org.apache.flink.table.operations.QueryOperationVisitor; +import org.apache.flink.table.operations.ScalaDataStreamQueryOperation; +import org.apache.flink.table.operations.SetQueryOperation; +import org.apache.flink.table.operations.SortQueryOperation; +import org.apache.flink.table.operations.TableSourceQueryOperation; +import org.apache.flink.table.operations.WindowAggregateQueryOperation; +import org.apache.flink.table.operations.WindowAggregateQueryOperation.ResolvedGroupWindow; +import org.apache.flink.table.operations.utils.QueryOperationDefaultVisitor; +import org.apache.flink.table.planner.calcite.FlinkContext; +import org.apache.flink.table.planner.calcite.FlinkRelBuilder; +import org.apache.flink.table.planner.calcite.FlinkTypeFactory; +import org.apache.flink.table.planner.expressions.PlannerProctimeAttribute; +import org.apache.flink.table.planner.expressions.PlannerRowtimeAttribute; +import org.apache.flink.table.planner.expressions.PlannerWindowEnd; +import org.apache.flink.table.planner.expressions.PlannerWindowReference; +import org.apache.flink.table.planner.expressions.PlannerWindowStart; +import org.apache.flink.table.planner.expressions.RexNodeExpression; +import org.apache.flink.table.planner.expressions.SqlAggFunctionVisitor; +import org.apache.flink.table.planner.expressions.converter.ExpressionConverter; +import org.apache.flink.table.planner.functions.utils.TableSqlFunction; +import org.apache.flink.table.planner.operations.DataStreamQueryOperation; +import org.apache.flink.table.planner.operations.PlannerQueryOperation; +import org.apache.flink.table.planner.operations.RichTableSourceQueryOperation; +import org.apache.flink.table.planner.plan.logical.LogicalWindow; +import org.apache.flink.table.planner.plan.logical.SessionGroupWindow; +import org.apache.flink.table.planner.plan.logical.SlidingGroupWindow; +import org.apache.flink.table.planner.plan.logical.TumblingGroupWindow; +import org.apache.flink.table.planner.plan.schema.DataStreamTable; +import org.apache.flink.table.planner.plan.schema.DataStreamTable$; +import org.apache.flink.table.planner.plan.schema.TableSourceTable; +import org.apache.flink.table.planner.plan.schema.TypedFlinkTableFunction; +import org.apache.flink.table.planner.plan.stats.FlinkStatistic; +import org.apache.flink.table.planner.sources.TableSourceUtil; +import org.apache.flink.table.sources.LookupableTableSource; +import org.apache.flink.table.sources.StreamTableSource; +import org.apache.flink.table.sources.TableSource; +import org.apache.flink.table.types.DataType; +import org.apache.flink.util.Preconditions; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.logical.LogicalTableFunctionScan; +import org.apache.calcite.rel.logical.LogicalTableScan; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilder.AggCall; +import org.apache.calcite.tools.RelBuilder.GroupKey; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import scala.Some; + +import static java.util.Arrays.asList; +import static java.util.stream.Collectors.toList; +import static org.apache.flink.table.expressions.ExpressionUtils.extractValue; +import static org.apache.flink.table.expressions.utils.ApiExpressionUtils.isFunctionOfKind; +import static org.apache.flink.table.functions.BuiltInFunctionDefinitions.AS; +import static org.apache.flink.table.functions.FunctionKind.AGGREGATE; +import static org.apache.flink.table.functions.FunctionKind.TABLE_AGGREGATE; +import static org.apache.flink.table.types.utils.TypeConversions.fromDataToLogicalType; +import static org.apache.flink.table.types.utils.TypeConversions.fromLegacyInfoToDataType; + +/** + * Converter from Flink's specific relational representation: {@link QueryOperation} to Calcite's specific relational + * representation: {@link RelNode}. + */ +@Internal +public class QueryOperationConverter extends QueryOperationDefaultVisitor { + + private final FlinkRelBuilder relBuilder; + private final SingleRelVisitor singleRelVisitor = new SingleRelVisitor(); + private final LookupCallResolver callResolver; + private final ExpressionConverter expressionConverter; + private final AggregateVisitor aggregateVisitor = new AggregateVisitor(); + private final TableAggregateVisitor tableAggregateVisitor = new TableAggregateVisitor(); + private final JoinExpressionVisitor joinExpressionVisitor = new JoinExpressionVisitor(); + + public QueryOperationConverter(FlinkRelBuilder relBuilder, FunctionLookup functionCatalog) { + this.relBuilder = relBuilder; + this.callResolver = new LookupCallResolver(functionCatalog); + this.expressionConverter = new ExpressionConverter(relBuilder); + } + + @Override + public RelNode defaultMethod(QueryOperation other) { + other.getChildren().forEach(child -> relBuilder.push(child.accept(this))); + return other.accept(singleRelVisitor); + } + + private class SingleRelVisitor implements QueryOperationVisitor { + + @Override + public RelNode visit(ProjectQueryOperation projection) { + List rexNodes = convertToRexNodes(projection.getProjectList()); + + return relBuilder.project(rexNodes, asList(projection.getTableSchema().getFieldNames()), true).build(); + } + + @Override + public RelNode visit(AggregateQueryOperation aggregate) { + List aggregations = aggregate.getAggregateExpressions() + .stream() + .map(this::getAggCall) + .collect(toList()); + + List groupings = convertToRexNodes(aggregate.getGroupingExpressions()); + GroupKey groupKey = relBuilder.groupKey(groupings); + return relBuilder.aggregate(groupKey, aggregations).build(); + } + + @Override + public RelNode visit(WindowAggregateQueryOperation windowAggregate) { + List aggregations = windowAggregate.getAggregateExpressions() + .stream() + .map(this::getAggCall) + .collect(toList()); + + List groupings = convertToRexNodes(windowAggregate.getGroupingExpressions()); + LogicalWindow logicalWindow = toLogicalWindow(windowAggregate.getGroupWindow()); + PlannerWindowReference windowReference = logicalWindow.aliasAttribute(); + List windowProperties = windowAggregate + .getWindowPropertiesExpressions() + .stream() + .map(expr -> convertToWindowProperty(expr.accept(callResolver), windowReference)) + .collect(toList()); + GroupKey groupKey = relBuilder.groupKey(groupings); + return relBuilder.windowAggregate(logicalWindow, groupKey, windowProperties, aggregations).build(); + } + + private FlinkRelBuilder.PlannerNamedWindowProperty convertToWindowProperty(Expression expression, + PlannerWindowReference windowReference) { + Preconditions.checkArgument(expression instanceof CallExpression, "This should never happened"); + CallExpression aliasExpr = (CallExpression) expression; + Preconditions.checkArgument( + BuiltInFunctionDefinitions.AS == aliasExpr.getFunctionDefinition(), + "This should never happened"); + String name = ((ValueLiteralExpression) aliasExpr.getChildren().get(1)).getValueAs(String.class) + .orElseThrow( + () -> new TableException("Invalid literal.")); + Expression windowPropertyExpr = aliasExpr.getChildren().get(0); + Preconditions.checkArgument(windowPropertyExpr instanceof CallExpression, "This should never happened"); + CallExpression windowPropertyCallExpr = (CallExpression) windowPropertyExpr; + FunctionDefinition fd = windowPropertyCallExpr.getFunctionDefinition(); + if (BuiltInFunctionDefinitions.WINDOW_START == fd) { + return new FlinkRelBuilder.PlannerNamedWindowProperty(name, new PlannerWindowStart(windowReference)); + } else if (BuiltInFunctionDefinitions.WINDOW_END == fd) { + return new FlinkRelBuilder.PlannerNamedWindowProperty(name, new PlannerWindowEnd(windowReference)); + } else if (BuiltInFunctionDefinitions.PROCTIME == fd) { + return new FlinkRelBuilder.PlannerNamedWindowProperty(name, new PlannerProctimeAttribute(windowReference)); + } else if (BuiltInFunctionDefinitions.ROWTIME == fd) { + return new FlinkRelBuilder.PlannerNamedWindowProperty(name, new PlannerRowtimeAttribute(windowReference)); + } else { + throw new TableException("Invalid literal."); + } + } + + /** + * Get the {@link AggCall} correspond to the aggregate or table aggregate expression. + */ + private AggCall getAggCall(Expression aggregateExpression) { + if (isFunctionOfKind(aggregateExpression, TABLE_AGGREGATE)) { + return aggregateExpression.accept(tableAggregateVisitor); + } else { + return aggregateExpression.accept(aggregateVisitor); + } + } + + @Override + public RelNode visit(JoinQueryOperation join) { + final Set corSet; + if (join.isCorrelated()) { + corSet = Collections.singleton(relBuilder.peek().getCluster().createCorrel()); + } else { + corSet = Collections.emptySet(); + } + + return relBuilder.join( + convertJoinType(join.getJoinType()), + join.getCondition().accept(joinExpressionVisitor), + corSet) + .build(); + } + + @Override + public RelNode visit(SetQueryOperation setOperation) { + switch (setOperation.getType()) { + case INTERSECT: + relBuilder.intersect(setOperation.isAll()); + break; + case MINUS: + relBuilder.minus(setOperation.isAll()); + break; + case UNION: + relBuilder.union(setOperation.isAll()); + break; + } + return relBuilder.build(); + } + + @Override + public RelNode visit(FilterQueryOperation filter) { + RexNode rexNode = convertExprToRexNode(filter.getCondition()); + return relBuilder.filter(rexNode).build(); + } + + @Override + public RelNode visit(DistinctQueryOperation distinct) { + return relBuilder.distinct().build(); + } + + @Override + public RelNode visit(SortQueryOperation sort) { + List rexNodes = convertToRexNodes(sort.getOrder()); + return relBuilder.sortLimit(sort.getOffset(), sort.getFetch(), rexNodes) + .build(); + } + + @Override + public RelNode visit(CalculatedQueryOperation calculatedTable) { + DataType resultType = fromLegacyInfoToDataType(calculatedTable.getResultType()); + TableFunction tableFunction = calculatedTable.getTableFunction(); + String[] fieldNames = calculatedTable.getTableSchema().getFieldNames(); + + TypedFlinkTableFunction function = new TypedFlinkTableFunction( + tableFunction, fieldNames, resultType); + + FlinkTypeFactory typeFactory = relBuilder.getTypeFactory(); + + TableSqlFunction sqlFunction = new TableSqlFunction( + FunctionIdentifier.of(tableFunction.functionIdentifier()), + tableFunction.toString(), + tableFunction, + resultType, + typeFactory, + function, + scala.Option.empty()); + + List parameters = convertToRexNodes(calculatedTable.getParameters()); + + return LogicalTableFunctionScan.create( + relBuilder.peek().getCluster(), + Collections.emptyList(), + relBuilder.call(sqlFunction, parameters), + function.getElementType(null), + function.getRowType(typeFactory, null, null), + null); + } + + @Override + public RelNode visit(CatalogQueryOperation catalogTable) { + ObjectIdentifier objectIdentifier = catalogTable.getTableIdentifier(); + return relBuilder.scan( + objectIdentifier.getCatalogName(), + objectIdentifier.getDatabaseName(), + objectIdentifier.getObjectName() + ).build(); + } + + @Override + public RelNode visit(QueryOperation other) { + if (other instanceof PlannerQueryOperation) { + return ((PlannerQueryOperation) other).getCalciteTree(); + } else if (other instanceof DataStreamQueryOperation) { + return convertToDataStreamScan((DataStreamQueryOperation) other); + } else if (other instanceof JavaDataStreamQueryOperation) { + JavaDataStreamQueryOperation dataStreamQueryOperation = (JavaDataStreamQueryOperation) other; + return convertToDataStreamScan( + dataStreamQueryOperation.getDataStream(), + dataStreamQueryOperation.getFieldIndices(), + dataStreamQueryOperation.getTableSchema(), + dataStreamQueryOperation.getIdentifier()); + } else if (other instanceof ScalaDataStreamQueryOperation) { + ScalaDataStreamQueryOperation dataStreamQueryOperation = (ScalaDataStreamQueryOperation) other; + return convertToDataStreamScan( + dataStreamQueryOperation.getDataStream(), + dataStreamQueryOperation.getFieldIndices(), + dataStreamQueryOperation.getTableSchema(), + dataStreamQueryOperation.getIdentifier()); + } + + throw new TableException("Unknown table operation: " + other); + } + + @Override + public RelNode visit(TableSourceQueryOperation tableSourceOperation) { + TableSource tableSource = tableSourceOperation.getTableSource(); + boolean isBatch; + if (tableSource instanceof LookupableTableSource) { + isBatch = tableSourceOperation.isBatch(); + } else if (tableSource instanceof StreamTableSource) { + isBatch = ((StreamTableSource) tableSource).isBounded(); + } else { + throw new TableException(String.format("%s is not supported.", tableSource.getClass().getSimpleName())); + } + + FlinkStatistic statistic; + ObjectIdentifier tableIdentifier; + if (tableSourceOperation instanceof RichTableSourceQueryOperation && + ((RichTableSourceQueryOperation) tableSourceOperation).getIdentifier() != null) { + tableIdentifier = ((RichTableSourceQueryOperation) tableSourceOperation).getIdentifier(); + statistic = ((RichTableSourceQueryOperation) tableSourceOperation).getStatistic(); + } else { + statistic = FlinkStatistic.UNKNOWN(); + // TableSourceScan requires a unique name of a Table for computing a digest. + // We are using the identity hash of the TableSource object. + String refId = "Unregistered_TableSource_" + System.identityHashCode(tableSource); + CatalogManager catalogManager = relBuilder.getCluster().getPlanner().getContext() + .unwrap(FlinkContext.class).getCatalogManager(); + tableIdentifier = catalogManager.qualifyIdentifier(UnresolvedIdentifier.of(refId)); + } + + RelDataType rowType = TableSourceUtil.getSourceRowType(relBuilder.getTypeFactory(), + tableSourceOperation.getTableSchema(), + scala.Option.apply(tableSource), + !isBatch); + TableSourceTable tableSourceTable = new TableSourceTable<>( + relBuilder.getRelOptSchema(), + tableIdentifier, + rowType, + statistic, + tableSource, + !isBatch, + ConnectorCatalogTable.source(tableSource, isBatch)); + return LogicalTableScan.create(relBuilder.getCluster(), tableSourceTable); + } + + private RelNode convertToDataStreamScan(DataStreamQueryOperation operation) { + List names; + ObjectIdentifier identifier = operation.getIdentifier(); + if (identifier != null) { + names = Arrays.asList( + identifier.getCatalogName(), + identifier.getDatabaseName(), + identifier.getObjectName()); + } else { + String refId = String.format("Unregistered_DataStream_%s", operation.getDataStream().getId()); + names = Collections.singletonList(refId); + } + + final RelDataType rowType = DataStreamTable$.MODULE$ + .getRowType(relBuilder.getTypeFactory(), + operation.getDataStream(), + operation.getTableSchema().getFieldNames(), + operation.getFieldIndices(), + scala.Option.apply(operation.getFieldNullables())); + DataStreamTable dataStreamTable = new DataStreamTable<>( + relBuilder.getRelOptSchema(), + names, + rowType, + operation.getDataStream(), + operation.isProducesUpdates(), + operation.isAccRetract(), + operation.getFieldIndices(), + operation.getTableSchema().getFieldNames(), + operation.getStatistic(), + scala.Option.apply(operation.getFieldNullables())); + return LogicalTableScan.create(relBuilder.getCluster(), dataStreamTable); + } + + private RelNode convertToDataStreamScan( + DataStream dataStream, + int[] fieldIndices, + TableSchema tableSchema, + Optional identifier) { + List names; + if (identifier.isPresent()) { + names = Arrays.asList( + identifier.get().getCatalogName(), + identifier.get().getDatabaseName(), + identifier.get().getObjectName()); + } else { + String refId = String.format("Unregistered_DataStream_%s", dataStream.getId()); + names = Collections.singletonList(refId); + } + final RelDataType rowType = DataStreamTable$.MODULE$ + .getRowType(relBuilder.getTypeFactory(), + dataStream, + tableSchema.getFieldNames(), + fieldIndices, + scala.Option.empty()); + DataStreamTable dataStreamTable = new DataStreamTable<>( + relBuilder.getRelOptSchema(), + names, + rowType, + dataStream, + false, + true, + fieldIndices, + tableSchema.getFieldNames(), + FlinkStatistic.UNKNOWN(), + scala.Option.empty()); + return LogicalTableScan.create(relBuilder.getCluster(), dataStreamTable); + } + + private List convertToRexNodes(List expressions) { + return expressions + .stream() + .map(QueryOperationConverter.this::convertExprToRexNode) + .collect(toList()); + } + + private LogicalWindow toLogicalWindow(ResolvedGroupWindow window) { + DataType windowType = window.getTimeAttribute().getOutputDataType(); + PlannerWindowReference windowReference = new PlannerWindowReference(window.getAlias(), + new Some<>(fromDataToLogicalType(windowType))); + switch (window.getType()) { + case SLIDE: + return new SlidingGroupWindow( + windowReference, + window.getTimeAttribute(), + window.getSize().orElseThrow(() -> new TableException("missed size parameters!")), + window.getSlide().orElseThrow(() -> new TableException("missed slide parameters!")) + ); + case SESSION: + return new SessionGroupWindow( + windowReference, + window.getTimeAttribute(), + window.getGap().orElseThrow(() -> new TableException("missed gap parameters!")) + ); + case TUMBLE: + return new TumblingGroupWindow( + windowReference, + window.getTimeAttribute(), + window.getSize().orElseThrow(() -> new TableException("missed size parameters!")) + ); + default: + throw new TableException("Unknown window type"); + } + } + + private JoinRelType convertJoinType(JoinType joinType) { + switch (joinType) { + case INNER: + return JoinRelType.INNER; + case LEFT_OUTER: + return JoinRelType.LEFT; + case RIGHT_OUTER: + return JoinRelType.RIGHT; + case FULL_OUTER: + return JoinRelType.FULL; + default: + throw new TableException("Unknown join type: " + joinType); + } + } + } + + private class JoinExpressionVisitor extends ExpressionDefaultVisitor { + + private static final int numberOfJoinInputs = 2; + + @Override + public RexNode visit(CallExpression callExpression) { + final List newChildren = callExpression.getChildren().stream().map(expr -> { + RexNode convertedNode = expr.accept(this); + return new RexNodeExpression(convertedNode, ((ResolvedExpression) expr).getOutputDataType()); + }).collect(Collectors.toList()); + + CallExpression newCall; + if (callExpression.getFunctionIdentifier().isPresent()) { + newCall = new CallExpression( + callExpression.getFunctionIdentifier().get(), callExpression.getFunctionDefinition(), newChildren, + callExpression.getOutputDataType()); + } else { + newCall = new CallExpression( + callExpression.getFunctionDefinition(), newChildren, callExpression.getOutputDataType()); + } + return convertExprToRexNode(newCall); + } + + @Override + public RexNode visit(FieldReferenceExpression fieldReference) { + return relBuilder.field(numberOfJoinInputs, fieldReference.getInputIndex(), fieldReference.getFieldIndex()); + } + + @Override + protected RexNode defaultMethod(Expression expression) { + return convertExprToRexNode(expression); + } + } + + private class AggregateVisitor extends ExpressionDefaultVisitor { + + @Override + public AggCall visit(CallExpression unresolvedCall) { + if (unresolvedCall.getFunctionDefinition() == AS) { + String aggregateName = extractValue(unresolvedCall.getChildren().get(1), String.class) + .orElseThrow(() -> new TableException("Unexpected name.")); + + Expression aggregate = unresolvedCall.getChildren().get(0); + if (isFunctionOfKind(aggregate, AGGREGATE)) { + return aggregate.accept(callResolver).accept( + new AggCallVisitor(relBuilder, expressionConverter, aggregateName, false)); + } + } + throw new TableException("Expected named aggregate. Got: " + unresolvedCall); + } + + @Override + protected AggCall defaultMethod(Expression expression) { + throw new TableException("Unexpected expression: " + expression); + } + + private class AggCallVisitor extends ExpressionDefaultVisitor { + + private final RelBuilder relBuilder; + private final SqlAggFunctionVisitor sqlAggFunctionVisitor; + private final ExpressionConverter expressionConverter; + private final String name; + private final boolean isDistinct; + + public AggCallVisitor(RelBuilder relBuilder, ExpressionConverter expressionConverter, String name, + boolean isDistinct) { + this.relBuilder = relBuilder; + this.sqlAggFunctionVisitor = new SqlAggFunctionVisitor((FlinkTypeFactory) relBuilder.getTypeFactory()); + this.expressionConverter = expressionConverter; + this.name = name; + this.isDistinct = isDistinct; + } + + @Override + public RelBuilder.AggCall visit(CallExpression call) { + FunctionDefinition def = call.getFunctionDefinition(); + if (BuiltInFunctionDefinitions.DISTINCT == def) { + Expression innerAgg = call.getChildren().get(0); + return innerAgg.accept(new AggCallVisitor(relBuilder, expressionConverter, name, true)); + } else { + SqlAggFunction sqlAggFunction = call.accept(sqlAggFunctionVisitor); + return relBuilder.aggregateCall( + sqlAggFunction, + isDistinct, + false, + null, + name, + call.getChildren().stream().map(expr -> expr.accept(expressionConverter)) + .collect(Collectors.toList())); + } + } + + @Override + protected RelBuilder.AggCall defaultMethod(Expression expression) { + throw new TableException("Unexpected expression: " + expression); + } + } + } + + private class TableAggregateVisitor extends ExpressionDefaultVisitor { + @Override + public AggCall visit(CallExpression call) { + if (isFunctionOfKind(call, TABLE_AGGREGATE)) { + return call.accept(new TableAggCallVisitor(relBuilder, expressionConverter)); + } + return defaultMethod(call); + } + + @Override + protected AggCall defaultMethod(Expression expression) { + throw new TableException("Expected table aggregate. Got: " + expression); + } + + private class TableAggCallVisitor extends ExpressionDefaultVisitor { + + private final RelBuilder relBuilder; + private final SqlAggFunctionVisitor sqlAggFunctionVisitor; + private final ExpressionConverter expressionConverter; + + public TableAggCallVisitor(RelBuilder relBuilder, ExpressionConverter expressionConverter) { + this.relBuilder = relBuilder; + this.sqlAggFunctionVisitor = new SqlAggFunctionVisitor((FlinkTypeFactory) relBuilder.getTypeFactory()); + this.expressionConverter = expressionConverter; + } + + @Override + public RelBuilder.AggCall visit(CallExpression call) { + SqlAggFunction sqlAggFunction = call.accept(sqlAggFunctionVisitor); + return relBuilder.aggregateCall( + sqlAggFunction, + false, + false, + null, + sqlAggFunction.toString(), + call.getChildren().stream().map(expr -> expr.accept(expressionConverter)).collect(toList())); + } + + @Override + protected RelBuilder.AggCall defaultMethod(Expression expression) { + throw new TableException("Expected table aggregate. Got: " + expression); + } + } + } + + private RexNode convertExprToRexNode(Expression expr) { + return expr.accept(callResolver).accept(expressionConverter); + } +} diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index 5e5cf0c79..1522f2e34 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -18,10 +18,10 @@ package com.dtstack.flink.sql.side.mongo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.dtstack.flink.sql.util.RowDataComplete; @@ -39,7 +39,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.types.Row; +import org.apache.flink.table.dataformat.GenericRow; import org.apache.flink.util.Collector; import org.bson.Document; import org.slf4j.Logger; @@ -99,13 +99,14 @@ protected void reloadCache() { } @Override - public void flatMap(Row input, Collector out) throws Exception { + public void flatMap(BaseRow input, Collector out) throws Exception { + GenericRow genericRow = (GenericRow) input; List inputParams = Lists.newArrayList(); for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = input.getField(conValIndex); + Object equalObj = genericRow.getField(conValIndex); if (equalObj == null) { if (sideInfo.getJoinType() == JoinType.LEFT) { - Row data = fillData(input, null); + BaseRow data = fillData(input, null); RowDataComplete.collectRow(out, data); } return; @@ -117,7 +118,7 @@ public void flatMap(Row input, Collector out) throws Exception { List> cacheList = cacheRef.get().get(key); if (CollectionUtils.isEmpty(cacheList)) { if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(input, null); + BaseRow row = fillData(input, null); RowDataComplete.collectRow(out, row); } else { return; @@ -127,7 +128,7 @@ public void flatMap(Row input, Collector out) throws Exception { } for (Map one : cacheList) { - Row row = fillData(input, one); + BaseRow row = fillData(input, one); RowDataComplete.collectRow(out, row); } } diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 311f58dc0..9fa8cd689 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -43,7 +43,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.types.Row; +import org.apache.flink.table.dataformat.GenericRow; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -92,8 +92,9 @@ public void connMongoDb() throws Exception { } @Override - public void handleAsyncInvoke(Map inputParams, Row input, ResultFuture resultFuture) throws Exception { - Row inputCopy = Row.copy(input); + public void handleAsyncInvoke(Map inputParams, BaseRow input, ResultFuture resultFuture) throws Exception { + GenericRow genericRow = (GenericRow) input; + GenericRow inputCopy = GenericRow.copyReference(genericRow); BasicDBObject basicDbObject = new BasicDBObject(); try { basicDbObject.putAll(inputParams); @@ -118,7 +119,7 @@ public void handleAsyncInvoke(Map inputParams, Row input, Result @Override public void apply(final Document document) { atomicInteger.incrementAndGet(); - Row row = fillData(inputCopy, document); + BaseRow row = fillData(inputCopy, document); if (openCache()) { cacheContent.add(document); } @@ -153,11 +154,13 @@ public String buildCacheKey(Map inputParams) { } @Override - public Row fillData(Row input, Object line) { + public BaseRow fillData(BaseRow input, Object line) { + GenericRow genericRow = (GenericRow) input; Document doc = (Document) line; - Row row = new Row(sideInfo.getOutFieldInfoList().size()); + GenericRow row = new GenericRow(sideInfo.getOutFieldInfoList().size()); + row.setHeader(genericRow.getHeader()); for (Map.Entry entry : sideInfo.getInFieldIndex().entrySet()) { - Object obj = input.getField(entry.getValue()); + Object obj = genericRow.getField(entry.getValue()); obj = convertTimeIndictorTypeInfo(entry.getValue(), obj); row.setField(entry.getKey(), obj); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index d0aa93927..ef0133cc3 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -24,7 +24,6 @@ import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import com.dtstack.flink.sql.util.JDBCUtils; import com.dtstack.flink.sql.util.RowDataComplete; -import com.dtstack.flink.sql.util.RowDataConvert; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; @@ -33,8 +32,8 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.configuration.Configuration; import org.apache.flink.table.dataformat.BaseRow; +import org.apache.flink.table.dataformat.GenericRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; import org.apache.flink.util.Collector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -104,14 +103,15 @@ protected void reloadCache() { } @Override - public void flatMap(Row value, Collector out) throws Exception { + public void flatMap(BaseRow value, Collector out) throws Exception { + GenericRow genericRow = (GenericRow) value; List equalValIndex = sideInfo.getEqualValIndex(); ArrayList inputParams = equalValIndex.stream() - .map(value::getField) + .map(genericRow::getField) .collect(Collectors.toCollection(ArrayList::new)); if (inputParams.size() != equalValIndex.size() && sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value, null); + BaseRow row = fillData(value, null); RowDataComplete.collectRow(out, row); return; } @@ -122,10 +122,10 @@ public void flatMap(Row value, Collector out) throws Exception { List> cacheList = cacheRef.get().get(cacheKey); if (CollectionUtils.isEmpty(cacheList) && sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value, null); + BaseRow row = fillData(value, null); RowDataComplete.collectRow(out, row); } else if (!CollectionUtils.isEmpty(cacheList)) { - cacheList.forEach(one -> out.collect(RowDataConvert.convertToBaseRow(fillData(value, one)))); + cacheList.forEach(one -> out.collect(fillData(value, one))); } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 4fed5f296..f0d8b4a3f 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -39,18 +39,15 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.types.Row; -import org.apache.hadoop.security.UserGroupInformation; +import org.apache.flink.table.dataformat.GenericRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.math.BigDecimal; -import java.security.PrivilegedAction; import java.sql.Timestamp; import java.time.Instant; import java.util.List; import java.util.Map; -import java.util.TimeZone; import java.util.concurrent.CountDownLatch; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -116,10 +113,10 @@ protected void init(BaseSideInfo sideInfo) { } @Override - protected void preInvoke(Row input, ResultFuture resultFuture) { } + protected void preInvoke(BaseRow input, ResultFuture resultFuture) { } @Override - public void handleAsyncInvoke(Map inputParams, Row input, ResultFuture resultFuture) throws Exception { + public void handleAsyncInvoke(Map inputParams, BaseRow input, ResultFuture resultFuture) throws Exception { AtomicLong networkLogCounter = new AtomicLong(0L); //network is unhealthy while (!connectionStatus.get()) { @@ -132,8 +129,8 @@ public void handleAsyncInvoke(Map inputParams, Row input, Result executor.execute(() -> connectWithRetry(params, input, resultFuture, rdbSqlClient)); } - protected void asyncQueryData( Map inputParams, - Row input, + protected void asyncQueryData(Map inputParams, + BaseRow input, ResultFuture resultFuture, SQLClient rdbSqlClient, AtomicLong failCounter, @@ -149,7 +146,7 @@ protected void asyncQueryData( Map inputParams, final protected void doAsyncQueryData( Map inputParams, - Row input, + BaseRow input, ResultFuture resultFuture, SQLClient rdbSqlClient, AtomicLong failCounter, @@ -181,7 +178,7 @@ final protected void doAsyncQueryData( }); } - private void connectWithRetry(Map inputParams, Row input, ResultFuture resultFuture, SQLClient rdbSqlClient) { + private void connectWithRetry(Map inputParams, BaseRow input, ResultFuture resultFuture, SQLClient rdbSqlClient) { AtomicLong failCounter = new AtomicLong(0); AtomicBoolean finishFlag = new AtomicBoolean(false); while (!finishFlag.get()) { @@ -255,11 +252,13 @@ public String buildCacheKey(Map inputParam) { } @Override - public Row fillData(Row input, Object line) { + public BaseRow fillData(BaseRow input, Object line) { + GenericRow genericRow = (GenericRow) input; JsonArray jsonArray = (JsonArray) line; - Row row = new Row(sideInfo.getOutFieldInfoList().size()); + GenericRow row = new GenericRow(sideInfo.getOutFieldInfoList().size()); + row.setHeader(input.getHeader()); for (Map.Entry entry : sideInfo.getInFieldIndex().entrySet()) { - Object obj = input.getField(entry.getValue()); + Object obj = genericRow.getField(entry.getValue()); obj = convertTimeIndictorTypeInfo(entry.getValue(), obj); row.setField(entry.getKey(), obj); } @@ -295,7 +294,7 @@ public void setRdbSqlClient(SQLClient rdbSqlClient) { this.rdbSqlClient = rdbSqlClient; } - private void handleQuery(SQLConnection connection, Map inputParams, Row input, ResultFuture resultFuture) { + private void handleQuery(SQLConnection connection, Map inputParams, BaseRow input, ResultFuture resultFuture) { String key = buildCacheKey(inputParams); JsonArray params = new JsonArray(Lists.newArrayList(inputParams.values())); connection.queryWithParams(sideInfo.getSqlCondition(), params, rs -> { @@ -308,10 +307,10 @@ private void handleQuery(SQLConnection connection, Map inputPara int resultSize = rs.result().getResults().size(); if (resultSize > 0) { - List rowList = Lists.newArrayList(); + List rowList = Lists.newArrayList(); for (JsonArray line : rs.result().getResults()) { - Row row = fillData(input, line); + BaseRow row = fillData(input, line); if (openCache()) { cacheContent.add(line); } diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 246417879..01aa46a2d 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -34,7 +34,7 @@ import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.types.Row; +import org.apache.flink.table.dataformat.GenericRow; import org.apache.flink.util.Collector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -82,7 +82,7 @@ public RedisAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List out) throws Exception { + public void flatMap(BaseRow input, Collector out) throws Exception { + GenericRow genericRow = (GenericRow) input; Map inputParams = Maps.newHashMap(); for(Integer conValIndex : sideInfo.getEqualValIndex()){ - Object equalObj = input.getField(conValIndex); + Object equalObj = genericRow.getField(conValIndex); if(equalObj == null){ if (sideInfo.getJoinType() == JoinType.LEFT) { - Row data = fillData(input, null); + BaseRow data = fillData(input, null); RowDataComplete.collectRow(out, data); } return; @@ -129,7 +130,7 @@ public void flatMap(Row input, Collector out) throws Exception { if (cacheMap == null){ if(sideInfo.getJoinType() == JoinType.LEFT){ - Row data = fillData(input, null); + BaseRow data = fillData(input, null); RowDataComplete.collectRow(out, data); }else{ return; @@ -138,7 +139,7 @@ public void flatMap(Row input, Collector out) throws Exception { return; } - Row newRow = fillData(input, cacheMap); + BaseRow newRow = fillData(input, cacheMap); RowDataComplete.collectRow(out, newRow); } diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index b8badbd9e..395e44b1a 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -18,19 +18,9 @@ package com.dtstack.flink.sql.side.redis; +import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.BaseAsyncReqRow; -import com.dtstack.flink.sql.util.RowDataComplete; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.async.RedisAsyncCommands; -import io.lettuce.core.cluster.api.async.RedisAdvancedClusterAsyncCommands; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.types.Row; - -import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; @@ -38,8 +28,10 @@ import com.dtstack.flink.sql.side.redis.enums.RedisType; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; +import com.dtstack.flink.sql.util.RowDataComplete; import io.lettuce.core.RedisClient; import io.lettuce.core.RedisFuture; +import io.lettuce.core.RedisURI; import io.lettuce.core.api.StatefulRedisConnection; import io.lettuce.core.api.async.RedisHashAsyncCommands; import io.lettuce.core.api.async.RedisKeyAsyncCommands; @@ -47,6 +39,10 @@ import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.dataformat.BaseRow; import java.util.List; import java.util.Map; @@ -122,12 +118,12 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ } @Override - public Row fillData(Row input, Object sideInput) { + public BaseRow fillData(BaseRow input, Object sideInput) { return redisSideReqRow.fillData(input, sideInput); } @Override - public void handleAsyncInvoke(Map inputParams, Row input, ResultFuture resultFuture) throws Exception { + public void handleAsyncInvoke(Map inputParams, BaseRow input, ResultFuture resultFuture) throws Exception { String key = buildCacheKey(inputParams); if(StringUtils.isBlank(key)){ return; @@ -138,7 +134,7 @@ public void handleAsyncInvoke(Map inputParams, Row input, Result public void accept(Map values) { if (MapUtils.isNotEmpty(values)) { try { - Row row = fillData(input, values); + BaseRow row = fillData(input, values); dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.SingleLine, row)); RowDataComplete.completeRow(resultFuture, row); } catch (Exception e) { diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java index e57cb5bff..d874d6dff 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java @@ -18,11 +18,12 @@ package com.dtstack.flink.sql.side.redis.table; -import com.dtstack.flink.sql.side.ISideReqRow; import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.ISideReqRow; import com.dtstack.flink.sql.util.TableUtils; +import org.apache.flink.table.dataformat.BaseRow; +import org.apache.flink.table.dataformat.GenericRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; import java.io.Serializable; import java.math.BigDecimal; @@ -53,11 +54,13 @@ public RedisSideReqRow(BaseSideInfo sideInfo, RedisSideTableInfo sideTableInfo) } @Override - public Row fillData(Row input, Object sideInput) { + public BaseRow fillData(BaseRow input, Object sideInput) { + GenericRow genericRow = (GenericRow) input; Map sideInputMap = (Map) sideInput; - Row row = new Row(sideInfo.getOutFieldInfoList().size()); + GenericRow row = new GenericRow(sideInfo.getOutFieldInfoList().size()); + row.setHeader(input.getHeader()); for(Map.Entry entry : sideInfo.getInFieldIndex().entrySet()){ - Object obj = input.getField(entry.getValue()); + Object obj = genericRow.getField(entry.getValue()); boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); if(obj instanceof LocalDateTime && isTimeIndicatorTypeInfo){ @@ -91,7 +94,7 @@ public String buildCacheKey(Map refData) { return keyBuilder.toString(); } - public void setRowField(Row row, Integer index, BaseSideInfo sideInfo, String value) { + public void setRowField(GenericRow row, Integer index, BaseSideInfo sideInfo, String value) { Integer keyIndex = sideInfo.getSideFieldIndex().get(index); String classType = sideInfo.getSideTableInfo().getFieldClassList().get(keyIndex).getName(); switch (classType){ From d0b712dedddac8c3d00ba5d5aee6a94768ceb515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Wed, 30 Dec 2020 16:19:31 +0800 Subject: [PATCH 36/80] =?UTF-8?q?[opt]=20=E8=B0=83=E6=95=B4=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java index fc7c79bd0..7a21e16cc 100644 --- a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java +++ b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java @@ -144,8 +144,13 @@ public static Connection getConnectWithRetry( return Objects.isNull(userName) ? DriverManager.getConnection(url) : DriverManager.getConnection(url, userName, password); } catch (Exception e) { - errorCause = e.getCause().toString(); - LOG.warn(errorMessage + e.getCause()); + if (Objects.isNull(e.getCause())) { + errorCause = e.getMessage(); + } else { + errorCause = e.getCause().toString(); + } + + LOG.warn(errorMessage + errorCause); LOG.warn("Connect will retry after [{}] s. Retry time [{}] ...", DEFAULT_RETRY_TIME_WAIT, i + 1); ThreadUtil.sleepSeconds(DEFAULT_RETRY_TIME_WAIT); } From c024153dc1c6614382cf17a592c1867405a15c59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Wed, 30 Dec 2020 17:40:29 +0800 Subject: [PATCH 37/80] =?UTF-8?q?[fix]=20=E8=A7=A3=E5=86=B3=E5=90=88?= =?UTF-8?q?=E5=B9=B6=E4=BB=A3=E7=A0=81=E7=9A=84=E9=97=AE=E9=A2=98=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java | 1 - 1 file changed, 1 deletion(-) diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 69221b4be..085ef7613 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -69,7 +69,6 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Wed, 30 Dec 2020 17:49:51 +0800 Subject: [PATCH 38/80] =?UTF-8?q?[hotfix-32959][core][launcher]=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E5=8F=8C=E5=B1=82group=20by=E7=BC=BA=E5=B0=91?= =?UTF-8?q?=E6=92=A4=E5=9B=9E=E6=95=B0=E6=8D=AE=EF=BC=8C=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E7=BB=93=E7=AE=97=E7=BB=93=E6=9E=9C=E9=94=99=E8=AF=AF=EF=BC=8C?= =?UTF-8?q?=E4=B8=BB=E8=A6=81=E6=98=AF=E5=B0=86=E7=BB=B4=E8=A1=A8=E4=B8=AD?= =?UTF-8?q?=E7=9A=84row=E7=B1=BB=E5=9E=8B=E6=94=B9=E6=88=90baserow?= =?UTF-8?q?=EF=BC=8C=E5=B0=86baserow=E5=86=8D=E6=AC=A1=E8=BF=9B=E8=A1=8C?= =?UTF-8?q?=E8=BD=AC=E6=8D=A2=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../side/cassandra/CassandraAllReqRow.java | 6 +-- .../side/cassandra/CassandraAsyncReqRow.java | 2 +- .../dtstack/flink/sql/side/BaseAllReqRow.java | 2 +- .../flink/sql/side/BaseAsyncReqRow.java | 6 +-- .../flink/sql/util/RowDataComplete.java | 15 +++--- .../flink/sql/util/RowDataConvert.java | 53 ++++++++++++++++++- .../Elasticsearch6AsyncReqRow.java | 2 +- .../flink/sql/side/hbase/HbaseAllReqRow.java | 6 +-- .../AbstractRowKeyModeDealer.java | 2 +- .../PreRowKeyModeDealerDealer.java | 2 +- .../rowkeydealer/RowKeyEqualModeDealer.java | 2 +- .../flink/sql/side/kudu/KuduAllReqRow.java | 4 +- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 2 +- .../flink/sql/side/mongo/MongoAllReqRow.java | 6 +-- .../sql/side/mongo/MongoAsyncReqRow.java | 2 +- .../side/rdb/all/AbstractRdbAllReqRow.java | 6 +-- .../sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- .../flink/sql/side/redis/RedisAllReqRow.java | 6 +-- .../sql/side/redis/RedisAsyncReqRow.java | 2 +- 19 files changed, 88 insertions(+), 40 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index 877fa1ec2..28830135a 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -110,7 +110,7 @@ public void flatMap(BaseRow input, Collector out) throws Exception { if (equalObj == null) { if (sideInfo.getJoinType() == JoinType.LEFT) { BaseRow row = fillData(input, null); - RowDataComplete.collectRow(out, row); + RowDataComplete.collectBaseRow(out, row); } return; } @@ -123,7 +123,7 @@ public void flatMap(BaseRow input, Collector out) throws Exception { if (CollectionUtils.isEmpty(cacheList)) { if (sideInfo.getJoinType() == JoinType.LEFT) { BaseRow row = fillData(input, null); - RowDataComplete.collectRow(out, row); + RowDataComplete.collectBaseRow(out, row); } else { return; } @@ -133,7 +133,7 @@ public void flatMap(BaseRow input, Collector out) throws Exception { for (Map one : cacheList) { BaseRow row = fillData(input, one); - RowDataComplete.collectRow(out, row); + RowDataComplete.collectBaseRow(out, row); } } diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index a44bece33..22e961a2f 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -199,7 +199,7 @@ public void onSuccess(List rows) { } rowList.add(row); } - RowDataComplete.completeRow(resultFuture, rowList); + RowDataComplete.completeBaseRow(resultFuture, rowList); if (openCache()) { putCache(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java index d09caaa16..c35c57bc8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java @@ -98,7 +98,7 @@ protected void sendOutputRow(BaseRow value, Object sideInput, Collector return; } BaseRow row = fillData(value, sideInput); - RowDataComplete.collectRow(out, row); + RowDataComplete.collectBaseRow(out, row); } @Override diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index d2380cfb3..c4c53708d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -141,7 +141,7 @@ protected void dealMissKey(BaseRow input, ResultFuture resultFuture) { //Reserved left table data try { BaseRow row = fillData(input, null); - RowDataComplete.completeRow(resultFuture, row); + RowDataComplete.completeBaseRow(resultFuture, row); } catch (Exception e) { dealFillDataError(input, resultFuture, e); } @@ -223,7 +223,7 @@ private void invokeWithCache(Map inputParams, BaseRow input, Res } else if (ECacheContentType.SingleLine == val.getType()) { try { BaseRow row = fillData(input, val.getContent()); - RowDataComplete.completeRow(resultFuture, row); + RowDataComplete.completeBaseRow(resultFuture, row); } catch (Exception e) { dealFillDataError(input, resultFuture, e); } @@ -234,7 +234,7 @@ private void invokeWithCache(Map inputParams, BaseRow input, Res BaseRow row = fillData(input, one); rowList.add(row); } - RowDataComplete.completeRow(resultFuture,rowList); + RowDataComplete.completeBaseRow(resultFuture,rowList); } catch (Exception e) { dealFillDataError(input, resultFuture, e); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/RowDataComplete.java b/core/src/main/java/com/dtstack/flink/sql/util/RowDataComplete.java index d2724a886..ac3886401 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/RowDataComplete.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/RowDataComplete.java @@ -34,22 +34,19 @@ */ public class RowDataComplete { - public static void completeRow(ResultFuture resultFuture, BaseRow row) { - resultFuture.complete(Collections.singleton(row)); + public static void completeBaseRow(ResultFuture resultFuture, BaseRow row) { + resultFuture.complete(Collections.singleton(RowDataConvert.convertToBaseRow(row))); } - public static void completeRow(ResultFuture resultFuture, List rowList) { - + public static void completeBaseRow(ResultFuture resultFuture, List rowList) { List baseRowList = Lists.newArrayList(); for (BaseRow baseRow : rowList) { - baseRowList.add(baseRow); + baseRowList.add(RowDataConvert.convertToBaseRow(baseRow)); } - resultFuture.complete(baseRowList); } - public static void collectRow(Collector out, BaseRow row) { - out.collect(row); + public static void collectBaseRow(Collector out, BaseRow row) { + out.collect(RowDataConvert.convertToBaseRow(row)); } - } \ No newline at end of file diff --git a/core/src/main/java/com/dtstack/flink/sql/util/RowDataConvert.java b/core/src/main/java/com/dtstack/flink/sql/util/RowDataConvert.java index 66d6e2d75..6da8937ae 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/RowDataConvert.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/RowDataConvert.java @@ -91,7 +91,7 @@ public static BaseRow convertToBaseRow(Tuple2 input) { } } - if(input.f0){ + if (input.f0) { BaseRowUtil.setAccumulate(genericRow); } else { BaseRowUtil.setRetract(genericRow); @@ -99,4 +99,55 @@ public static BaseRow convertToBaseRow(Tuple2 input) { return genericRow; } + + + public static BaseRow convertToBaseRow(BaseRow input) { + GenericRow row = (GenericRow) input; + int length = row.getArity(); + GenericRow genericRow = new GenericRow(length); + genericRow.setHeader(row.getHeader()); + for (int i = 0; i < length; i++) { + if (row.getField(i) == null) { + genericRow.setField(i, row.getField(i)); + } else if (row.getField(i) instanceof String) { + genericRow.setField(i, BinaryString.fromString((String) row.getField(i))); + } else if (row.getField(i) instanceof Timestamp) { + SqlTimestamp newTimestamp = SqlTimestamp.fromTimestamp(((Timestamp) row.getField(i))); + genericRow.setField(i, newTimestamp); + } else if (row.getField(i) instanceof LocalDateTime) { + genericRow.setField(i, SqlTimestamp.fromLocalDateTime((LocalDateTime) row.getField(i))); + } else if (row.getField(i) instanceof Time) { + genericRow.setField(i, DataFormatConverters.TimeConverter.INSTANCE.toInternal((Time) row.getField(i))); + } else if (row.getField(i) instanceof Double || row.getField(i).getClass().equals(double.class)) { + genericRow.setField(i, DataFormatConverters.DoubleConverter.INSTANCE.toInternal((Double) row.getField(i))); + } else if (row.getField(i) instanceof Float || row.getField(i).getClass().equals(float.class)) { + genericRow.setField(i, DataFormatConverters.FloatConverter.INSTANCE.toInternal((Float) row.getField(i))); + } else if (row.getField(i) instanceof Long || row.getField(i).getClass().equals(long.class)) { + genericRow.setField(i, DataFormatConverters.LongConverter.INSTANCE.toInternal((Long) row.getField(i))); + } else if (row.getField(i) instanceof Boolean || row.getField(i).getClass().equals(boolean.class)) { + genericRow.setField(i, DataFormatConverters.BooleanConverter.INSTANCE.toInternal((Boolean) row.getField(i))); + } else if (row.getField(i) instanceof Integer || row.getField(i).getClass().equals(int.class)) { + genericRow.setField(i, DataFormatConverters.IntConverter.INSTANCE.toInternal((Integer) row.getField(i))); + } else if (row.getField(i) instanceof Short || row.getField(i).getClass().equals(short.class)) { + genericRow.setField(i, DataFormatConverters.ShortConverter.INSTANCE.toInternal((Short) row.getField(i))); + } else if (row.getField(i) instanceof Byte || row.getField(i).getClass().equals(byte.class)) { + genericRow.setField(i, DataFormatConverters.ByteConverter.INSTANCE.toInternal((Byte) row.getField(i))); + } else if (row.getField(i) instanceof Date) { + genericRow.setField(i, DataFormatConverters.DateConverter.INSTANCE.toInternal((Date) row.getField(i))); + } else if (row.getField(i) instanceof LocalDate) { + genericRow.setField(i, DataFormatConverters.LocalDateConverter.INSTANCE.toInternal((LocalDate) row.getField(i))); + } else if (row.getField(i) instanceof LocalTime) { + genericRow.setField(i, DataFormatConverters.LocalTimeConverter.INSTANCE.toInternal((LocalTime) row.getField(i))); + } else if (row.getField(i) instanceof BigDecimal) { + BigDecimal tempDecimal = (BigDecimal) row.getField(i); + int precision = ((BigDecimal) row.getField(i)).precision(); + int scale = ((BigDecimal) row.getField(i)).scale(); + DataFormatConverters.DecimalConverter decimalConverter = new DataFormatConverters.DecimalConverter(precision, scale); + genericRow.setField(i, decimalConverter.toExternal(Decimal.fromBigDecimal(tempDecimal, precision, scale))); + } else { + genericRow.setField(i, row.getField(i)); + } + } + return genericRow; + } } \ No newline at end of file diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index d08e7a55f..47c53414f 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -125,7 +125,7 @@ public void onResponse(SearchResponse searchResponse) { searchHits = searchResponse.getHits().getHits(); } dealCacheData(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); - RowDataComplete.completeRow(resultFuture, rowList); + RowDataComplete.completeBaseRow(resultFuture, rowList); } catch (Exception e) { dealFillDataError(input, resultFuture, e); } finally { diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 9cc6f2976..1827fdbaf 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -153,7 +153,7 @@ public void flatMap(BaseRow input, Collector out) throws Exception { if (equalObj == null) { if (sideInfo.getJoinType() == JoinType.LEFT) { BaseRow data = fillData(input, null); - RowDataComplete.collectRow(out, data); + RowDataComplete.collectBaseRow(out, data); } return; } @@ -171,13 +171,13 @@ public void flatMap(BaseRow input, Collector out) throws Exception { if (entry.getKey().startsWith(rowKeyStr)) { cacheList = cacheRef.get().get(entry.getKey()); BaseRow row = fillData(input, cacheList); - RowDataComplete.collectRow(out, row); + RowDataComplete.collectBaseRow(out, row); } } } else { cacheList = cacheRef.get().get(rowKeyStr); BaseRow row = fillData(input, cacheList); - RowDataComplete.collectRow(out, row); + RowDataComplete.collectBaseRow(out, row); } } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java index d82ac9a82..47c15c6bb 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java @@ -78,7 +78,7 @@ protected void dealMissKey(BaseRow input, ResultFuture resultFuture) { try { //保留left 表数据 BaseRow row = fillData(input, null); - RowDataComplete.completeRow(resultFuture, row); + RowDataComplete.completeBaseRow(resultFuture, row); } catch (Exception e) { resultFuture.completeExceptionally(e); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java index 0ab250129..df2694238 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java @@ -136,7 +136,7 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, } if (rowList.size() > 0){ - RowDataComplete.completeRow(resultFuture, rowList); + RowDataComplete.completeBaseRow(resultFuture, rowList); } if(openCache){ diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index 16d480c1e..d7b65098e 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -96,7 +96,7 @@ public void asyncGetData(String tableName, String rowKeyStr, BaseRow input, Resu if(openCache){ sideCache.putCache(rowKeyStr, CacheObj.buildCacheObj(ECacheContentType.SingleLine, sideVal)); } - RowDataComplete.completeRow(resultFuture, row); + RowDataComplete.completeBaseRow(resultFuture, row); } catch (Exception e) { resultFuture.completeExceptionally(e); } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 42aa8a65d..d70932ead 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -104,14 +104,14 @@ public void flatMap(BaseRow input, Collector out) throws Exception { if (CollectionUtils.isEmpty(cacheList)) { if (sideInfo.getJoinType() == JoinType.LEFT) { BaseRow row = fillData(input, null); - RowDataComplete.collectRow(out, row); + RowDataComplete.collectBaseRow(out, row); } return; } for (Map one : cacheList) { BaseRow row = fillData(input, one); - RowDataComplete.collectRow(out, row); + RowDataComplete.collectBaseRow(out, row); } } diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index 03e6646b0..d8ff6974e 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -278,7 +278,7 @@ public Deferred> call(RowResultIterator results) throws Exception { if (openCache()) { putCache(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); } - RowDataComplete.completeRow(resultFuture, rowList); + RowDataComplete.completeBaseRow(resultFuture, rowList); } else { dealMissKey(input, resultFuture); if (openCache()) { diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index 1522f2e34..8e4292438 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -107,7 +107,7 @@ public void flatMap(BaseRow input, Collector out) throws Exception { if (equalObj == null) { if (sideInfo.getJoinType() == JoinType.LEFT) { BaseRow data = fillData(input, null); - RowDataComplete.collectRow(out, data); + RowDataComplete.collectBaseRow(out, data); } return; } @@ -119,7 +119,7 @@ public void flatMap(BaseRow input, Collector out) throws Exception { if (CollectionUtils.isEmpty(cacheList)) { if (sideInfo.getJoinType() == JoinType.LEFT) { BaseRow row = fillData(input, null); - RowDataComplete.collectRow(out, row); + RowDataComplete.collectBaseRow(out, row); } else { return; } @@ -129,7 +129,7 @@ public void flatMap(BaseRow input, Collector out) throws Exception { for (Map one : cacheList) { BaseRow row = fillData(input, one); - RowDataComplete.collectRow(out, row); + RowDataComplete.collectBaseRow(out, row); } } diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 9fa8cd689..9b8376cbc 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -123,7 +123,7 @@ public void apply(final Document document) { if (openCache()) { cacheContent.add(document); } - RowDataComplete.completeRow(resultFuture, row); + RowDataComplete.completeBaseRow(resultFuture, row); } }; SingleResultCallback callbackWhenFinished = new SingleResultCallback() { diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index ef0133cc3..a13c32ad6 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -112,7 +112,7 @@ public void flatMap(BaseRow value, Collector out) throws Exception { if (inputParams.size() != equalValIndex.size() && sideInfo.getJoinType() == JoinType.LEFT) { BaseRow row = fillData(value, null); - RowDataComplete.collectRow(out, row); + RowDataComplete.collectBaseRow(out, row); return; } @@ -123,9 +123,9 @@ public void flatMap(BaseRow value, Collector out) throws Exception { List> cacheList = cacheRef.get().get(cacheKey); if (CollectionUtils.isEmpty(cacheList) && sideInfo.getJoinType() == JoinType.LEFT) { BaseRow row = fillData(value, null); - RowDataComplete.collectRow(out, row); + RowDataComplete.collectBaseRow(out, row); } else if (!CollectionUtils.isEmpty(cacheList)) { - cacheList.forEach(one -> out.collect(fillData(value, one))); + cacheList.forEach(one -> RowDataComplete.collectBaseRow(out, fillData(value, one))); } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index f0d8b4a3f..ac0c90d23 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -320,7 +320,7 @@ private void handleQuery(SQLConnection connection, Map inputPara if (openCache()) { putCache(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); } - RowDataComplete.completeRow(resultFuture, rowList); + RowDataComplete.completeBaseRow(resultFuture, rowList); } else { dealMissKey(input, resultFuture); if (openCache()) { diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 01aa46a2d..6c08cc15e 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -117,7 +117,7 @@ public void flatMap(BaseRow input, Collector out) throws Exception { if(equalObj == null){ if (sideInfo.getJoinType() == JoinType.LEFT) { BaseRow data = fillData(input, null); - RowDataComplete.collectRow(out, data); + RowDataComplete.collectBaseRow(out, data); } return; } @@ -131,7 +131,7 @@ public void flatMap(BaseRow input, Collector out) throws Exception { if (cacheMap == null){ if(sideInfo.getJoinType() == JoinType.LEFT){ BaseRow data = fillData(input, null); - RowDataComplete.collectRow(out, data); + RowDataComplete.collectBaseRow(out, data); }else{ return; } @@ -140,7 +140,7 @@ public void flatMap(BaseRow input, Collector out) throws Exception { } BaseRow newRow = fillData(input, cacheMap); - RowDataComplete.collectRow(out, newRow); + RowDataComplete.collectBaseRow(out, newRow); } private void loadData(Map> tmpCache) throws SQLException { diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 395e44b1a..7c5c15a98 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -136,7 +136,7 @@ public void accept(Map values) { try { BaseRow row = fillData(input, values); dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.SingleLine, row)); - RowDataComplete.completeRow(resultFuture, row); + RowDataComplete.completeBaseRow(resultFuture, row); } catch (Exception e) { dealFillDataError(input, resultFuture, e); } From f3552de9f933c3a5f69a85818607e10ec768973c Mon Sep 17 00:00:00 2001 From: wuren Date: Wed, 30 Dec 2020 19:28:46 +0800 Subject: [PATCH 39/80] [fix-33847][core][kafka] Map and Array type support in the kafka sink. --- .../flink/sql/exec/ExecuteProcessHelper.java | 10 +++++-- .../com/dtstack/flink/sql/util/ClassUtil.java | 6 ++++ .../dtstack/flink/sql/util/DataTypeUtils.java | 29 +++++++++++++++++++ docs/plugin/kafkaSink.md | 26 +++++++++++++++++ docs/pluginsInfo.md | 2 +- .../sql/sink/kafka/AbstractKafkaSink.java | 16 ++++++++-- 6 files changed, 83 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 0d447a016..c5ca61c85 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -382,10 +382,14 @@ public static Set registerTable( , pluginLoadMode); pluginClassPathSets.add(sourceTablePathUrl); } else if (tableInfo instanceof AbstractTargetTableInfo) { - TableSink tableSink = StreamSinkFactory.getTableSink((AbstractTargetTableInfo) tableInfo, localSqlPluginPath, pluginLoadMode); - TypeInformation[] flinkTypes = DataTypeUtils.transformTypes(tableInfo.getFieldClasses()); - tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); + // TODO Kafka Sink直接注册，其他的Sink要修复才可以。 + if (tableInfo.getType().startsWith("kafka")) { + tableEnv.registerTableSink(tableInfo.getName(), tableSink); + } else { + TypeInformation[] flinkTypes = DataTypeUtils.transformTypes(tableInfo.getFieldClasses()); + tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); + } URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode( tableInfo.getType() diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index ec88bdbe9..b7d2f12fc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -25,6 +25,8 @@ import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; +import java.util.HashMap; +import java.util.Map; /** * Reason: TODO ADD REASON(可选) @@ -41,6 +43,10 @@ public static Class stringConvertClass(String str) { if (lowerStr.startsWith("array")) { return Array.newInstance(Integer.class, 0).getClass(); } + if (lowerStr.startsWith("map")) { + Map m = new HashMap(); + return m.getClass(); + } switch (lowerStr) { case "boolean": diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index ff7512328..0d6400bde 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -46,6 +47,8 @@ import static org.apache.flink.table.api.DataTypes.DECIMAL; import static org.apache.flink.table.api.DataTypes.TIMESTAMP; +import static org.apache.commons.lang3.StringUtils.split; + /** * @program: flink.sql * @author: wuren @@ -55,6 +58,7 @@ public class DataTypeUtils { private final static Pattern COMPOSITE_TYPE_PATTERN = Pattern.compile("(.+?)<(.+)>"); private final static String ARRAY = "ARRAY"; + private final static String MAP = "MAP"; private final static String ROW = "ROW"; private final static char FIELD_DELIMITER = ','; private final static char TYPE_DELIMITER = ' '; @@ -105,6 +109,30 @@ public static TypeInformation convertToArray(String arrayTypeString) { return Types.OBJECT_ARRAY(elementType); } + /** + * 目前Map里只支持基本类型 + * @param mapTypeString + * @return + */ + public static TypeInformation convertToMap(String mapTypeString) { + Matcher matcher = matchCompositeType(mapTypeString); + final String errorMsg = mapTypeString + "convert to map type error!"; + Preconditions.checkState(matcher.find(), errorMsg); + + String normalizedType = normalizeType(matcher.group(1)); + Preconditions.checkState(MAP.equals(normalizedType), errorMsg); + + String kvTypeString = matcher.group(2); + String[] kvTypeStringList = StringUtils.split(kvTypeString, ","); + final String mapTypeErrorMsg = "There can only be key and value two types in map declaration."; + Preconditions.checkState(kvTypeStringList.length == 2, mapTypeErrorMsg); + String keyTypeString = normalizeType(kvTypeStringList[0]); + String valueTypeString = normalizeType(kvTypeStringList[1]); + TypeInformation keyType = convertToAtomicType(keyTypeString); + TypeInformation valueType = convertToAtomicType(valueTypeString); + return Types.MAP(keyType, valueType); + } + /** * 目前ROW里只支持基本类型 * @@ -158,6 +186,7 @@ public static TypeInformation getRowTypeInformation(String[] fieldTypes, Cl return new RowTypeInfo(types, fieldTypes); } + private static Tuple2 genFieldInfo(Iterable fieldInfoStrs) { ArrayList types = Lists.newArrayList(); ArrayList fieldNames = Lists.newArrayList(); diff --git a/docs/plugin/kafkaSink.md b/docs/plugin/kafkaSink.md index 86b01f3ee..b3a1614bc 100644 --- a/docs/plugin/kafkaSink.md +++ b/docs/plugin/kafkaSink.md @@ -221,3 +221,29 @@ into from MyTable a ``` +## MAP类型示例 +目前Kafka Sink支持Map类型 +```sql +CREATE TABLE ods( + id INT, + name STRING +) WITH ( + ... +); + +CREATE TABLE dwd ( + id INT, + dids MAP> +) WITH ( + type ='kafka', + bootstrapServers ='localhost:9092', + offsetReset ='latest', + groupId='wuren_foo', + topic ='luna_foo', + parallelism ='1' +); + +INSERT INTO dwd + SELECT ods.id, MAP['foo', 1, 'bar', 2] AS dids + FROM ods; +``` diff --git a/docs/pluginsInfo.md b/docs/pluginsInfo.md index 88fda90c9..e9d23cdbd 100644 --- a/docs/pluginsInfo.md +++ b/docs/pluginsInfo.md @@ -1,9 +1,9 @@ ### 1 插件列表 #### 1.1 源表插件 * [kafka 源表插件](plugin/kafkaSource.md) -* [kafka 结果表插件](plugin/kafkaSink.md) #### 1.2 结果表插件 +* [kafka 结果表插件](plugin/kafkaSink.md) * [elasticsearch 结果表插件](plugin/elasticsearchSink.md) * [hbase 结果表插件](plugin/hbaseSink.md) * [mysql 结果表插件](plugin/mysqlSink.md) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index 9844e10dd..fa6f8adc6 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; +import com.dtstack.flink.sql.util.DataTypeUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -38,6 +39,7 @@ import org.apache.flink.util.Preconditions; import org.apache.kafka.clients.consumer.ConsumerConfig; +import java.util.HashMap; import java.util.Optional; import java.util.Properties; import java.util.stream.IntStream; @@ -77,11 +79,21 @@ protected Properties getKafkaProperties(KafkaSinkTableInfo KafkaSinkTableInfo) { } return props; } - + // TODO Source有相同的方法日后可以合并 protected TypeInformation[] getTypeInformations(KafkaSinkTableInfo kafka11SinkTableInfo) { + String[] fieldTypes = kafka11SinkTableInfo.getFieldTypes(); Class[] fieldClasses = kafka11SinkTableInfo.getFieldClasses(); TypeInformation[] types = IntStream.range(0, fieldClasses.length) - .mapToObj(i -> TypeInformation.of(fieldClasses[i])) + .mapToObj( + i -> { + if (fieldClasses[i].isArray()) { + return DataTypeUtils.convertToArray(fieldTypes[i]); + } + if (fieldClasses[i] == new HashMap().getClass()) { + return DataTypeUtils.convertToMap(fieldTypes[i]); + } + return TypeInformation.of(fieldClasses[i]); + }) .toArray(TypeInformation[]::new); return types; } From e3f1b0ea7466cbae7e3f361f24fc3a788a72130a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Thu, 31 Dec 2020 14:37:24 +0800 Subject: [PATCH 40/80] =?UTF-8?q?[opt-34201][redis]=20Redis=20Sink=20?= =?UTF-8?q?=E9=87=8D=E6=9E=84.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/redis/RedisOutputFormat.java | 149 +++++++----------- .../flink/sql/sink/redis/RedisSink.java | 8 +- .../flink/sql/sink/redis/enums/RedisType.java | 2 +- 3 files changed, 62 insertions(+), 97 deletions(-) diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index ddfb876df..47704226c 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -37,10 +37,9 @@ import java.io.Closeable; import java.io.IOException; -import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -48,33 +47,19 @@ */ public class RedisOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(RedisOutputFormat.class); - + protected String[] fieldNames; + protected TypeInformation[] fieldTypes; + protected List primaryKeys; + protected int timeout = 10000; private String url; - - private String database; - + private String database = "0"; private String tableName; - private String password; - private int redisType; - private String maxTotal; - private String maxIdle; - private String minIdle; - private String masterName; - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected List primaryKeys; - - protected int timeout; - private JedisPool pool; private JedisCommands jedis; @@ -83,8 +68,13 @@ public class RedisOutputFormat extends AbstractDtRichOutputFormat { private GenericObjectPoolConfig poolConfig; - private RedisOutputFormat(){ + private RedisOutputFormat() { } + + public static RedisOutputFormatBuilder buildRedisOutputFormat() { + return new RedisOutputFormatBuilder(); + } + @Override public void configure(Configuration parameters) { @@ -96,15 +86,15 @@ public void open(int taskNumber, int numTasks) throws IOException { initMetric(); } - private GenericObjectPoolConfig setPoolConfig(String maxTotal, String maxIdle, String minIdle){ + private GenericObjectPoolConfig setPoolConfig(String maxTotal, String maxIdle, String minIdle) { GenericObjectPoolConfig config = new GenericObjectPoolConfig(); - if (maxTotal != null){ + if (maxTotal != null) { config.setMaxTotal(Integer.parseInt(maxTotal)); } - if (maxIdle != null){ + if (maxIdle != null) { config.setMaxIdle(Integer.parseInt(maxIdle)); } - if (minIdle != null){ + if (minIdle != null) { config.setMinIdle(Integer.parseInt(minIdle)); } return config; @@ -121,31 +111,23 @@ private void establishConnection() { for (String ipPort : nodes) { ipPorts.add(ipPort); String[] ipPortPair = StringUtils.split(ipPort, ":"); - addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.valueOf(ipPortPair[1].trim()))); - } - if (timeout == 0){ - timeout = 10000; - } - if (database == null) - { - database = "0"; + addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.parseInt(ipPortPair[1].trim()))); } - switch (redisType){ - //单机 - case 1: + switch (RedisType.parse(redisType)) { + case STANDALONE: pool = new JedisPool(poolConfig, firstIp, Integer.parseInt(firstPort), timeout, password, Integer.parseInt(database)); jedis = pool.getResource(); break; - //哨兵 - case 2: + case SENTINEL: jedisSentinelPool = new JedisSentinelPool(masterName, ipPorts, poolConfig, timeout, password, Integer.parseInt(database)); jedis = jedisSentinelPool.getResource(); break; - //集群 - case 3: + case CLUSTER: jedis = new JedisCluster(addresses, timeout, timeout, 10, password, poolConfig); + break; default: + throw new RuntimeException("unsupported redis type[ " + redisType + "]"); } } @@ -160,38 +142,14 @@ public void writeRecord(Tuple2 record) throws IOException { if (row.getArity() != fieldNames.length) { return; } - - HashMap map = new HashMap<>(8); - for (String primaryKey : primaryKeys) { - for (int i = 0; i < fieldNames.length; i++) { - if (fieldNames[i].equals(primaryKey)) { - map.put(primaryKey, i); - } - } - } - - List kvList = new LinkedList<>(); - for (String primaryKey : primaryKeys){ - StringBuilder primaryKv = new StringBuilder(); - int index = map.get(primaryKey).intValue(); - primaryKv.append(primaryKey).append(":").append(row.getField(index)); - kvList.add(primaryKv.toString()); - } - - String perKey = String.join(":", kvList); + Map refData = Maps.newHashMap(); for (int i = 0; i < fieldNames.length; i++) { - StringBuilder key = new StringBuilder(); - key.append(tableName).append(":").append(perKey).append(":").append(fieldNames[i]); - - String value = "null"; - Object field = row.getField(i); - if (field != null) { - value = field.toString(); - } - jedis.set(key.toString(), value); + refData.put(fieldNames[i], row.getField(i)); } + String redisKey = buildCacheKey(refData); + refData.forEach((key, value) -> jedis.hset(redisKey, key, String.valueOf(value))); - if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0){ + if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0) { LOG.info(record.toString()); } outRecords.inc(); @@ -205,96 +163,103 @@ public void close() throws IOException { if (pool != null) { pool.close(); } - if (jedis != null){ - if (jedis instanceof Closeable){ + if (jedis != null) { + if (jedis instanceof Closeable) { ((Closeable) jedis).close(); } } } - public static RedisOutputFormatBuilder buildRedisOutputFormat(){ - return new RedisOutputFormatBuilder(); + public String buildCacheKey(Map refData) { + StringBuilder keyBuilder = new StringBuilder(tableName); + for (String primaryKey : primaryKeys) { + if (!refData.containsKey(primaryKey)) { + return null; + } + keyBuilder.append("_").append(refData.get(primaryKey)); + } + return keyBuilder.toString(); } - public static class RedisOutputFormatBuilder{ + public static class RedisOutputFormatBuilder { private final RedisOutputFormat redisOutputFormat; - protected RedisOutputFormatBuilder(){ + protected RedisOutputFormatBuilder() { this.redisOutputFormat = new RedisOutputFormat(); } - public RedisOutputFormatBuilder setUrl(String url){ + public RedisOutputFormatBuilder setUrl(String url) { redisOutputFormat.url = url; return this; } - public RedisOutputFormatBuilder setDatabase(String database){ + public RedisOutputFormatBuilder setDatabase(String database) { redisOutputFormat.database = database; return this; } - public RedisOutputFormatBuilder setTableName(String tableName){ + public RedisOutputFormatBuilder setTableName(String tableName) { redisOutputFormat.tableName = tableName; return this; } - public RedisOutputFormatBuilder setPassword(String password){ + public RedisOutputFormatBuilder setPassword(String password) { redisOutputFormat.password = password; return this; } - public RedisOutputFormatBuilder setFieldNames(String[] fieldNames){ + public RedisOutputFormatBuilder setFieldNames(String[] fieldNames) { redisOutputFormat.fieldNames = fieldNames; return this; } - public RedisOutputFormatBuilder setFieldTypes(TypeInformation[] fieldTypes){ + public RedisOutputFormatBuilder setFieldTypes(TypeInformation[] fieldTypes) { redisOutputFormat.fieldTypes = fieldTypes; return this; } - public RedisOutputFormatBuilder setPrimaryKeys(List primaryKeys){ + public RedisOutputFormatBuilder setPrimaryKeys(List primaryKeys) { redisOutputFormat.primaryKeys = primaryKeys; return this; } - public RedisOutputFormatBuilder setTimeout(int timeout){ + public RedisOutputFormatBuilder setTimeout(int timeout) { redisOutputFormat.timeout = timeout; return this; } - public RedisOutputFormatBuilder setRedisType(int redisType){ + public RedisOutputFormatBuilder setRedisType(int redisType) { redisOutputFormat.redisType = redisType; return this; } - public RedisOutputFormatBuilder setMaxTotal(String maxTotal){ + public RedisOutputFormatBuilder setMaxTotal(String maxTotal) { redisOutputFormat.maxTotal = maxTotal; return this; } - public RedisOutputFormatBuilder setMaxIdle(String maxIdle){ + public RedisOutputFormatBuilder setMaxIdle(String maxIdle) { redisOutputFormat.maxIdle = maxIdle; return this; } - public RedisOutputFormatBuilder setMinIdle(String minIdle){ + public RedisOutputFormatBuilder setMinIdle(String minIdle) { redisOutputFormat.minIdle = minIdle; return this; } - public RedisOutputFormatBuilder setMasterName(String masterName){ + public RedisOutputFormatBuilder setMasterName(String masterName) { redisOutputFormat.masterName = masterName; return this; } - public RedisOutputFormat finish(){ - if (redisOutputFormat.url == null){ + public RedisOutputFormat finish() { + if (redisOutputFormat.url == null) { throw new IllegalArgumentException("No URL supplied."); } - if (redisOutputFormat.tableName == null){ + if (redisOutputFormat.tableName == null) { throw new IllegalArgumentException("No tablename supplied."); } diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java index faad0bd4a..1e68cc51b 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java @@ -71,7 +71,7 @@ public class RedisSink implements RetractStreamTableSink, IStreamSinkGener< protected String registerTableName; - public RedisSink(){ + public RedisSink() { } @@ -108,7 +108,7 @@ public void emitDataStream(DataStream> dataStream) { @Override public DataStreamSink> consumeDataStream(DataStream> dataStream) { RedisOutputFormat.RedisOutputFormatBuilder builder = RedisOutputFormat.buildRedisOutputFormat(); - builder.setUrl(this.url) + RedisOutputFormat redisOutputFormat = builder.setUrl(this.url) .setDatabase(this.database) .setTableName(this.tableName) .setPassword(this.password) @@ -120,8 +120,8 @@ public DataStreamSink> consumeDataStream(DataStream Date: Tue, 5 Jan 2021 15:11:20 +0800 Subject: [PATCH 41/80] [feat] add dirty manager and resolve conflict. --- .../java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index c019d9c58..51d79e6af 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -124,14 +124,9 @@ public DataStreamSink> consumeDataStream(DataStream Date: Tue, 5 Jan 2021 15:17:27 +0800 Subject: [PATCH 42/80] [hotfix-34625][redis5][sink]add expire attribute for redis sink's key --- docs/plugin/redisSink.md | 1 + .../sql/sink/redis/RedisOutputFormat.java | 27 ++++++++++++++++++- .../flink/sql/sink/redis/RedisSink.java | 6 ++++- .../sql/sink/redis/table/RedisSinkParser.java | 4 +++ .../sql/sink/redis/table/RedisTableInfo.java | 12 +++++++++ 5 files changed, 48 insertions(+), 2 deletions(-) diff --git a/docs/plugin/redisSink.md b/docs/plugin/redisSink.md index eb0109f06..21d68aca4 100644 --- a/docs/plugin/redisSink.md +++ b/docs/plugin/redisSink.md @@ -44,6 +44,7 @@ redis5.0 |minIdle|最小空闲连接数|否||0| |masterName| 哨兵模式下的masterName|否|| |primarykeys|主键字段，多个字段以逗号分割|是|| +|keyExpiredTime|redis sink的key的过期时间。默认是0（永不过期），单位是ms。|否|| ## 5.样例： diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index ddfb876df..d545334df 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -75,6 +75,8 @@ public class RedisOutputFormat extends AbstractDtRichOutputFormat { protected int timeout; + protected long keyExpiredTime; + private JedisPool pool; private JedisCommands jedis; @@ -188,7 +190,7 @@ public void writeRecord(Tuple2 record) throws IOException { if (field != null) { value = field.toString(); } - jedis.set(key.toString(), value); + saveKey(key.toString(), value); } if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0){ @@ -197,6 +199,24 @@ public void writeRecord(Tuple2 record) throws IOException { outRecords.inc(); } + /** + * 1. save key and value. + * 2. set expired time for key when keyExpiredTime has been set. + * @param key + * @param value + */ + private void saveKey(String key, String value) { + if (keyExpiredTime != 0L) { + boolean keyExist = jedis.exists(key); + if (keyExist) { + jedis.del(key); + } + jedis.set(key, value, "NX", "PX", keyExpiredTime); + } else { + jedis.set(key, value); + } + } + @Override public void close() throws IOException { if (jedisSentinelPool != null) { @@ -289,6 +309,11 @@ public RedisOutputFormatBuilder setMasterName(String masterName){ return this; } + public RedisOutputFormatBuilder setKeyExpiredTime(long keyExpiredTime){ + redisOutputFormat.keyExpiredTime = keyExpiredTime; + return this; + } + public RedisOutputFormat finish(){ if (redisOutputFormat.url == null){ throw new IllegalArgumentException("No URL supplied."); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java index faad0bd4a..a1936c3bd 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java @@ -71,6 +71,8 @@ public class RedisSink implements RetractStreamTableSink, IStreamSinkGener< protected String registerTableName; + protected long keyExpiredTime; + public RedisSink(){ } @@ -92,6 +94,7 @@ public RedisSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.parallelism = Objects.isNull(redisTableInfo.getParallelism()) ? parallelism : redisTableInfo.getParallelism(); this.registerTableName = redisTableInfo.getName(); + this.keyExpiredTime = redisTableInfo.getKeyExpiredTime(); return this; } @@ -120,7 +123,8 @@ public DataStreamSink> consumeDataStream(DataStream primaryKeysList = Lists.newArrayList(); if (!StringUtils.isEmpty(primaryKeysStr)) { diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java index 2425f8c38..6377a4113 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java @@ -49,6 +49,8 @@ public class RedisTableInfo extends AbstractTargetTableInfo { public static final String PRIMARY_KEYS_NAME = "primarykeys"; + public static final String KEY_EXPIRED_TIME = "keyExpiredTime"; + public RedisTableInfo(){ setType(CURR_TYPE); } @@ -73,6 +75,8 @@ public RedisTableInfo(){ private String masterName; + private long keyExpiredTime; + public String getUrl() { return url; } @@ -153,6 +157,14 @@ public void setMasterName(String masterName) { this.masterName = masterName; } + public long getKeyExpiredTime() { + return keyExpiredTime; + } + + public void setKeyExpiredTime(long keyExpiredTime) { + this.keyExpiredTime = keyExpiredTime; + } + @Override public boolean check() { Preconditions.checkNotNull(url, "redis field of URL is required"); From a95170cb510c3d25bed4d418ba75987131176ed6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Tue, 5 Jan 2021 18:01:05 +0800 Subject: [PATCH 43/80] =?UTF-8?q?[fix-31342][rdb]=20=E4=BF=AE=E5=A4=8Drdb?= =?UTF-8?q?=20sink=20=E8=B5=84=E6=BA=90=E6=A0=A1=E9=AA=8C=E7=A9=BA?= =?UTF-8?q?=E6=8C=87=E9=92=88=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java | 1 + 1 file changed, 1 insertion(+) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java index 340f1a3ea..ec613bd56 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java @@ -51,6 +51,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Tue, 5 Jan 2021 18:56:03 +0800 Subject: [PATCH 44/80] [fix-34344][kafka] remove partition key blank char. --- .../flink/sql/sink/kafka/AbstractKafkaSink.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index 560df5c28..709c1a9e4 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -39,6 +39,7 @@ import org.apache.flink.util.Preconditions; import org.apache.kafka.clients.consumer.ConsumerConfig; +import java.util.Arrays; import java.util.HashMap; import java.util.Optional; import java.util.Properties; @@ -113,8 +114,13 @@ protected TableSchema buildTableSchema(String[] fieldNames, TypeInformation[] } protected String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo) { - if (StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())) { - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + String keysStr = kafkaSinkTableInfo.getPartitionKeys(); + if (StringUtils.isNotBlank(keysStr)) { + String[] keys = keysStr.split(","); + String[] cleanedKeys = Arrays.stream(keys) + .map(x -> x.trim()) + .toArray(String[]::new); + return cleanedKeys; } return null; } From 81719264b124633c6399385775daa81f9466a1c8 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 5 Jan 2021 19:20:01 +0800 Subject: [PATCH 45/80] [fix-34344][kafka] use apache common split --- .../com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index 709c1a9e4..f0f215c27 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -116,7 +116,7 @@ protected TableSchema buildTableSchema(String[] fieldNames, TypeInformation[] protected String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo) { String keysStr = kafkaSinkTableInfo.getPartitionKeys(); if (StringUtils.isNotBlank(keysStr)) { - String[] keys = keysStr.split(","); + String[] keys = StringUtils.split(keysStr, ","); String[] cleanedKeys = Arrays.stream(keys) .map(x -> x.trim()) .toArray(String[]::new); From 95cb6821d480e39be35604d356ae2d16d9fd99b9 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 5 Jan 2021 21:14:27 +0800 Subject: [PATCH 46/80] [fix-34193][root][pom] delete plugins dir in maven clean phase --- pom.xml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pom.xml b/pom.xml index 1bad3677d..8f293ab9d 100644 --- a/pom.xml +++ b/pom.xml @@ -140,6 +140,26 @@ + + maven-antrun-plugin + 1.8 + + + clean-plugins + + clean + + run + + + + + + + + + + From 42c0830378f4325cfa85d57a58d34a109ece40a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Wed, 6 Jan 2021 09:54:44 +0800 Subject: [PATCH 47/80] =?UTF-8?q?[fix]=E4=BF=AE=E5=A4=8D=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E8=84=8F=E6=95=B0=E6=8D=AE=E6=8F=92=E4=BB=B6=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=97=A0=E6=B3=95=E6=89=A7=E8=A1=8C=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/source/kafka/table/KafkaSourceParser.java | 1 - localTest/pom.xml | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index d7b8d3613..de566fe35 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -70,7 +70,6 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map1.0-SNAPSHOT--> + + com.dtstack.flink + dirtyConsumer.mysql + 1.0-SNAPSHOT + + + + com.dtstack.flink + dirtyConsumer.console + 1.0-SNAPSHOT + + com.dtstack.flink sql.source.kafka11 From ec93bfbfb9c76c2bfa99fc74d54b98b7488d9d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Wed, 6 Jan 2021 16:07:04 +0800 Subject: [PATCH 48/80] =?UTF-8?q?[opt-34257][kudu]=20=E4=BC=98=E5=8C=96kud?= =?UTF-8?q?u=E6=8F=92=E4=BB=B6=EF=BC=8C=E6=94=B9=E7=94=A8KuduClient=20API?= =?UTF-8?q?=E4=BB=A3=E6=9B=BFAsyncKuduClient?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/kudu/KuduOutputFormat.java | 221 +++++++++--------- .../dtstack/flink/sql/sink/kudu/KuduSink.java | 21 +- 2 files changed, 113 insertions(+), 129 deletions(-) diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index 6f2a31caa..e8b508f1f 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -25,8 +25,6 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.kudu.client.AsyncKuduClient; -import org.apache.kudu.client.AsyncKuduSession; import org.apache.kudu.client.KuduClient; import org.apache.kudu.client.KuduException; import org.apache.kudu.client.KuduSession; @@ -52,26 +50,12 @@ public class KuduOutputFormat extends AbstractDtRichOutputFormat { private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger(KuduOutputFormat.class); - - public enum WriteMode { - // insert - INSERT, - // update - UPDATE, - // update or insert - UPSERT - } - + protected String[] fieldNames; + TypeInformation[] fieldTypes; + boolean enableKrb; private String kuduMasters; - private String tableName; - private WriteMode writeMode; - - protected String[] fieldNames; - - TypeInformation[] fieldTypes; - private KuduClient client; private KuduTable table; @@ -90,11 +74,14 @@ public enum WriteMode { private String principal; private String keytab; private String krb5conf; - boolean enableKrb; private KuduOutputFormat() { } + public static KuduOutputFormatBuilder buildKuduOutputFormat() { + return new KuduOutputFormatBuilder(); + } + @Override public void configure(Configuration parameters) { @@ -112,11 +99,11 @@ private void establishConnection() throws IOException { kuduClientBuilder.workerCount(workerCount); } if (null != defaultSocketReadTimeoutMs) { - kuduClientBuilder.workerCount(defaultSocketReadTimeoutMs); + kuduClientBuilder.defaultSocketReadTimeoutMs(defaultSocketReadTimeoutMs); } if (null != defaultOperationTimeoutMs) { - kuduClientBuilder.workerCount(defaultOperationTimeoutMs); + kuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); } if (enableKrb) { @@ -130,10 +117,16 @@ private void establishConnection() throws IOException { } else { client = kuduClientBuilder.build(); } - LOG.info("connect kudu is succeed!"); + if (client.tableExists(tableName)) { table = client.openTable(tableName); } + if (Objects.isNull(table)) { + throw new IllegalArgumentException( + String.format("Table [%s] Open Failed , please check table exists", tableName)); + } + LOG.info("connect kudu is succeed!"); + session = client.newSession(); } @@ -188,8 +181,97 @@ public void close() { } } - public static KuduOutputFormatBuilder buildKuduOutputFormat() { - return new KuduOutputFormatBuilder(); + private Operation toOperation(WriteMode writeMode, Row row) { + Operation operation = toOperation(writeMode); + PartialRow partialRow = operation.getRow(); + + for (int index = 0; index < row.getArity(); index++) { + //解决kudu中全小写字段找不到的bug + String fieldName = fieldNames[index].toLowerCase(); + if (row.getField(index) == null) { + partialRow.setNull(fieldName); + } else { + if (fieldTypes[index].getTypeClass() == String.class) { + partialRow.addString(fieldName, (String) row.getField(index)); + continue; + } + if (fieldTypes[index].getTypeClass() == Float.class) { + partialRow.addFloat(fieldName, (Float) row.getField(index)); + continue; + } + if (fieldTypes[index].getTypeClass() == Byte.class) { + partialRow.addByte(fieldName, (Byte) row.getField(index)); + continue; + } + + if (fieldTypes[index].getTypeClass() == Short.class) { + partialRow.addShort(fieldName, (Short) row.getField(index)); + continue; + } + + if (fieldTypes[index].getTypeClass() == Integer.class) { + partialRow.addInt(fieldName, (Integer) row.getField(index)); + continue; + } + + if (fieldTypes[index].getTypeClass() == Long.class) { + partialRow.addLong(fieldName, (Long) row.getField(index)); + continue; + } + + if (fieldTypes[index].getTypeClass() == Double.class) { + partialRow.addDouble(fieldName, (Double) row.getField(index)); + continue; + } + + if (fieldTypes[index].getTypeClass() == BigDecimal.class) { + partialRow.addDecimal(fieldName, (BigDecimal) row.getField(index)); + continue; + } + if (fieldTypes[index].getTypeClass() == Boolean.class) { + partialRow.addBoolean(fieldName, (Boolean) row.getField(index)); + continue; + } + + if (fieldTypes[index].getTypeClass() == Date.class) { + partialRow.addTimestamp(fieldName, new Timestamp(((Date) row.getField(index)).getTime())); + continue; + } + + if (fieldTypes[index].getTypeClass() == Timestamp.class) { + partialRow.addTimestamp(fieldName, (Timestamp) row.getField(index)); + continue; + } + + if (fieldTypes[index].getTypeClass() == byte[].class) { + partialRow.addBinary(fieldName, (byte[]) row.getField(index)); + continue; + } + throw new IllegalArgumentException("Illegal var type: " + fieldTypes[index]); + } + } + return operation; + + } + + private Operation toOperation(WriteMode writeMode) { + switch (writeMode) { + case INSERT: + return table.newInsert(); + case UPDATE: + return table.newUpdate(); + default: + return table.newUpsert(); + } + } + + public enum WriteMode { + // insert + INSERT, + // update + UPDATE, + // update or insert + UPSERT } public static class KuduOutputFormatBuilder { @@ -276,93 +358,4 @@ public KuduOutputFormat finish() { } } - private Operation toOperation(WriteMode writeMode, Row row) { - if (null == table) { - throw new IllegalArgumentException("Table Open Failed , please check table exists"); - } - Operation operation = toOperation(writeMode); - PartialRow partialRow = operation.getRow(); - - for (int index = 0; index < row.getArity(); index++) { - //解决kudu中全小写字段找不到的bug - String fieldName = fieldNames[index].toLowerCase(); - if (row.getField(index) == null) { - partialRow.setNull(fieldName); - } else { - if (fieldTypes[index].getTypeClass() == String.class) { - partialRow.addString(fieldName, (String) row.getField(index)); - continue; - } - if (fieldTypes[index].getTypeClass() == Float.class) { - partialRow.addFloat(fieldName, (Float) row.getField(index)); - continue; - } - if (fieldTypes[index].getTypeClass() == Byte.class) { - partialRow.addByte(fieldName, (Byte) row.getField(index)); - continue; - } - - if (fieldTypes[index].getTypeClass() == Short.class) { - partialRow.addShort(fieldName, (Short) row.getField(index)); - continue; - } - - if (fieldTypes[index].getTypeClass() == Integer.class) { - partialRow.addInt(fieldName, (Integer) row.getField(index)); - continue; - } - - if (fieldTypes[index].getTypeClass() == Long.class) { - partialRow.addLong(fieldName, (Long) row.getField(index)); - continue; - } - - if (fieldTypes[index].getTypeClass() == Double.class) { - partialRow.addDouble(fieldName, (Double) row.getField(index)); - continue; - } - - if (fieldTypes[index].getTypeClass() == BigDecimal.class) { - partialRow.addDecimal(fieldName, (BigDecimal) row.getField(index)); - continue; - } - if (fieldTypes[index].getTypeClass() == Boolean.class) { - partialRow.addBoolean(fieldName, (Boolean) row.getField(index)); - continue; - } - - if (fieldTypes[index].getTypeClass() == Date.class) { - partialRow.addTimestamp(fieldName, new Timestamp(((Date) row.getField(index)).getTime())); - continue; - } - - if (fieldTypes[index].getTypeClass() == Timestamp.class) { - partialRow.addTimestamp(fieldName, (Timestamp) row.getField(index)); - continue; - } - - if (fieldTypes[index].getTypeClass() == byte[].class) { - partialRow.addBinary(fieldName, (byte[]) row.getField(index)); - continue; - } - throw new IllegalArgumentException("Illegal var type: " + fieldTypes[index]); - } - } - return operation; - - } - - private Operation toOperation(WriteMode writeMode) { - switch (writeMode) { - case INSERT: - return table.newInsert(); - case UPDATE: - return table.newUpdate(); - case UPSERT: - return table.newUpsert(); - default: - return table.newUpsert(); - } - } - } \ No newline at end of file diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java index ef7d1402d..e057b281e 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java @@ -21,28 +21,19 @@ public class KuduSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { + protected String[] fieldNames; + TypeInformation[] fieldTypes; + boolean enableKrb; private String kuduMasters; - private String tableName; - private KuduOutputFormat.WriteMode writeMode; - - protected String[] fieldNames; - - TypeInformation[] fieldTypes; - private Integer workerCount; - private Integer defaultOperationTimeoutMs; - private Integer defaultSocketReadTimeoutMs; - private int parallelism = 1; - private String principal; private String keytab; private String krb5conf; - boolean enableKrb; @Override public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { @@ -71,7 +62,7 @@ public void emitDataStream(DataStream> dataStream) { @Override public DataStreamSink> consumeDataStream(DataStream> dataStream) { KuduOutputFormat.KuduOutputFormatBuilder builder = KuduOutputFormat.buildKuduOutputFormat(); - builder.setKuduMasters(this.kuduMasters) + KuduOutputFormat kuduOutputFormat = builder.setKuduMasters(this.kuduMasters) .setTableName(this.tableName) .setWriteMode(writeMode) .setWorkerCount(this.workerCount) @@ -82,8 +73,8 @@ public DataStreamSink> consumeDataStream(DataStream Date: Wed, 6 Jan 2021 19:45:55 +0800 Subject: [PATCH 49/80] [fix-34400][root][pom] optimize pom for cleaning output dir --- pom.xml | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/pom.xml b/pom.xml index 8f293ab9d..5f8d868e2 100644 --- a/pom.xml +++ b/pom.xml @@ -141,24 +141,19 @@ - maven-antrun-plugin - 1.8 - - - clean-plugins - - clean - - run - - - - - - - - - + org.apache.maven.plugins + maven-clean-plugin + 3.1.0 + + + + ${basedir}/plugins + + + ${basedir}/sqlplugins + + + From 023cae3d339fcaf6f4ed6a51270392f9e0aab611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Wed, 6 Jan 2021 21:25:58 +0800 Subject: [PATCH 50/80] =?UTF-8?q?[feat]=20=E4=BF=AE=E5=A4=8D=E8=84=8F?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=8F=92=E4=BB=B6=E7=BA=BF=E7=A8=8B=E4=B8=BA?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E7=BA=BF=E7=A8=8B=E6=97=B6=EF=BC=8C=E4=B8=BB?= =?UTF-8?q?=E7=BA=BF=E7=A8=8B=E5=BC=82=E5=B8=B8=E5=81=9C=E6=AD=A2=E5=90=8E?= =?UTF-8?q?=EF=BC=8CdirtyConsumer=E7=BA=BF=E7=A8=8B=E4=B8=8D=E5=81=9C?= =?UTF-8?q?=E6=AD=A2=E9=97=AE=E9=A2=98=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../manager/DirtyDataManager.java | 29 +++++++++----- .../sql/dirtyManager/manager/DirtyKeys.java | 40 +++++++++++++++++++ .../sql/dirtyManager/manager/TestMain.java | 34 ---------------- .../flink/sql/factory/DTThreadFactory.java | 21 ++++++++-- .../source/kafka/table/KafkaSourceParser.java | 1 - localTest/pom.xml | 12 ++++++ 6 files changed, 89 insertions(+), 48 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyKeys.java delete mode 100644 core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java index b0b6cab6d..c13ea55fb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyDataManager.java @@ -33,6 +33,17 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import static com.dtstack.flink.sql.dirtyManager.manager.DirtyKeys.DEFAULT_BLOCKING_INTERVAL; +import static com.dtstack.flink.sql.dirtyManager.manager.DirtyKeys.DEFAULT_ERROR_LIMIT_RATE; +import static com.dtstack.flink.sql.dirtyManager.manager.DirtyKeys.DEFAULT_PRINT_LIMIT; +import static com.dtstack.flink.sql.dirtyManager.manager.DirtyKeys.DEFAULT_TYPE; +import static com.dtstack.flink.sql.dirtyManager.manager.DirtyKeys.DIRTY_BLOCK_STR; +import static com.dtstack.flink.sql.dirtyManager.manager.DirtyKeys.DIRTY_LIMIT_RATE_STR; +import static com.dtstack.flink.sql.dirtyManager.manager.DirtyKeys.PLUGIN_LOAD_MODE_STR; +import static com.dtstack.flink.sql.dirtyManager.manager.DirtyKeys.PLUGIN_PATH_STR; +import static com.dtstack.flink.sql.dirtyManager.manager.DirtyKeys.PLUGIN_TYPE_STR; +import static com.dtstack.flink.sql.dirtyManager.manager.DirtyKeys.PRINT_LIMIT_STR; + /** * @author tiezhu * Company dtstack @@ -43,11 +54,7 @@ public class DirtyDataManager implements Serializable { public final static int MAX_POOL_SIZE_LIMIT = 5; private static final long serialVersionUID = 7190970299538893497L; private static final Logger LOG = LoggerFactory.getLogger(DirtyDataManager.class); - private static final String DIRTY_BLOCK_STR = "blockingInterval"; - private static final String DIRTY_LIMIT_RATE_STR = "errorLimitRate"; private final static int MAX_TASK_QUEUE_SIZE = 100; - private final static String DEFAULT_ERROR_LIMIT_RATE = "0.8"; - private final static String DEFAULT_BLOCKING_INTERVAL = "60"; public static AbstractDirtyDataConsumer consumer; private static ThreadPoolExecutor dirtyDataConsumer; @@ -77,14 +84,14 @@ public static DirtyDataManager newInstance(Properties properties) { manager.blockingInterval = Long.parseLong(String.valueOf(properties.getOrDefault(DIRTY_BLOCK_STR, DEFAULT_BLOCKING_INTERVAL))); manager.errorLimitRate = Double.parseDouble(String.valueOf(properties.getOrDefault(DIRTY_LIMIT_RATE_STR, DEFAULT_ERROR_LIMIT_RATE))); consumer = DirtyConsumerFactory.getDirtyConsumer( - properties.getProperty("type") - , properties.getProperty("pluginPath") - , properties.getProperty("pluginLoadMode") + properties.getProperty(PLUGIN_TYPE_STR, DEFAULT_TYPE) + , properties.getProperty(PLUGIN_PATH_STR) + , properties.getProperty(PLUGIN_LOAD_MODE_STR) ); consumer.init(properties); consumer.setQueue(new LinkedBlockingQueue<>()); dirtyDataConsumer = new ThreadPoolExecutor(MAX_POOL_SIZE_LIMIT, MAX_POOL_SIZE_LIMIT, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(MAX_TASK_QUEUE_SIZE), new DTThreadFactory("dirtyDataConsumer"), new ThreadPoolExecutor.CallerRunsPolicy()); + new LinkedBlockingQueue<>(MAX_TASK_QUEUE_SIZE), new DTThreadFactory("dirtyDataConsumer", true), new ThreadPoolExecutor.CallerRunsPolicy()); dirtyDataConsumer.execute(consumer); return manager; } catch (Exception e) { @@ -99,8 +106,8 @@ public static DirtyDataManager newInstance(Properties properties) { */ public static String buildDefaultDirty() { JSONObject jsonObject = new JSONObject(); - jsonObject.put("type", "console"); - jsonObject.put("printLimit", "1000"); + jsonObject.put(PLUGIN_TYPE_STR, DEFAULT_TYPE); + jsonObject.put(PRINT_LIMIT_STR, DEFAULT_PRINT_LIMIT); return jsonObject.toJSONString(); } @@ -128,6 +135,8 @@ public void collectDirtyData(String dataInfo, String cause) { LOG.warn("dirty Data insert error ... Failed number: " + errorCount.incrementAndGet()); LOG.warn("error dirty data:" + dirtyDataEntity.toString()); if (errorCount.get() > Math.ceil(count.longValue() * errorLimitRate)) { + // close consumer and manager + close(); throw new RuntimeException(String.format("The number of failed number 【%s】 reaches the limit, manager fails", errorCount.get())); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyKeys.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyKeys.java new file mode 100644 index 000000000..9fc134690 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/DirtyKeys.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.dirtyManager.manager; + +/** + * Date: 2021/1/6 + * + * @author tiezhu + * Company dtstack + */ +public class DirtyKeys { + public final static String DEFAULT_TYPE = "console"; + public final static String DEFAULT_BLOCKING_INTERVAL = "60"; + public final static String DEFAULT_ERROR_LIMIT_RATE = "0.8"; + public final static String DEFAULT_PRINT_LIMIT = "1000"; + + public final static String DIRTY_BLOCK_STR = "blockingInterval"; + public final static String DIRTY_LIMIT_RATE_STR = "errorLimitRate"; + public final static String PLUGIN_TYPE_STR = "type"; + public final static String PLUGIN_PATH_STR = "pluginPath"; + public final static String PLUGIN_LOAD_MODE_STR = "pluginLoadMode"; + + public final static String PRINT_LIMIT_STR = "printLimit"; +} diff --git a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java b/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java deleted file mode 100644 index 5d8f38382..000000000 --- a/core/src/main/java/com/dtstack/flink/sql/dirtyManager/manager/TestMain.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.dtstack.flink.sql.dirtyManager.manager; - -import java.util.Properties; - -/** - * @author tiezhu - * Company dtstack - * Date 2020/8/28 星期五 - */ -public class TestMain { - private static final Integer DATA_NUMBER = 1000; - - public static void main(String[] args) throws Exception { - Properties properties = new Properties(); - properties.put("type", "console"); - properties.put("pluginPath", "/Users/wtz/IdeaProjects/flinkStreamSQLDemoThree/sqlplugins"); - properties.put("url", "jdbc:mysql://localhost:3306/tiezhu"); - properties.put("userName", "root"); - properties.put("password", "abc123"); - properties.put("isCreatedTable", "false"); - properties.put("batchSize", "1"); - properties.put("tableName", "DirtyDataFromMysql_2020_09_14_10_51_50"); - - DirtyDataManager manager = DirtyDataManager.newInstance(properties); - for (int i = 0; i < DATA_NUMBER; i++) { - Thread.sleep(100); - manager.collectDirtyData("testDirtyData" + i, - new Exception("testException" + i).getMessage()); - if (i == 50) { - manager.close(); - } - } - } -} diff --git a/core/src/main/java/com/dtstack/flink/sql/factory/DTThreadFactory.java b/core/src/main/java/com/dtstack/flink/sql/factory/DTThreadFactory.java index e9f9dfa64..ec4065f40 100644 --- a/core/src/main/java/com/dtstack/flink/sql/factory/DTThreadFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/factory/DTThreadFactory.java @@ -16,7 +16,6 @@ * limitations under the License. */ - package com.dtstack.flink.sql.factory; @@ -33,6 +32,7 @@ public class DTThreadFactory implements ThreadFactory { private final static AtomicInteger THREAD_NUMBER = new AtomicInteger(1); private final ThreadGroup group; private final String namePrefix; + private Boolean isDaemon = false; public DTThreadFactory(String factoryName) { SecurityManager s = System.getSecurityManager(); @@ -43,14 +43,29 @@ public DTThreadFactory(String factoryName) { "-thread-"; } + public DTThreadFactory(String factoryName, Boolean isDaemon) { + SecurityManager s = System.getSecurityManager(); + group = (s != null) ? s.getThreadGroup() : + Thread.currentThread().getThreadGroup(); + namePrefix = factoryName + "-pool-" + + POOL_NUMBER.getAndIncrement() + + "-thread-"; + this.isDaemon = isDaemon; + } + @Override public Thread newThread(Runnable r) { Thread t = new Thread(group, r, namePrefix + THREAD_NUMBER.getAndIncrement(), 0); - if (t.isDaemon()) { - t.setDaemon(false); + if (this.isDaemon) { + t.setDaemon(true); + } else { + if (t.isDaemon()) { + t.setDaemon(false); + } } + if (t.getPriority() != Thread.NORM_PRIORITY) { t.setPriority(Thread.NORM_PRIORITY); } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 69221b4be..085ef7613 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -69,7 +69,6 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map1.0-SNAPSHOT--> + + com.dtstack.flink + dirtyConsumer.console + 1.0-SNAPSHOT + + + + com.dtstack.flink + dirtyConsumer.mysql + 1.0-SNAPSHOT + + com.dtstack.flink sql.source.kafka11 From 10fbea1a2dc416dbcbe029c613fa35d8a2fb023c Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 7 Jan 2021 15:32:36 +0800 Subject: [PATCH 51/80] [hotfix-31811][core][launcher]merge flinkStreamSql v1.10 into flinkStreamSql v1.11.1 conflict --- core/pom.xml | 5 - .../flink/sql/side/TupleKeySelector.java | 19 +- .../sql/side/operator/SideAsyncOperator.java | 22 +- dirtyData/console/pom.xml | 5 - .../PreRowKeyModeDealerDealer.java | 2 +- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 1 - .../planner/plan/QueryOperationConverter.java | 652 ------------------ .../flink/sql/localTest/LocalTest.java | 2 +- .../sql/side/redis/RedisAsyncReqRow.java | 12 +- .../sql/side/redis/table/RedisSideReqRow.java | 13 +- 10 files changed, 34 insertions(+), 699 deletions(-) delete mode 100644 launcher/src/main/java/org/apache/flink/table/planner/plan/QueryOperationConverter.java diff --git a/core/pom.xml b/core/pom.xml index 0aa111b9f..dc827f2ba 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -108,11 +108,6 @@ joda-time 2.5 - - org.slf4j - slf4j-simple - 1.7.30 - log4j diff --git a/core/src/main/java/com/dtstack/flink/sql/side/TupleKeySelector.java b/core/src/main/java/com/dtstack/flink/sql/side/TupleKeySelector.java index dd7f617ac..03c2da3c4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/TupleKeySelector.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/TupleKeySelector.java @@ -21,29 +21,30 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.typeutils.ResultTypeQueryable; -import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.table.dataformat.GenericRow; +import org.apache.flink.types.Row; /** * Date: 2020/3/25 * Company: www.dtstack.com * @author maqi */ -public class TupleKeySelector implements ResultTypeQueryable, KeySelector { +public class TupleKeySelector implements ResultTypeQueryable, KeySelector { - private TypeInformation returnType; + private int[] keyFields; + private TypeInformation returnType; - public TupleKeySelector(TypeInformation returnType) { + public TupleKeySelector(int[] keyFields, TypeInformation returnType) { + this.keyFields = keyFields; this.returnType = returnType; } @Override - public BaseRow getKey(BaseRow value) throws Exception { - return GenericRow.copyReference((GenericRow)value); + public Row getKey(Row value) throws Exception { + return Row.project(value, keyFields); } @Override - public TypeInformation getProducedType() { + public TypeInformation getProducedType() { return returnType; } -} +} \ No newline at end of file diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java index b1f9264e5..a80226b86 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java @@ -20,15 +20,14 @@ package com.dtstack.flink.sql.side.operator; import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.AsyncDataStream; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.table.dataformat.BaseRow; import java.util.List; import java.util.concurrent.TimeUnit; @@ -50,8 +49,13 @@ public class SideAsyncOperator { private static final String ORDERED = "ordered"; - private static BaseAsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, - List outFieldInfoList, AbstractSideTableInfo sideTableInfo, String pluginLoadMode) throws Exception { + private static BaseAsyncReqRow loadAsyncReq(String sideType + , String sqlRootDir + , RowTypeInfo rowTypeInfo + , JoinInfo joinInfo + , List outFieldInfoList + , AbstractSideTableInfo sideTableInfo + , String pluginLoadMode) throws Exception { String pathOfType = String.format(PATH_FORMAT, sideType); String pluginJarPath = PluginUtil.getJarFileDirPath(pathOfType, sqlRootDir, pluginLoadMode); String className = PluginUtil.getSqlSideClassName(sideType, "side", OPERATOR_TYPE); @@ -61,8 +65,14 @@ private static BaseAsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } - public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, - List outFieldInfoList, AbstractSideTableInfo sideTableInfo, String pluginLoadMode) throws Exception { + public static DataStream getSideJoinDataStream(DataStream inputStream + , String sideType + , String sqlRootDir + , RowTypeInfo rowTypeInfo + , JoinInfo joinInfo + , List outFieldInfoList + , AbstractSideTableInfo sideTableInfo + , String pluginLoadMode) throws Exception { BaseAsyncReqRow asyncDbReq = loadAsyncReq(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo, pluginLoadMode); //TODO How much should be set for the degree of parallelism? Timeout? capacity settings? diff --git a/dirtyData/console/pom.xml b/dirtyData/console/pom.xml index a9d0c1fc5..d65d0284c 100644 --- a/dirtyData/console/pom.xml +++ b/dirtyData/console/pom.xml @@ -80,11 +80,6 @@ 4.8.2 test - - org.slf4j - slf4j-simple - 1.7.30 - log4j diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java index 21b51a6de..b250d840c 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java @@ -33,7 +33,6 @@ import org.apache.calcite.sql.JoinType; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.types.Row; -import org.apache.flink.table.dataformat.BaseRow; import org.hbase.async.BinaryPrefixComparator; import org.hbase.async.Bytes; import org.hbase.async.CompareFilter; @@ -46,6 +45,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index 9f63131c2..7b65c7f4d 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -19,7 +19,6 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.dataformat.BaseRow; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; import org.apache.hadoop.security.UserGroupInformation; diff --git a/launcher/src/main/java/org/apache/flink/table/planner/plan/QueryOperationConverter.java b/launcher/src/main/java/org/apache/flink/table/planner/plan/QueryOperationConverter.java deleted file mode 100644 index e942d81b3..000000000 --- a/launcher/src/main/java/org/apache/flink/table/planner/plan/QueryOperationConverter.java +++ /dev/null @@ -1,652 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.table.planner.plan; - -import org.apache.flink.annotation.Internal; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.table.api.TableException; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.catalog.CatalogManager; -import org.apache.flink.table.catalog.ConnectorCatalogTable; -import org.apache.flink.table.catalog.FunctionLookup; -import org.apache.flink.table.catalog.ObjectIdentifier; -import org.apache.flink.table.catalog.UnresolvedIdentifier; -import org.apache.flink.table.expressions.CallExpression; -import org.apache.flink.table.expressions.Expression; -import org.apache.flink.table.expressions.ExpressionDefaultVisitor; -import org.apache.flink.table.expressions.FieldReferenceExpression; -import org.apache.flink.table.expressions.ResolvedExpression; -import org.apache.flink.table.expressions.ValueLiteralExpression; -import org.apache.flink.table.expressions.resolver.LookupCallResolver; -import org.apache.flink.table.functions.BuiltInFunctionDefinitions; -import org.apache.flink.table.functions.FunctionDefinition; -import org.apache.flink.table.functions.FunctionIdentifier; -import org.apache.flink.table.functions.TableFunction; -import org.apache.flink.table.operations.AggregateQueryOperation; -import org.apache.flink.table.operations.CalculatedQueryOperation; -import org.apache.flink.table.operations.CatalogQueryOperation; -import org.apache.flink.table.operations.DistinctQueryOperation; -import org.apache.flink.table.operations.FilterQueryOperation; -import org.apache.flink.table.operations.JavaDataStreamQueryOperation; -import org.apache.flink.table.operations.JoinQueryOperation; -import org.apache.flink.table.operations.JoinQueryOperation.JoinType; -import org.apache.flink.table.operations.ProjectQueryOperation; -import org.apache.flink.table.operations.QueryOperation; -import org.apache.flink.table.operations.QueryOperationVisitor; -import org.apache.flink.table.operations.ScalaDataStreamQueryOperation; -import org.apache.flink.table.operations.SetQueryOperation; -import org.apache.flink.table.operations.SortQueryOperation; -import org.apache.flink.table.operations.TableSourceQueryOperation; -import org.apache.flink.table.operations.WindowAggregateQueryOperation; -import org.apache.flink.table.operations.WindowAggregateQueryOperation.ResolvedGroupWindow; -import org.apache.flink.table.operations.utils.QueryOperationDefaultVisitor; -import org.apache.flink.table.planner.calcite.FlinkContext; -import org.apache.flink.table.planner.calcite.FlinkRelBuilder; -import org.apache.flink.table.planner.calcite.FlinkTypeFactory; -import org.apache.flink.table.planner.expressions.PlannerProctimeAttribute; -import org.apache.flink.table.planner.expressions.PlannerRowtimeAttribute; -import org.apache.flink.table.planner.expressions.PlannerWindowEnd; -import org.apache.flink.table.planner.expressions.PlannerWindowReference; -import org.apache.flink.table.planner.expressions.PlannerWindowStart; -import org.apache.flink.table.planner.expressions.RexNodeExpression; -import org.apache.flink.table.planner.expressions.SqlAggFunctionVisitor; -import org.apache.flink.table.planner.expressions.converter.ExpressionConverter; -import org.apache.flink.table.planner.functions.utils.TableSqlFunction; -import org.apache.flink.table.planner.operations.DataStreamQueryOperation; -import org.apache.flink.table.planner.operations.PlannerQueryOperation; -import org.apache.flink.table.planner.operations.RichTableSourceQueryOperation; -import org.apache.flink.table.planner.plan.logical.LogicalWindow; -import org.apache.flink.table.planner.plan.logical.SessionGroupWindow; -import org.apache.flink.table.planner.plan.logical.SlidingGroupWindow; -import org.apache.flink.table.planner.plan.logical.TumblingGroupWindow; -import org.apache.flink.table.planner.plan.schema.DataStreamTable; -import org.apache.flink.table.planner.plan.schema.DataStreamTable$; -import org.apache.flink.table.planner.plan.schema.TableSourceTable; -import org.apache.flink.table.planner.plan.schema.TypedFlinkTableFunction; -import org.apache.flink.table.planner.plan.stats.FlinkStatistic; -import org.apache.flink.table.planner.sources.TableSourceUtil; -import org.apache.flink.table.sources.LookupableTableSource; -import org.apache.flink.table.sources.StreamTableSource; -import org.apache.flink.table.sources.TableSource; -import org.apache.flink.table.types.DataType; -import org.apache.flink.util.Preconditions; - -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.CorrelationId; -import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.logical.LogicalTableFunctionScan; -import org.apache.calcite.rel.logical.LogicalTableScan; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.SqlAggFunction; -import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.tools.RelBuilder.AggCall; -import org.apache.calcite.tools.RelBuilder.GroupKey; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import scala.Some; - -import static java.util.Arrays.asList; -import static java.util.stream.Collectors.toList; -import static org.apache.flink.table.expressions.ExpressionUtils.extractValue; -import static org.apache.flink.table.expressions.utils.ApiExpressionUtils.isFunctionOfKind; -import static org.apache.flink.table.functions.BuiltInFunctionDefinitions.AS; -import static org.apache.flink.table.functions.FunctionKind.AGGREGATE; -import static org.apache.flink.table.functions.FunctionKind.TABLE_AGGREGATE; -import static org.apache.flink.table.types.utils.TypeConversions.fromDataToLogicalType; -import static org.apache.flink.table.types.utils.TypeConversions.fromLegacyInfoToDataType; - -/** - * Converter from Flink's specific relational representation: {@link QueryOperation} to Calcite's specific relational - * representation: {@link RelNode}. - */ -@Internal -public class QueryOperationConverter extends QueryOperationDefaultVisitor { - - private final FlinkRelBuilder relBuilder; - private final SingleRelVisitor singleRelVisitor = new SingleRelVisitor(); - private final LookupCallResolver callResolver; - private final ExpressionConverter expressionConverter; - private final AggregateVisitor aggregateVisitor = new AggregateVisitor(); - private final TableAggregateVisitor tableAggregateVisitor = new TableAggregateVisitor(); - private final JoinExpressionVisitor joinExpressionVisitor = new JoinExpressionVisitor(); - - public QueryOperationConverter(FlinkRelBuilder relBuilder, FunctionLookup functionCatalog) { - this.relBuilder = relBuilder; - this.callResolver = new LookupCallResolver(functionCatalog); - this.expressionConverter = new ExpressionConverter(relBuilder); - } - - @Override - public RelNode defaultMethod(QueryOperation other) { - other.getChildren().forEach(child -> relBuilder.push(child.accept(this))); - return other.accept(singleRelVisitor); - } - - private class SingleRelVisitor implements QueryOperationVisitor { - - @Override - public RelNode visit(ProjectQueryOperation projection) { - List rexNodes = convertToRexNodes(projection.getProjectList()); - - return relBuilder.project(rexNodes, asList(projection.getTableSchema().getFieldNames()), true).build(); - } - - @Override - public RelNode visit(AggregateQueryOperation aggregate) { - List aggregations = aggregate.getAggregateExpressions() - .stream() - .map(this::getAggCall) - .collect(toList()); - - List groupings = convertToRexNodes(aggregate.getGroupingExpressions()); - GroupKey groupKey = relBuilder.groupKey(groupings); - return relBuilder.aggregate(groupKey, aggregations).build(); - } - - @Override - public RelNode visit(WindowAggregateQueryOperation windowAggregate) { - List aggregations = windowAggregate.getAggregateExpressions() - .stream() - .map(this::getAggCall) - .collect(toList()); - - List groupings = convertToRexNodes(windowAggregate.getGroupingExpressions()); - LogicalWindow logicalWindow = toLogicalWindow(windowAggregate.getGroupWindow()); - PlannerWindowReference windowReference = logicalWindow.aliasAttribute(); - List windowProperties = windowAggregate - .getWindowPropertiesExpressions() - .stream() - .map(expr -> convertToWindowProperty(expr.accept(callResolver), windowReference)) - .collect(toList()); - GroupKey groupKey = relBuilder.groupKey(groupings); - return relBuilder.windowAggregate(logicalWindow, groupKey, windowProperties, aggregations).build(); - } - - private FlinkRelBuilder.PlannerNamedWindowProperty convertToWindowProperty(Expression expression, - PlannerWindowReference windowReference) { - Preconditions.checkArgument(expression instanceof CallExpression, "This should never happened"); - CallExpression aliasExpr = (CallExpression) expression; - Preconditions.checkArgument( - BuiltInFunctionDefinitions.AS == aliasExpr.getFunctionDefinition(), - "This should never happened"); - String name = ((ValueLiteralExpression) aliasExpr.getChildren().get(1)).getValueAs(String.class) - .orElseThrow( - () -> new TableException("Invalid literal.")); - Expression windowPropertyExpr = aliasExpr.getChildren().get(0); - Preconditions.checkArgument(windowPropertyExpr instanceof CallExpression, "This should never happened"); - CallExpression windowPropertyCallExpr = (CallExpression) windowPropertyExpr; - FunctionDefinition fd = windowPropertyCallExpr.getFunctionDefinition(); - if (BuiltInFunctionDefinitions.WINDOW_START == fd) { - return new FlinkRelBuilder.PlannerNamedWindowProperty(name, new PlannerWindowStart(windowReference)); - } else if (BuiltInFunctionDefinitions.WINDOW_END == fd) { - return new FlinkRelBuilder.PlannerNamedWindowProperty(name, new PlannerWindowEnd(windowReference)); - } else if (BuiltInFunctionDefinitions.PROCTIME == fd) { - return new FlinkRelBuilder.PlannerNamedWindowProperty(name, new PlannerProctimeAttribute(windowReference)); - } else if (BuiltInFunctionDefinitions.ROWTIME == fd) { - return new FlinkRelBuilder.PlannerNamedWindowProperty(name, new PlannerRowtimeAttribute(windowReference)); - } else { - throw new TableException("Invalid literal."); - } - } - - /** - * Get the {@link AggCall} correspond to the aggregate or table aggregate expression. - */ - private AggCall getAggCall(Expression aggregateExpression) { - if (isFunctionOfKind(aggregateExpression, TABLE_AGGREGATE)) { - return aggregateExpression.accept(tableAggregateVisitor); - } else { - return aggregateExpression.accept(aggregateVisitor); - } - } - - @Override - public RelNode visit(JoinQueryOperation join) { - final Set corSet; - if (join.isCorrelated()) { - corSet = Collections.singleton(relBuilder.peek().getCluster().createCorrel()); - } else { - corSet = Collections.emptySet(); - } - - return relBuilder.join( - convertJoinType(join.getJoinType()), - join.getCondition().accept(joinExpressionVisitor), - corSet) - .build(); - } - - @Override - public RelNode visit(SetQueryOperation setOperation) { - switch (setOperation.getType()) { - case INTERSECT: - relBuilder.intersect(setOperation.isAll()); - break; - case MINUS: - relBuilder.minus(setOperation.isAll()); - break; - case UNION: - relBuilder.union(setOperation.isAll()); - break; - } - return relBuilder.build(); - } - - @Override - public RelNode visit(FilterQueryOperation filter) { - RexNode rexNode = convertExprToRexNode(filter.getCondition()); - return relBuilder.filter(rexNode).build(); - } - - @Override - public RelNode visit(DistinctQueryOperation distinct) { - return relBuilder.distinct().build(); - } - - @Override - public RelNode visit(SortQueryOperation sort) { - List rexNodes = convertToRexNodes(sort.getOrder()); - return relBuilder.sortLimit(sort.getOffset(), sort.getFetch(), rexNodes) - .build(); - } - - @Override - public RelNode visit(CalculatedQueryOperation calculatedTable) { - DataType resultType = fromLegacyInfoToDataType(calculatedTable.getResultType()); - TableFunction tableFunction = calculatedTable.getTableFunction(); - String[] fieldNames = calculatedTable.getTableSchema().getFieldNames(); - - TypedFlinkTableFunction function = new TypedFlinkTableFunction( - tableFunction, fieldNames, resultType); - - FlinkTypeFactory typeFactory = relBuilder.getTypeFactory(); - - TableSqlFunction sqlFunction = new TableSqlFunction( - FunctionIdentifier.of(tableFunction.functionIdentifier()), - tableFunction.toString(), - tableFunction, - resultType, - typeFactory, - function, - scala.Option.empty()); - - List parameters = convertToRexNodes(calculatedTable.getParameters()); - - return LogicalTableFunctionScan.create( - relBuilder.peek().getCluster(), - Collections.emptyList(), - relBuilder.call(sqlFunction, parameters), - function.getElementType(null), - function.getRowType(typeFactory, null, null), - null); - } - - @Override - public RelNode visit(CatalogQueryOperation catalogTable) { - ObjectIdentifier objectIdentifier = catalogTable.getTableIdentifier(); - return relBuilder.scan( - objectIdentifier.getCatalogName(), - objectIdentifier.getDatabaseName(), - objectIdentifier.getObjectName() - ).build(); - } - - @Override - public RelNode visit(QueryOperation other) { - if (other instanceof PlannerQueryOperation) { - return ((PlannerQueryOperation) other).getCalciteTree(); - } else if (other instanceof DataStreamQueryOperation) { - return convertToDataStreamScan((DataStreamQueryOperation) other); - } else if (other instanceof JavaDataStreamQueryOperation) { - JavaDataStreamQueryOperation dataStreamQueryOperation = (JavaDataStreamQueryOperation) other; - return convertToDataStreamScan( - dataStreamQueryOperation.getDataStream(), - dataStreamQueryOperation.getFieldIndices(), - dataStreamQueryOperation.getTableSchema(), - dataStreamQueryOperation.getIdentifier()); - } else if (other instanceof ScalaDataStreamQueryOperation) { - ScalaDataStreamQueryOperation dataStreamQueryOperation = (ScalaDataStreamQueryOperation) other; - return convertToDataStreamScan( - dataStreamQueryOperation.getDataStream(), - dataStreamQueryOperation.getFieldIndices(), - dataStreamQueryOperation.getTableSchema(), - dataStreamQueryOperation.getIdentifier()); - } - - throw new TableException("Unknown table operation: " + other); - } - - @Override - public RelNode visit(TableSourceQueryOperation tableSourceOperation) { - TableSource tableSource = tableSourceOperation.getTableSource(); - boolean isBatch; - if (tableSource instanceof LookupableTableSource) { - isBatch = tableSourceOperation.isBatch(); - } else if (tableSource instanceof StreamTableSource) { - isBatch = ((StreamTableSource) tableSource).isBounded(); - } else { - throw new TableException(String.format("%s is not supported.", tableSource.getClass().getSimpleName())); - } - - FlinkStatistic statistic; - ObjectIdentifier tableIdentifier; - if (tableSourceOperation instanceof RichTableSourceQueryOperation && - ((RichTableSourceQueryOperation) tableSourceOperation).getIdentifier() != null) { - tableIdentifier = ((RichTableSourceQueryOperation) tableSourceOperation).getIdentifier(); - statistic = ((RichTableSourceQueryOperation) tableSourceOperation).getStatistic(); - } else { - statistic = FlinkStatistic.UNKNOWN(); - // TableSourceScan requires a unique name of a Table for computing a digest. - // We are using the identity hash of the TableSource object. - String refId = "Unregistered_TableSource_" + System.identityHashCode(tableSource); - CatalogManager catalogManager = relBuilder.getCluster().getPlanner().getContext() - .unwrap(FlinkContext.class).getCatalogManager(); - tableIdentifier = catalogManager.qualifyIdentifier(UnresolvedIdentifier.of(refId)); - } - - RelDataType rowType = TableSourceUtil.getSourceRowType(relBuilder.getTypeFactory(), - tableSourceOperation.getTableSchema(), - scala.Option.apply(tableSource), - !isBatch); - TableSourceTable tableSourceTable = new TableSourceTable<>( - relBuilder.getRelOptSchema(), - tableIdentifier, - rowType, - statistic, - tableSource, - !isBatch, - ConnectorCatalogTable.source(tableSource, isBatch)); - return LogicalTableScan.create(relBuilder.getCluster(), tableSourceTable); - } - - private RelNode convertToDataStreamScan(DataStreamQueryOperation operation) { - List names; - ObjectIdentifier identifier = operation.getIdentifier(); - if (identifier != null) { - names = Arrays.asList( - identifier.getCatalogName(), - identifier.getDatabaseName(), - identifier.getObjectName()); - } else { - String refId = String.format("Unregistered_DataStream_%s", operation.getDataStream().getId()); - names = Collections.singletonList(refId); - } - - final RelDataType rowType = DataStreamTable$.MODULE$ - .getRowType(relBuilder.getTypeFactory(), - operation.getDataStream(), - operation.getTableSchema().getFieldNames(), - operation.getFieldIndices(), - scala.Option.apply(operation.getFieldNullables())); - DataStreamTable dataStreamTable = new DataStreamTable<>( - relBuilder.getRelOptSchema(), - names, - rowType, - operation.getDataStream(), - operation.isProducesUpdates(), - operation.isAccRetract(), - operation.getFieldIndices(), - operation.getTableSchema().getFieldNames(), - operation.getStatistic(), - scala.Option.apply(operation.getFieldNullables())); - return LogicalTableScan.create(relBuilder.getCluster(), dataStreamTable); - } - - private RelNode convertToDataStreamScan( - DataStream dataStream, - int[] fieldIndices, - TableSchema tableSchema, - Optional identifier) { - List names; - if (identifier.isPresent()) { - names = Arrays.asList( - identifier.get().getCatalogName(), - identifier.get().getDatabaseName(), - identifier.get().getObjectName()); - } else { - String refId = String.format("Unregistered_DataStream_%s", dataStream.getId()); - names = Collections.singletonList(refId); - } - final RelDataType rowType = DataStreamTable$.MODULE$ - .getRowType(relBuilder.getTypeFactory(), - dataStream, - tableSchema.getFieldNames(), - fieldIndices, - scala.Option.empty()); - DataStreamTable dataStreamTable = new DataStreamTable<>( - relBuilder.getRelOptSchema(), - names, - rowType, - dataStream, - false, - true, - fieldIndices, - tableSchema.getFieldNames(), - FlinkStatistic.UNKNOWN(), - scala.Option.empty()); - return LogicalTableScan.create(relBuilder.getCluster(), dataStreamTable); - } - - private List convertToRexNodes(List expressions) { - return expressions - .stream() - .map(QueryOperationConverter.this::convertExprToRexNode) - .collect(toList()); - } - - private LogicalWindow toLogicalWindow(ResolvedGroupWindow window) { - DataType windowType = window.getTimeAttribute().getOutputDataType(); - PlannerWindowReference windowReference = new PlannerWindowReference(window.getAlias(), - new Some<>(fromDataToLogicalType(windowType))); - switch (window.getType()) { - case SLIDE: - return new SlidingGroupWindow( - windowReference, - window.getTimeAttribute(), - window.getSize().orElseThrow(() -> new TableException("missed size parameters!")), - window.getSlide().orElseThrow(() -> new TableException("missed slide parameters!")) - ); - case SESSION: - return new SessionGroupWindow( - windowReference, - window.getTimeAttribute(), - window.getGap().orElseThrow(() -> new TableException("missed gap parameters!")) - ); - case TUMBLE: - return new TumblingGroupWindow( - windowReference, - window.getTimeAttribute(), - window.getSize().orElseThrow(() -> new TableException("missed size parameters!")) - ); - default: - throw new TableException("Unknown window type"); - } - } - - private JoinRelType convertJoinType(JoinType joinType) { - switch (joinType) { - case INNER: - return JoinRelType.INNER; - case LEFT_OUTER: - return JoinRelType.LEFT; - case RIGHT_OUTER: - return JoinRelType.RIGHT; - case FULL_OUTER: - return JoinRelType.FULL; - default: - throw new TableException("Unknown join type: " + joinType); - } - } - } - - private class JoinExpressionVisitor extends ExpressionDefaultVisitor { - - private static final int numberOfJoinInputs = 2; - - @Override - public RexNode visit(CallExpression callExpression) { - final List newChildren = callExpression.getChildren().stream().map(expr -> { - RexNode convertedNode = expr.accept(this); - return new RexNodeExpression(convertedNode, ((ResolvedExpression) expr).getOutputDataType()); - }).collect(Collectors.toList()); - - CallExpression newCall; - if (callExpression.getFunctionIdentifier().isPresent()) { - newCall = new CallExpression( - callExpression.getFunctionIdentifier().get(), callExpression.getFunctionDefinition(), newChildren, - callExpression.getOutputDataType()); - } else { - newCall = new CallExpression( - callExpression.getFunctionDefinition(), newChildren, callExpression.getOutputDataType()); - } - return convertExprToRexNode(newCall); - } - - @Override - public RexNode visit(FieldReferenceExpression fieldReference) { - return relBuilder.field(numberOfJoinInputs, fieldReference.getInputIndex(), fieldReference.getFieldIndex()); - } - - @Override - protected RexNode defaultMethod(Expression expression) { - return convertExprToRexNode(expression); - } - } - - private class AggregateVisitor extends ExpressionDefaultVisitor { - - @Override - public AggCall visit(CallExpression unresolvedCall) { - if (unresolvedCall.getFunctionDefinition() == AS) { - String aggregateName = extractValue(unresolvedCall.getChildren().get(1), String.class) - .orElseThrow(() -> new TableException("Unexpected name.")); - - Expression aggregate = unresolvedCall.getChildren().get(0); - if (isFunctionOfKind(aggregate, AGGREGATE)) { - return aggregate.accept(callResolver).accept( - new AggCallVisitor(relBuilder, expressionConverter, aggregateName, false)); - } - } - throw new TableException("Expected named aggregate. Got: " + unresolvedCall); - } - - @Override - protected AggCall defaultMethod(Expression expression) { - throw new TableException("Unexpected expression: " + expression); - } - - private class AggCallVisitor extends ExpressionDefaultVisitor { - - private final RelBuilder relBuilder; - private final SqlAggFunctionVisitor sqlAggFunctionVisitor; - private final ExpressionConverter expressionConverter; - private final String name; - private final boolean isDistinct; - - public AggCallVisitor(RelBuilder relBuilder, ExpressionConverter expressionConverter, String name, - boolean isDistinct) { - this.relBuilder = relBuilder; - this.sqlAggFunctionVisitor = new SqlAggFunctionVisitor((FlinkTypeFactory) relBuilder.getTypeFactory()); - this.expressionConverter = expressionConverter; - this.name = name; - this.isDistinct = isDistinct; - } - - @Override - public RelBuilder.AggCall visit(CallExpression call) { - FunctionDefinition def = call.getFunctionDefinition(); - if (BuiltInFunctionDefinitions.DISTINCT == def) { - Expression innerAgg = call.getChildren().get(0); - return innerAgg.accept(new AggCallVisitor(relBuilder, expressionConverter, name, true)); - } else { - SqlAggFunction sqlAggFunction = call.accept(sqlAggFunctionVisitor); - return relBuilder.aggregateCall( - sqlAggFunction, - isDistinct, - false, - null, - name, - call.getChildren().stream().map(expr -> expr.accept(expressionConverter)) - .collect(Collectors.toList())); - } - } - - @Override - protected RelBuilder.AggCall defaultMethod(Expression expression) { - throw new TableException("Unexpected expression: " + expression); - } - } - } - - private class TableAggregateVisitor extends ExpressionDefaultVisitor { - @Override - public AggCall visit(CallExpression call) { - if (isFunctionOfKind(call, TABLE_AGGREGATE)) { - return call.accept(new TableAggCallVisitor(relBuilder, expressionConverter)); - } - return defaultMethod(call); - } - - @Override - protected AggCall defaultMethod(Expression expression) { - throw new TableException("Expected table aggregate. Got: " + expression); - } - - private class TableAggCallVisitor extends ExpressionDefaultVisitor { - - private final RelBuilder relBuilder; - private final SqlAggFunctionVisitor sqlAggFunctionVisitor; - private final ExpressionConverter expressionConverter; - - public TableAggCallVisitor(RelBuilder relBuilder, ExpressionConverter expressionConverter) { - this.relBuilder = relBuilder; - this.sqlAggFunctionVisitor = new SqlAggFunctionVisitor((FlinkTypeFactory) relBuilder.getTypeFactory()); - this.expressionConverter = expressionConverter; - } - - @Override - public RelBuilder.AggCall visit(CallExpression call) { - SqlAggFunction sqlAggFunction = call.accept(sqlAggFunctionVisitor); - return relBuilder.aggregateCall( - sqlAggFunction, - false, - false, - null, - sqlAggFunction.toString(), - call.getChildren().stream().map(expr -> expr.accept(expressionConverter)).collect(toList())); - } - - @Override - protected RelBuilder.AggCall defaultMethod(Expression expression) { - throw new TableException("Expected table aggregate. Got: " + expression); - } - } - } - - private RexNode convertExprToRexNode(Expression expr) { - return expr.accept(callResolver).accept(expressionConverter); - } -} diff --git a/localTest/src/main/java/com/dtstack/flink/sql/localTest/LocalTest.java b/localTest/src/main/java/com/dtstack/flink/sql/localTest/LocalTest.java index dfb2f4157..d4f831d3e 100644 --- a/localTest/src/main/java/com/dtstack/flink/sql/localTest/LocalTest.java +++ b/localTest/src/main/java/com/dtstack/flink/sql/localTest/LocalTest.java @@ -50,7 +50,7 @@ public static void main(String[] args) throws Exception { setLogLevel("INFO"); List propertiesList = new ArrayList<>(); - String sqlPath = "/Users/wtz/dtstack/sql/test/JoinDemoOne.sql"; + String sqlPath = "/Users/chuixue/Desktop/tmp/sqlFile.sql"; Map conf = new HashMap<>(); JSONObject properties = new JSONObject(); diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 96fb7f34f..e79144d31 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -18,19 +18,9 @@ package com.dtstack.flink.sql.side.redis; +import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.BaseAsyncReqRow; -import com.dtstack.flink.sql.util.RowDataComplete; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.async.RedisAsyncCommands; -import io.lettuce.core.cluster.api.async.RedisAdvancedClusterAsyncCommands; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.types.Row; - -import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java index d874d6dff..b8aa79771 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java @@ -21,9 +21,8 @@ import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.ISideReqRow; import com.dtstack.flink.sql.util.TableUtils; -import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.table.dataformat.GenericRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; import java.io.Serializable; import java.math.BigDecimal; @@ -54,13 +53,11 @@ public RedisSideReqRow(BaseSideInfo sideInfo, RedisSideTableInfo sideTableInfo) } @Override - public BaseRow fillData(BaseRow input, Object sideInput) { - GenericRow genericRow = (GenericRow) input; + public Row fillData(Row input, Object sideInput) { Map sideInputMap = (Map) sideInput; - GenericRow row = new GenericRow(sideInfo.getOutFieldInfoList().size()); - row.setHeader(input.getHeader()); + Row row = new Row(sideInfo.getOutFieldInfoList().size()); for(Map.Entry entry : sideInfo.getInFieldIndex().entrySet()){ - Object obj = genericRow.getField(entry.getValue()); + Object obj = input.getField(entry.getValue()); boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); if(obj instanceof LocalDateTime && isTimeIndicatorTypeInfo){ @@ -94,7 +91,7 @@ public String buildCacheKey(Map refData) { return keyBuilder.toString(); } - public void setRowField(GenericRow row, Integer index, BaseSideInfo sideInfo, String value) { + public void setRowField(Row row, Integer index, BaseSideInfo sideInfo, String value) { Integer keyIndex = sideInfo.getSideFieldIndex().get(index); String classType = sideInfo.getSideTableInfo().getFieldClassList().get(keyIndex).getName(); switch (classType){ From 787d30208ba91d1d3cfd45937423a138d3515285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Thu, 7 Jan 2021 17:53:31 +0800 Subject: [PATCH 52/80] =?UTF-8?q?[fix-34473][core]=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E5=85=A8=E5=B1=80=E5=B9=B6=E8=A1=8C=E5=BA=A6?= =?UTF-8?q?=E5=90=8E=EF=BC=8C=E5=85=A8=E9=87=8F=E7=BB=B4=E8=A1=A8=E5=8D=95?= =?UTF-8?q?=E7=8B=AC=E8=AE=BE=E7=BD=AE=E5=B9=B6=E8=A1=8C=E5=BA=A6=E4=B8=8D?= =?UTF-8?q?=E7=94=9F=E6=95=88=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/operator/SideWithAllCacheOperator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java index 18db27297..94901d8f9 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java @@ -60,6 +60,6 @@ private static BaseAllReqRow loadFlatMap(String sideType, String sqlRootDir, Row public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo, String pluginLoadMode) throws Exception { BaseAllReqRow allReqRow = loadFlatMap(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo, pluginLoadMode); - return inputStream.flatMap(allReqRow); + return inputStream.flatMap(allReqRow).setParallelism(sideTableInfo.getParallelism()); } } From 551c329a8ff3a920b40dc2d92fbedc68e0a07d90 Mon Sep 17 00:00:00 2001 From: chuixue Date: Mon, 11 Jan 2021 12:36:10 +0800 Subject: [PATCH 53/80] [hotfix-32959][core][launcher]fix group by window no retract --- core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java | 1 + .../flink/table/planner/plan/QueryOperationConverter.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 789677d35..a29c3a372 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -525,6 +525,7 @@ private void joinFun(Object pollObj, RowTypeInfo typeInfo = new RowTypeInfo(fieldDataTypes, targetTable.getSchema().getFieldNames()); DataStream adaptStream = tableEnv.toRetractStream(targetTable, typeInfo) + .filter(f -> f.f0) .map(f -> RowDataConvert.convertToBaseRow(f)); //join side table before keyby ===> Reducing the size of each dimension table cache of async diff --git a/launcher/src/main/java/org/apache/flink/table/planner/plan/QueryOperationConverter.java b/launcher/src/main/java/org/apache/flink/table/planner/plan/QueryOperationConverter.java index e942d81b3..6e1f15dce 100644 --- a/launcher/src/main/java/org/apache/flink/table/planner/plan/QueryOperationConverter.java +++ b/launcher/src/main/java/org/apache/flink/table/planner/plan/QueryOperationConverter.java @@ -442,7 +442,7 @@ private RelNode convertToDataStreamScan( rowType, dataStream, false, - true, + false, fieldIndices, tableSchema.getFieldNames(), FlinkStatistic.UNKNOWN(), From 7d60026703d8838ce0175d1c4d5bcce4bfa30348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Tue, 12 Jan 2021 15:53:36 +0800 Subject: [PATCH 54/80] =?UTF-8?q?[feat-34632]=20HBase=E6=94=AF=E6=8C=81Bat?= =?UTF-8?q?chSize?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/hbase/HbaseOutputFormat.java | 187 +++++++++--------- .../flink/sql/sink/hbase/HbaseSink.java | 48 +++-- 2 files changed, 120 insertions(+), 115 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index beda5cfbc..4db2903d9 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -19,12 +19,12 @@ package com.dtstack.flink.sql.sink.hbase; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.util.ExecutorThreadFactory; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; import org.apache.hadoop.hbase.AuthUtil; @@ -47,29 +47,25 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; /** * @author: jingzhen@dtstack.com * date: 2017-6-29 */ -public class HbaseOutputFormat extends AbstractDtRichOutputFormat { +public class HbaseOutputFormat extends AbstractDtRichOutputFormat> { private static final Logger LOG = LoggerFactory.getLogger(HbaseOutputFormat.class); - + private final List records = new ArrayList<>(); private String host; private String zkParent; private String rowkey; private String tableName; private String[] columnNames; - private String updateMode; - private String[] columnTypes; private Map columnNameFamily; - private boolean kerberosAuthEnable; private String regionserverKeytabFile; private String regionserverPrincipal; @@ -77,50 +73,44 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private String zookeeperSaslClient; private String clientPrincipal; private String clientKeytabFile; - private String[] families; private String[] qualifiers; - private transient org.apache.hadoop.conf.Configuration conf; private transient Connection conn; private transient Table table; - private transient ChoreService choreService; - + /** + * 批量写入的参数 + */ private Integer batchSize; private Long batchWaitInterval; + /** + * 定时任务 + */ + private transient ScheduledExecutorService scheduler; + private transient ScheduledFuture scheduledFuture; - private transient ScheduledExecutorService executor; - private transient ScheduledFuture scheduledFuture; - - private final List records = new CopyOnWriteArrayList<>(); + private HbaseOutputFormat() { + } + public static HbaseOutputFormatBuilder buildHbaseOutputFormat() { + return new HbaseOutputFormatBuilder(); + } @Override public void configure(Configuration parameters) { - LOG.warn("---configure---"); - conf = HBaseConfiguration.create(); + // 这里不要做耗时较长的操作，否则会导致AKKA通信超时 + // DO NOTHING } @Override public void open(int taskNumber, int numTasks) throws IOException { LOG.warn("---open---"); + conf = HBaseConfiguration.create(); openConn(); table = conn.getTable(TableName.valueOf(tableName)); LOG.warn("---open end(get table from hbase) ---"); initMetric(); - - // 设置定时任务 - if (batchWaitInterval > 0) { - this.executor = Executors.newScheduledThreadPool( - 1, new ExecutorThreadFactory("hbase-sink-flusher")); - this.scheduledFuture = this.executor.scheduleAtFixedRate(() -> { - if (!records.isEmpty()) { - dealBatchOperation(records); - records.clear(); - } - }, batchWaitInterval, batchWaitInterval, TimeUnit.MILLISECONDS); - } } private void openConn() { @@ -137,13 +127,44 @@ private void openConn() { } catch (Exception e) { throw new RuntimeException(e); } + initScheduledTask(batchWaitInterval); + } + /** + * 初始化定时写入任务 + * + * @param batchWaitInterval 定时任务时间 + */ + private void initScheduledTask(Long batchWaitInterval) { + try { + if (batchWaitInterval > 0) { + this.scheduler = new ScheduledThreadPoolExecutor( + 1, + new DTThreadFactory("hbase-batch-flusher") + ); + + this.scheduledFuture = this.scheduler.scheduleWithFixedDelay( + () -> { + synchronized (this) { + if (!records.isEmpty()) { + dealBatchOperation(records); + records.clear(); + } + } + }, batchWaitInterval, batchWaitInterval, TimeUnit.MILLISECONDS + ); + } + } catch (Exception e) { + LOG.error("init schedule task failed !"); + throw new RuntimeException(e); + } } + private void openKerberosConn() throws Exception { conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); - LOG.info("kerberos config:{}", this.toString()); + LOG.info("kerberos config:{}", this.conf.toString()); Preconditions.checkArgument(!StringUtils.isEmpty(clientPrincipal), " clientPrincipal not null!"); Preconditions.checkArgument(!StringUtils.isEmpty(clientKeytabFile), " clientKeytabFile not null!"); @@ -173,17 +194,13 @@ private void openKerberosConn() throws Exception { }); } - @Override - public void writeRecord(Tuple2 tuple2) { - Tuple2 tupleTrans = tuple2; - Boolean retract = tupleTrans.f0; - Row row = tupleTrans.f1; - if (retract) { + public void writeRecord(Tuple2 record) { + if (record.f0) { if (this.batchSize != 0) { - writeBatchRecord(row); + writeBatchRecord(Row.copy(record.f1)); } else { - dealInsert(row); + dealInsert(record.f1); } } } @@ -223,7 +240,8 @@ protected void dealBatchOperation(List records) { } // 打印结果 if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0) { - LOG.info(records.toString()); + // 只打印最后一条数据 + LOG.info(records.get(records.size() - 1).toString()); } } catch (IOException | InterruptedException e) { LOG.error("", e); @@ -233,6 +251,7 @@ protected void dealBatchOperation(List records) { protected void dealInsert(Row record) { Put put = getPutByRow(record); if (put == null || put.isEmpty()) { + // 记录脏数据 outDirtyRecords.inc(); return; } @@ -306,8 +325,8 @@ public void close() throws IOException { if (scheduledFuture != null) { scheduledFuture.cancel(false); - if (executor != null) { - executor.shutdownNow(); + if (scheduler != null) { + scheduler.shutdownNow(); } } @@ -318,16 +337,48 @@ public void close() throws IOException { } - private HbaseOutputFormat() { + private void fillSyncKerberosConfig(org.apache.hadoop.conf.Configuration config, + String regionserverPrincipal, + String zookeeperSaslClient, + String securityKrb5Conf) { + if (StringUtils.isEmpty(regionserverPrincipal)) { + throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); + } + config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KERBEROS_PRINCIPAL, regionserverPrincipal); + config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL, regionserverPrincipal); + config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHORIZATION, "true"); + config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHENTICATION, "kerberos"); + + + if (!StringUtils.isEmpty(zookeeperSaslClient)) { + System.setProperty(HbaseConfigUtils.KEY_ZOOKEEPER_SASL_CLIENT, zookeeperSaslClient); + } + + if (!StringUtils.isEmpty(securityKrb5Conf)) { + String krb5ConfPath = System.getProperty("user.dir") + File.separator + securityKrb5Conf; + LOG.info("krb5ConfPath:{}", krb5ConfPath); + System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_KRB5_CONF, krb5ConfPath); + } } - public static HbaseOutputFormatBuilder buildHbaseOutputFormat() { - return new HbaseOutputFormatBuilder(); + @Override + public String toString() { + return "HbaseOutputFormat kerberos{" + + "kerberosAuthEnable=" + kerberosAuthEnable + + ", regionserverKeytabFile='" + regionserverKeytabFile + '\'' + + ", regionserverPrincipal='" + regionserverPrincipal + '\'' + + ", securityKrb5Conf='" + securityKrb5Conf + '\'' + + ", zookeeperSaslClient='" + zookeeperSaslClient + '\'' + + ", clientPrincipal='" + clientPrincipal + '\'' + + ", clientKeytabFile='" + clientKeytabFile + '\'' + + ", batchSize='" + batchSize + '\'' + + ", batchWaitInterval='" + batchWaitInterval + '\'' + + '}'; } public static class HbaseOutputFormatBuilder { - private HbaseOutputFormat format; + private final HbaseOutputFormat format; private HbaseOutputFormatBuilder() { format = new HbaseOutputFormat(); @@ -359,11 +410,6 @@ public HbaseOutputFormatBuilder setColumnNames(String[] columnNames) { return this; } - public HbaseOutputFormatBuilder setColumnTypes(String[] columnTypes) { - format.columnTypes = columnTypes; - return this; - } - public HbaseOutputFormatBuilder setColumnNameFamily(Map columnNameFamily) { format.columnNameFamily = columnNameFamily; return this; @@ -438,44 +484,5 @@ public HbaseOutputFormat finish() { return format; } - } - - private void fillSyncKerberosConfig(org.apache.hadoop.conf.Configuration config, String regionserverPrincipal, - String zookeeperSaslClient, String securityKrb5Conf) throws IOException { - if (StringUtils.isEmpty(regionserverPrincipal)) { - throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); - } - config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KERBEROS_PRINCIPAL, regionserverPrincipal); - config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL, regionserverPrincipal); - config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHORIZATION, "true"); - config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHENTICATION, "kerberos"); - - - if (!StringUtils.isEmpty(zookeeperSaslClient)) { - System.setProperty(HbaseConfigUtils.KEY_ZOOKEEPER_SASL_CLIENT, zookeeperSaslClient); - } - - if (!StringUtils.isEmpty(securityKrb5Conf)) { - String krb5ConfPath = System.getProperty("user.dir") + File.separator + securityKrb5Conf; - LOG.info("krb5ConfPath:{}", krb5ConfPath); - System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_KRB5_CONF, krb5ConfPath); - } - } - - @Override - public String toString() { - return "HbaseOutputFormat kerberos{" + - "kerberosAuthEnable=" + kerberosAuthEnable + - ", regionserverKeytabFile='" + regionserverKeytabFile + '\'' + - ", regionserverPrincipal='" + regionserverPrincipal + '\'' + - ", securityKrb5Conf='" + securityKrb5Conf + '\'' + - ", zookeeperSaslClient='" + zookeeperSaslClient + '\'' + - ", clientPrincipal='" + clientPrincipal + '\'' + - ", clientKeytabFile='" + clientKeytabFile + '\'' + - ", batchSize='" + batchSize + '\'' + - ", batchWaitInterval='" + batchWaitInterval + '\'' + - '}'; - } - } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index 3a7d1145c..f111a6bb9 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -17,7 +17,6 @@ */ - package com.dtstack.flink.sql.sink.hbase; import com.dtstack.flink.sql.sink.IStreamSinkGener; @@ -40,13 +39,13 @@ /** * Date: 2018/09/14 * Company: www.dtstack.com + * * @author sishu.yss */ public class HbaseSink implements RetractStreamTableSink, IStreamSinkGener { protected String[] fieldNames; protected Map columnNameFamily; - TypeInformation[] fieldTypes; protected String zookeeperQuorum; protected String port; protected String parent; @@ -54,20 +53,18 @@ public class HbaseSink implements RetractStreamTableSink, IStreamSinkGener< protected String updateMode; protected String rowkey; protected String registerTabName; - protected boolean kerberosAuthEnable; protected String regionserverKeytabFile; protected String regionserverPrincipal; protected String securityKrb5Conf; protected String zookeeperSaslClient; - + protected String batchSize; + protected String batchWaitInterval; + TypeInformation[] fieldTypes; private String clientPrincipal; private String clientKeytabFile; private int parallelism = 1; - protected String batchSize; - protected String batchWaitInterval; - public HbaseSink() { // TO DO NOTHING } @@ -81,7 +78,7 @@ public HbaseSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.tableName = hbaseTableInfo.getTableName(); this.rowkey = hbaseTableInfo.getRowkey(); this.columnNameFamily = hbaseTableInfo.getColumnNameFamily(); - this.registerTabName = hbaseTableInfo.getName(); + this.registerTabName = hbaseTableInfo.getName(); this.kerberosAuthEnable = hbaseTableInfo.isKerberosAuthEnable(); this.regionserverKeytabFile = hbaseTableInfo.getRegionserverKeytabFile(); @@ -110,31 +107,32 @@ public void emitDataStream(DataStream> dataStream) { @Override public DataStreamSink> consumeDataStream(DataStream> dataStream) { HbaseOutputFormat.HbaseOutputFormatBuilder builder = HbaseOutputFormat.buildHbaseOutputFormat(); - builder.setHost(this.zookeeperQuorum).setZkParent(this.parent).setTable(this.tableName); - builder.setRowkey(rowkey); - builder.setColumnNames(fieldNames); - builder.setColumnNameFamily(columnNameFamily); - builder.setKerberosAuthEnable(kerberosAuthEnable); - builder.setRegionserverKeytabFile(regionserverKeytabFile); - builder.setRegionserverPrincipal(regionserverPrincipal); - builder.setSecurityKrb5Conf(securityKrb5Conf); - builder.setZookeeperSaslClient(zookeeperSaslClient); + HbaseOutputFormat outputFormat = builder + .setHost(this.zookeeperQuorum) + .setZkParent(this.parent) + .setTable(this.tableName) + .setRowkey(rowkey) + .setColumnNames(fieldNames) + .setColumnNameFamily(columnNameFamily) + .setKerberosAuthEnable(kerberosAuthEnable) + .setRegionserverKeytabFile(regionserverKeytabFile) + .setRegionserverPrincipal(regionserverPrincipal) + .setSecurityKrb5Conf(securityKrb5Conf) + .setZookeeperSaslClient(zookeeperSaslClient) + .setClientPrincipal(clientPrincipal) + .setClientKeytabFile(clientKeytabFile) + .setBatchSize(Integer.parseInt(batchSize)) + .setBatchWaitInterval(Long.parseLong(batchWaitInterval)) + .finish(); - builder.setClientPrincipal(clientPrincipal); - builder.setClientKeytabFile(clientKeytabFile); - - builder.setBatchSize(Integer.parseInt(batchSize)); - builder.setBatchWaitInterval(Long.parseLong(batchWaitInterval)); - - HbaseOutputFormat outputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(outputFormat); DataStreamSink dataStreamSink = dataStream.addSink(richSinkFunction).name(registerTabName); if (parallelism > 0) { dataStreamSink.setParallelism(parallelism); } - + return dataStreamSink; } From bd501e69368c537fcf9e1e4d541f12beef001483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Tue, 12 Jan 2021 17:13:06 +0800 Subject: [PATCH 55/80] =?UTF-8?q?[feat-34632][HBase]=20HBase=20BatchSize?= =?UTF-8?q?=20=E6=B7=BB=E5=8A=A0synchronized=E5=85=B3=E9=94=AE=E5=AD=97?= =?UTF-8?q?=E9=81=BF=E5=85=8D=E7=BA=BF=E7=A8=8B=E5=AE=89=E5=85=A8=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/hbase/HbaseOutputFormat.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 4db2903d9..a10f6a7a8 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -59,7 +59,6 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat> { private static final Logger LOG = LoggerFactory.getLogger(HbaseOutputFormat.class); - private final List records = new ArrayList<>(); private String host; private String zkParent; private String rowkey; @@ -79,6 +78,8 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat records; + private transient volatile boolean closed = false; /** * 批量写入的参数 */ @@ -106,6 +107,7 @@ public void configure(Configuration parameters) { @Override public void open(int taskNumber, int numTasks) throws IOException { LOG.warn("---open---"); + records = new ArrayList<>(); conf = HBaseConfiguration.create(); openConn(); table = conn.getTable(TableName.valueOf(tableName)); @@ -145,10 +147,9 @@ private void initScheduledTask(Long batchWaitInterval) { this.scheduledFuture = this.scheduler.scheduleWithFixedDelay( () -> { - synchronized (this) { + synchronized (HbaseOutputFormat.this) { if (!records.isEmpty()) { dealBatchOperation(records); - records.clear(); } } }, batchWaitInterval, batchWaitInterval, TimeUnit.MILLISECONDS @@ -198,7 +199,7 @@ private void openKerberosConn() throws Exception { public void writeRecord(Tuple2 record) { if (record.f0) { if (this.batchSize != 0) { - writeBatchRecord(Row.copy(record.f1)); + writeBatchRecord(record.f1); } else { dealInsert(record.f1); } @@ -210,12 +211,10 @@ public void writeBatchRecord(Row row) { // 数据累计到batchSize之后开始处理 if (records.size() == this.batchSize) { dealBatchOperation(records); - // 添加完数据之后数据清空records - records.clear(); } } - protected void dealBatchOperation(List records) { + protected synchronized void dealBatchOperation(List records) { // A null in the result array means that the call for that action failed, even after retries. Object[] results = new Object[records.size()]; try { @@ -245,6 +244,9 @@ protected void dealBatchOperation(List records) { } } catch (IOException | InterruptedException e) { LOG.error("", e); + } finally { + // 添加完数据之后数据清空records + records.clear(); } } @@ -318,7 +320,12 @@ private Map rowConvertMap(Row record) { } @Override - public void close() throws IOException { + public synchronized void close() throws IOException { + if (closed) { + return; + } + + closed = true; if (!records.isEmpty()) { dealBatchOperation(records); } @@ -334,7 +341,6 @@ public void close() throws IOException { conn.close(); conn = null; } - } private void fillSyncKerberosConfig(org.apache.hadoop.conf.Configuration config, From 7cc034b5f9f5aa3152fac53d7b9b2f6c0b9c8ab9 Mon Sep 17 00:00:00 2001 From: chuixue Date: Tue, 12 Jan 2021 17:25:10 +0800 Subject: [PATCH 56/80] [hotfix-31811][launcher]merge v1.10 ro v1.11.1 conlict resolve --- .../apache/flink/table/planner/plan/QueryOperationConverter.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 launcher/src/main/java/org/apache/flink/table/planner/plan/QueryOperationConverter.java diff --git a/launcher/src/main/java/org/apache/flink/table/planner/plan/QueryOperationConverter.java b/launcher/src/main/java/org/apache/flink/table/planner/plan/QueryOperationConverter.java deleted file mode 100644 index e69de29bb..000000000 From c36117c3185bfb51f766161f01a943db6c752cc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Tue, 12 Jan 2021 20:06:14 +0800 Subject: [PATCH 57/80] [feat]add hbase batch size --- .../flink/sql/environment/StreamEnvConfigManager.java | 5 ++--- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 5 +++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index 6e53289af..1c043d6ee 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -85,7 +85,7 @@ public static void streamExecutionEnvironmentConfig(StreamExecutionEnvironment s ExecutionConfig exeConfig = streamEnv.getConfig(); if (exeConfig.getGlobalJobParameters() == null) { exeConfig.setGlobalJobParameters(globalJobParameters); - } else if (exeConfig.getGlobalJobParameters() instanceof ExecutionConfig.GlobalJobParameters) { + } else if (exeConfig.getGlobalJobParameters() != null) { exeConfig.setGlobalJobParameters(globalJobParameters); } @@ -367,10 +367,9 @@ private static Long getTtlTime(Integer timeNumber, String timeUnit) { } } - private static StreamExecutionEnvironment disableChainOperator(StreamExecutionEnvironment env, Configuration configuration) { + private static void disableChainOperator(StreamExecutionEnvironment env, Configuration configuration) { if(configuration.getBoolean("disableChainOperator", false)) { env.disableOperatorChaining(); } - return env; } } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 699ed639c..434ba694d 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.sink.hbase; -import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.dirtyManager.manager.DirtyDataManager; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; @@ -93,6 +93,7 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat scheduledFuture; + private DirtyDataManager dirtyDataManager; private HbaseOutputFormat() { } @@ -100,7 +101,6 @@ private HbaseOutputFormat() { public static HbaseOutputFormatBuilder buildHbaseOutputFormat() { return new HbaseOutputFormatBuilder(); } - private DirtyDataManager dirtyDataManager; @Override public void configure(Configuration parameters) { @@ -164,6 +164,7 @@ private void initScheduledTask(Long batchWaitInterval) { throw new RuntimeException(e); } } + private void openKerberosConn() throws Exception { conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); From 42dc7779c9e578e91e32122a4c8763d6b7eced48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Mon, 21 Dec 2020 19:32:45 +0800 Subject: [PATCH 58/80] =?UTF-8?q?[github]=20=E4=BF=AE=E6=94=B9readme?= =?UTF-8?q?=E6=8B=9B=E8=81=98=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f804f3f4..4207fd0ca 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,18 @@ FlinkStreamSQL [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) ## 技术交流 -- 招聘**大数据平台开发工程师**，想了解岗位详细信息可以添加本人微信号ysqwhiletrue，注明招聘，如有意者发送简历至[sishu@dtstack.com](mailto:sishu@dtstack.com) +- 招聘**Flink开发工程师**，如果有兴趣，请联系思枢【微信号ysqwhiletrue】，注明招聘
+ Flink开发工程师JD要求：
+ 1.负责袋鼠云基于Flink的衍生框架数据同步flinkx和实时计算flinkstreamsql框架的开发；
+ 2.调研和把握当前最新大数据实时计算技术，将其中的合适技术引入到平台中，改善产品，提升竞争力；
+ 职位要求：
+ 1、本科及以上学历，3年及以上的Flink开发经验，精通Java，熟悉Scala、Python优先考虑；
+ 2、熟悉Flink原理，有基于Flink做过二次源码的开发，在github上贡献者Flink源码者优先；
+ 3、有机器学习、数据挖掘相关经验者优先；
+ 4、对新技术有快速学习和上手能力，对代码有一定的洁癖；
+ 加分项：
+ 1.在GitHub或其他平台上有过开源项目;
+ 可以添加本人微信号ysqwhiletrue，注明招聘，如有意者发送简历至sishu@dtstack.com
- 我们使用[钉钉](https://www.dingtalk.com/)沟通交流，可以搜索群号[**30537511**]或者扫描下面的二维码进入钉钉群
From a94c9ef114fd8f5bb58900726565a5410f4a4644 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 12 Jan 2021 20:39:15 +0800 Subject: [PATCH 59/80] [hotfix][root][pom] comment aws plugin --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a17287ff1..039a764e7 100644 --- a/pom.xml +++ b/pom.xml @@ -36,7 +36,7 @@ elasticsearch6 cassandra kudu - aws + dirtyData From 77d2a01b91d7c0b5f0ef1e893b211128adf04f07 Mon Sep 17 00:00:00 2001 From: mowen Date: Wed, 13 Jan 2021 00:27:48 +0800 Subject: [PATCH 60/80] [hotfix-33089][core]add pks into sidetableinfo's fields. --- .../dtstack/flink/sql/side/BaseSideInfo.java | 2 +- .../flink/sql/table/AbstractTableParser.java | 60 +++++++++++++------ 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java index 1239df53a..d83d16110 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java @@ -118,7 +118,7 @@ public void parseSelectFields(JoinInfo joinInfo){ public String getTargetFieldType(String fieldName){ int fieldIndex = sideTableInfo.getFieldList().indexOf(fieldName); if(fieldIndex == -1){ - throw new RuntimeException(sideTableInfo.getName() + "can't find field: " + fieldName); + throw new RuntimeException(sideTableInfo.getName() + " can't find field: " + fieldName); } return sideTableInfo.getFieldTypes()[fieldIndex]; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index e5ad0bf8d..50a7029f0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.table; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.base.Preconditions; @@ -26,6 +27,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -105,30 +107,50 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { continue; } - Tuple2 t = extractType(fieldRow, tableInfo.getName()); - String fieldName = t.f0; - String fieldType = t.f1; + handleKeyNotHaveAlias(fieldRow, tableInfo); + } - Class fieldClass; - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; + /* + * check whether filed list contains pks and then add pks into field list. + * because some no-sql database is not primary key. eg :redis、hbase etc... + */ + if (tableInfo instanceof AbstractSideTableInfo) { + List pks = new ArrayList<>(); + tableInfo.getPrimaryKeys().stream().forEach(pk -> { + if (!tableInfo.getFieldList().contains(pk)) { + pks.add(String.format("%s varchar", pk)); + } + }); + pks.stream().forEach(pk -> { + handleKeyNotHaveAlias(pk, tableInfo); + }); + } - Matcher matcher = charTypePattern.matcher(fieldType); - if (matcher.find()) { - fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); - fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); - fieldExtraInfo.setLength(Integer.parseInt(matcher.group(1))); - } else { - fieldClass = dbTypeConvertToJavaType(fieldType); - } + tableInfo.finish(); + } - tableInfo.addPhysicalMappings(fieldName, fieldName); - tableInfo.addField(fieldName); - tableInfo.addFieldClass(fieldClass); - tableInfo.addFieldType(fieldType); - tableInfo.addFieldExtraInfo(fieldExtraInfo); + private void handleKeyNotHaveAlias(String fieldRow, AbstractTableInfo tableInfo) { + Tuple2 t = extractType(fieldRow, tableInfo.getName()); + String fieldName = t.f0; + String fieldType = t.f1; + + Class fieldClass; + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; + + Matcher matcher = charTypePattern.matcher(fieldType); + if (matcher.find()) { + fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); + fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); + fieldExtraInfo.setLength(Integer.parseInt(matcher.group(1))); + } else { + fieldClass = dbTypeConvertToJavaType(fieldType); } - tableInfo.finish(); + tableInfo.addPhysicalMappings(fieldName, fieldName); + tableInfo.addField(fieldName); + tableInfo.addFieldClass(fieldClass); + tableInfo.addFieldType(fieldType); + tableInfo.addFieldExtraInfo(fieldExtraInfo); } private Tuple2 extractType(String fieldRow, String tableName) { From b15b8b729e13294ac7242e0fd83c16077c7565ff Mon Sep 17 00:00:00 2001 From: mowen Date: Wed, 13 Jan 2021 00:29:51 +0800 Subject: [PATCH 61/80] [hotfix-33089][hbase][async-side]add rowkey value to sideVal. --- .../side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java | 2 ++ .../side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java | 6 ++++++ .../sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java | 6 ++++++ 3 files changed, 14 insertions(+) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java index 47c15c6bb..87c8b4e39 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java @@ -55,6 +55,8 @@ public abstract class AbstractRowKeyModeDealer { protected List outFieldInfoList; + protected static final String ROWKEY = "rowkey"; + //key:Returns the value of the position, returns the index values ​​in the input data protected Map inFieldIndex = Maps.newHashMap(); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java index df2694238..84eccf8ae 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java @@ -110,6 +110,12 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, //The order of the fields defined in the data conversion table List sideVal = Lists.newArrayList(); for (String key : colNames) { + + if (ROWKEY.equalsIgnoreCase(key)) { + sideVal.add(rowKeyStr); + continue; + } + Object val = sideMap.get(key); if (val == null) { LOG.error("can't get data with column {}", key); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index d7b65098e..e8d6d1608 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -84,6 +84,12 @@ public void asyncGetData(String tableName, String rowKeyStr, BaseRow input, Resu //The order of the fields defined in the data conversion table List sideVal = Lists.newArrayList(); for(String key : colNames){ + + if (ROWKEY.equalsIgnoreCase(key)) { + sideVal.add(rowKeyStr); + continue; + } + Object val = sideMap.get(key); if(val == null){ LOG.error("can't get data with column {}", key); From 0be71efab705ff06c891e614a584549ad9fde42a Mon Sep 17 00:00:00 2001 From: mowen Date: Wed, 13 Jan 2021 17:54:37 +0800 Subject: [PATCH 62/80] [hotfix-33089][hbase-side]optimze code --- .../dtstack/flink/sql/table/AbstractTableParser.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 50a7029f0..01d2e8344 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -115,15 +115,9 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { * because some no-sql database is not primary key. eg :redis、hbase etc... */ if (tableInfo instanceof AbstractSideTableInfo) { - List pks = new ArrayList<>(); - tableInfo.getPrimaryKeys().stream().forEach(pk -> { - if (!tableInfo.getFieldList().contains(pk)) { - pks.add(String.format("%s varchar", pk)); - } - }); - pks.stream().forEach(pk -> { - handleKeyNotHaveAlias(pk, tableInfo); - }); + tableInfo.getPrimaryKeys().stream() + .filter(pk -> !tableInfo.getFieldList().contains(pk)) + .forEach(pk -> handleKeyNotHaveAlias(String.format("%s varchar", pk), tableInfo)); } tableInfo.finish(); From 8930f0b471335307a0beb8e46b5cb9f679162f52 Mon Sep 17 00:00:00 2001 From: mowen Date: Wed, 13 Jan 2021 17:58:24 +0800 Subject: [PATCH 63/80] [hotfix-33089][docs]modified hbase doc. --- docs/plugin/hbaseSide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index 550dfbae0..2ee7c3a9f 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -146,7 +146,7 @@ CREATE TABLE MyResult( CREATE TABLE sideTable ( wtz:message varchar as message, wtz:info varchar as info , - PRIMARY KEY (md5(rowkey1) + rowkey2 + 'test'), + PRIMARY KEY (rowkey), PERIOD FOR SYSTEM_TIME ) WITH ( type = 'hbase', @@ -174,7 +174,7 @@ into MyTable a left join sideTable b - on a.id=b.rowkey1 and a.name = b.rowkey2; + on a.id=b.rowkey; ``` ### kerberos维表示例 ``` From 36c3f5055b55a1de881e58bc3366ddd23701ee85 Mon Sep 17 00:00:00 2001 From: mowen Date: Wed, 13 Jan 2021 19:42:31 +0800 Subject: [PATCH 64/80] [hotfix-33089][hbase-side]Compatible with composite primary keys. --- .../flink/sql/table/AbstractTableParser.java | 13 ++++++++++++- docs/plugin/hbaseSide.md | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 01d2e8344..29ae59a3c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.table; +import com.dtstack.flink.sql.parser.SqlParser; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.DtStringUtil; @@ -26,6 +27,8 @@ import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Arrays; @@ -45,6 +48,8 @@ public abstract class AbstractTableParser { + private static final Logger LOG = LoggerFactory.getLogger(AbstractTableParser.class); + private static final String PRIMARY_KEY = "primaryKey"; private static final String NEST_JSON_FIELD_KEY = "nestFieldKey"; private static final String CHAR_TYPE_NO_LENGTH = "CHAR"; @@ -117,7 +122,13 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { if (tableInfo instanceof AbstractSideTableInfo) { tableInfo.getPrimaryKeys().stream() .filter(pk -> !tableInfo.getFieldList().contains(pk)) - .forEach(pk -> handleKeyNotHaveAlias(String.format("%s varchar", pk), tableInfo)); + .forEach(pk -> { + try { + handleKeyNotHaveAlias(String.format("%s varchar", pk), tableInfo); + } catch (Exception e) { + LOG.error(String.format("Handle primary key failed. Reason: %s", e.getMessage())); + } + }); } tableInfo.finish(); diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index 2ee7c3a9f..550dfbae0 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -146,7 +146,7 @@ CREATE TABLE MyResult( CREATE TABLE sideTable ( wtz:message varchar as message, wtz:info varchar as info , - PRIMARY KEY (rowkey), + PRIMARY KEY (md5(rowkey1) + rowkey2 + 'test'), PERIOD FOR SYSTEM_TIME ) WITH ( type = 'hbase', @@ -174,7 +174,7 @@ into MyTable a left join sideTable b - on a.id=b.rowkey; + on a.id=b.rowkey1 and a.name = b.rowkey2; ``` ### kerberos维表示例 ``` From a5fb25ae482f9aca55e99b683ec397f5d75b6c6b Mon Sep 17 00:00:00 2001 From: mowen Date: Wed, 13 Jan 2021 19:44:53 +0800 Subject: [PATCH 65/80] [hotfix-33089][hbase-side]optimize imports. --- .../java/com/dtstack/flink/sql/table/AbstractTableParser.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 29ae59a3c..4c95a4e30 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.table; -import com.dtstack.flink.sql.parser.SqlParser; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.DtStringUtil; @@ -30,7 +30,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; From 4e5b3f7fda0123e7e342b6856f3622dcb58a8678 Mon Sep 17 00:00:00 2001 From: mowen Date: Wed, 13 Jan 2021 19:45:40 +0800 Subject: [PATCH 66/80] [hotfix-33089][hbase-side]optimize imports. --- .../java/com/dtstack/flink/sql/table/AbstractTableParser.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 4c95a4e30..62dd38dbb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -19,7 +19,6 @@ package com.dtstack.flink.sql.table; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.DtStringUtil; From 003b35eeddbb61e1f3751780140d5600df6b5a61 Mon Sep 17 00:00:00 2001 From: mowen Date: Thu, 14 Jan 2021 10:50:51 +0800 Subject: [PATCH 67/80] [hotfix-33089][core]optimize parseFieldsInfo method. --- .../dtstack/flink/sql/table/AbstractTableParser.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 62dd38dbb..b9e7c9ae7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -119,14 +119,8 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { */ if (tableInfo instanceof AbstractSideTableInfo) { tableInfo.getPrimaryKeys().stream() - .filter(pk -> !tableInfo.getFieldList().contains(pk)) - .forEach(pk -> { - try { - handleKeyNotHaveAlias(String.format("%s varchar", pk), tableInfo); - } catch (Exception e) { - LOG.error(String.format("Handle primary key failed. Reason: %s", e.getMessage())); - } - }); + .filter(pk -> (!tableInfo.getFieldList().contains(pk) && pk.equals("rowkey"))) + .forEach(pk -> handleKeyNotHaveAlias(String.format("%s varchar", pk), tableInfo)); } tableInfo.finish(); From d02ab0176ce27f0b3b6009b5fd5fe6cd3750316b Mon Sep 17 00:00:00 2001 From: mowen Date: Thu, 14 Jan 2021 10:51:58 +0800 Subject: [PATCH 68/80] [hotfix-33089][core]remove log from AbstractTableParser. --- .../java/com/dtstack/flink/sql/table/AbstractTableParser.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index b9e7c9ae7..1e30095b0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -26,8 +26,6 @@ import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; @@ -46,8 +44,6 @@ public abstract class AbstractTableParser { - private static final Logger LOG = LoggerFactory.getLogger(AbstractTableParser.class); - private static final String PRIMARY_KEY = "primaryKey"; private static final String NEST_JSON_FIELD_KEY = "nestFieldKey"; private static final String CHAR_TYPE_NO_LENGTH = "CHAR"; From 5294ced4ab849d91d661e7cc75bb61a1d510a40a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Thu, 14 Jan 2021 20:06:14 +0800 Subject: [PATCH 69/80] [opt] resolve conflict. --- .../side/clickhouse/table/ClickhouseTableFunction.java | 4 ++-- .../java/com/dtstack/flink/sql/option/Options.java | 10 ++++++---- .../side/hbase/rowkeydealer/RowKeyEqualModeDealer.java | 2 ++ .../sql/side/impala/table/ImpalaTableFunction.java | 6 +++--- .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 2 -- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseTableFunction.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseTableFunction.java index a114898ea..e14e1a92f 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseTableFunction.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseTableFunction.java @@ -18,10 +18,10 @@ package com.dtstack.flink.sql.side.clickhouse.table; +import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.clickhouse.ClickhouseAllSideInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbTableFunction; -import com.dtstack.flink.sql.util.JDBCUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +46,7 @@ public ClickhouseTableFunction(AbstractSideTableInfo sideTableInfo, String[] loo public Connection getConn(String dbUrl, String userName, String passWord) { try { Connection connection ; - JDBCUtils.forName(CLICKHOUSE_DRIVER, getClass().getClassLoader()); + ClassLoaderManager.forName(CLICKHOUSE_DRIVER, getClass().getClassLoader()); // ClickHouseProperties contains all properties if (userName == null) { connection = DriverManager.getConnection(dbUrl); diff --git a/core/src/main/java/com/dtstack/flink/sql/option/Options.java b/core/src/main/java/com/dtstack/flink/sql/option/Options.java index f45ff627e..69c49056a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/Options.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/Options.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,6 +20,7 @@ import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.enums.EPluginLoadMode; +import com.dtstack.flink.sql.enums.PlannerType; /** @@ -192,13 +193,14 @@ public void setAddShipfile(String addShipfile) { this.addShipfile = addShipfile; } + public String getPlanner() { + return planner; + } + public void setPlanner(String planner) { this.planner = planner; } - public String getPlanner() { - return planner; - } public String getDirtyProperties() { return dirtyProperties; } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index 09c6ee66d..9b8435857 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -38,9 +38,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; /** * Reason: diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaTableFunction.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaTableFunction.java index 10c65aedb..0a36895b5 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaTableFunction.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaTableFunction.java @@ -18,10 +18,10 @@ package com.dtstack.flink.sql.side.impala.table; +import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.ImpalaAllSideInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbTableFunction; -import com.dtstack.flink.sql.util.JDBCUtils; import com.dtstack.flink.sql.util.KrbUtils; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; @@ -58,7 +58,7 @@ public Connection getConn(String dbUrl, String userName, String password) { try { Connection connection; String url = getUrl(); - JDBCUtils.forName(IMPALA_DRIVER, getClass().getClassLoader()); + ClassLoaderManager.forName(IMPALA_DRIVER, getClass().getClassLoader()); // Kerberos if (impalaSideTableInfo.getAuthMech() == 1) { String keyTabFilePath = impalaSideTableInfo.getKeyTabFilePath(); @@ -87,7 +87,7 @@ public String getUrl() throws IOException { String newUrl = ""; Integer authMech = impalaSideTableInfo.getAuthMech(); - StringBuffer urlBuffer = new StringBuffer(impalaSideTableInfo.getUrl()); + StringBuilder urlBuffer = new StringBuilder(impalaSideTableInfo.getUrl()); if (authMech == 0) { newUrl = urlBuffer.toString(); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index b3d0b688b..dad34cd5e 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -40,8 +40,6 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.types.Row; -import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.table.dataformat.GenericRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From f046ceeceae014bc1efad1ec60a311eb25a6ec68 Mon Sep 17 00:00:00 2001 From: chuixue Date: Mon, 18 Jan 2021 14:46:46 +0800 Subject: [PATCH 70/80] [feat-31342][rdb]add rdb resource check on taskmanager --- .../flink/sql/core/rdb/JdbcCheckKeys.java | 1 + .../flink/sql/core/rdb/JdbcResourceCheck.java | 29 ++++++++++++++----- .../sql/core/rdb/util/JdbcConnectUtil.java | 16 ++++++++-- .../side/rdb/all/AbstractRdbAllReqRow.java | 2 +- .../sql/side/rdb/async/RdbAsyncReqRow.java | 2 ++ .../sql/side/rdb/table/RdbSideTableInfo.java | 2 ++ .../flink/sql/sink/rdb/JDBCOptions.java | 2 ++ .../sql/sink/rdb/table/RdbTableInfo.java | 3 +- 8 files changed, 45 insertions(+), 12 deletions(-) diff --git a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcCheckKeys.java b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcCheckKeys.java index 3b1dfcb27..6eef874fa 100644 --- a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcCheckKeys.java +++ b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcCheckKeys.java @@ -29,6 +29,7 @@ public class JdbcCheckKeys { public static final String USER_NAME_KEY = "userName"; public static final String PASSWORD_KEY = "password"; public static final String TABLE_TYPE_KEY = "tableType"; + public static final String NEED_CHECK = "needCheck"; public static final String SCHEMA_KEY = "schema"; public static final String TABLE_NAME_KEY = "tableName"; // create 语句中的name diff --git a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java index 1d95cb728..514d4c5d9 100644 --- a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java +++ b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java @@ -65,18 +65,23 @@ public static JdbcResourceCheck getInstance() { @Override public void checkResourceStatus(Map checkProperties) { - if (!NEED_CHECK) { - LOG.info("Ignore checking data source ..."); + if (!NEED_CHECK || !Boolean.parseBoolean(checkProperties.get(JdbcCheckKeys.NEED_CHECK))) { + LOG.warn("Ignore checking [{}] type data source , tableName is [{}]." + , checkProperties.get(JdbcCheckKeys.TABLE_TYPE_KEY) + , checkProperties.get(JdbcCheckKeys.TABLE_NAME_KEY)); return; } - LOG.info("start checking data source ..."); + + LOG.info("start checking [{}] type data source , tableName is [{}]." + , checkProperties.get(JdbcCheckKeys.TABLE_TYPE_KEY) + , checkProperties.get(JdbcCheckKeys.TABLE_NAME_KEY)); List privilegeList = new ArrayList<>(); if (checkProperties.get(TABLE_TYPE_KEY).equalsIgnoreCase(SIDE_STR)) { privilegeList.add(SELECT_STR); } if (checkProperties.get(TABLE_TYPE_KEY).equalsIgnoreCase(SINK_STR)) { privilegeList.add(INSERT_STR); - privilegeList.add(DELETE_STR); + // privilegeList.add(DELETE_STR); } checkPrivilege( checkProperties.get(JdbcCheckKeys.DRIVER_NAME) @@ -87,8 +92,11 @@ public void checkResourceStatus(Map checkProperties) { , checkProperties.get(JdbcCheckKeys.SCHEMA_KEY) , privilegeList ); - LOG.info(String.format("data source is available and user [%s] has the corresponding permissions %s for [%s] table...", - checkProperties.get(JdbcCheckKeys.USER_NAME_KEY), privilegeList.toString(), checkProperties.get(TABLE_TYPE_KEY))); + LOG.info("data source is available and user [{}] has the corresponding permissions {} for [{}] type , tableName is [{}]" + , checkProperties.get(JdbcCheckKeys.USER_NAME_KEY) + , privilegeList.toString() + , checkProperties.get(JdbcCheckKeys.TABLE_TYPE_KEY) + , checkProperties.get(JdbcCheckKeys.TABLE_NAME_KEY)); } public void checkPrivilege( @@ -108,8 +116,13 @@ public void checkPrivilege( statement = connection.createStatement(); for (String s : privilegeList) { privilege = s; - statement.executeQuery( - String.format(PRIVILEGE_SQL_MAP.get(privilege.toLowerCase()), tableInfo)); + if (privilege.startsWith(SELECT_STR)) { + statement.executeQuery( + String.format(PRIVILEGE_SQL_MAP.get(privilege.toLowerCase()), tableInfo)); + } else { + statement.executeUpdate( + String.format(PRIVILEGE_SQL_MAP.get(privilege.toLowerCase()), tableInfo, tableInfo)); + } } } catch (SQLException sqlException) { if (sqlException.getMessage().contains("command denied")) { diff --git a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java index 7a21e16cc..41e01f0f4 100644 --- a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java +++ b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java @@ -39,7 +39,7 @@ */ public class JdbcConnectUtil { private static final int DEFAULT_RETRY_NUM = 3; - private static final long DEFAULT_RETRY_TIME_WAIT = 5L; + private static final long DEFAULT_RETRY_TIME_WAIT = 3L; private static final int DEFAULT_VALID_TIME = 10; private static final Logger LOG = LoggerFactory.getLogger(JdbcConnectUtil.class); @@ -136,7 +136,7 @@ public static Connection getConnectWithRetry( + "\nerror message: "); String errorCause = null; - ClassLoaderManager.forName(driverName); + forName(driverName); Preconditions.checkNotNull(url, "url can't be null!"); for (int i = 0; i < DEFAULT_RETRY_NUM; i++) { @@ -157,4 +157,16 @@ public static Connection getConnectWithRetry( } throw new SuppressRestartsException(new Throwable(errorMessage + errorCause)); } + + /** + * @param clazz + */ + public synchronized static void forName(String clazz) { + try { + Class driverClass = Class.forName(clazz); + driverClass.newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index ca96f0ee5..0db619957 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -85,8 +85,8 @@ public void open(Configuration parameters) throws Exception { JdbcResourceCheck.getInstance().checkResourceStatus(tableInfo.getCheckProperties()); } } - LOG.info("rdb dim table config info: {} ", tableInfo.toString()); super.open(parameters); + LOG.info("rdb dim table config info: {} ", tableInfo.toString()); } @Override diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index eb7959399..d4cd6c5b5 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -19,9 +19,11 @@ package com.dtstack.flink.sql.side.rdb.async; +import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.core.rdb.JdbcResourceCheck; import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.factory.DTThreadFactory; +import com.dtstack.flink.sql.resource.ResourceCheck; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.CacheMissVal; diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java index f3200dd76..6099c583e 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java @@ -19,6 +19,7 @@ import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.core.rdb.JdbcResourceCheck; +import com.dtstack.flink.sql.resource.ResourceCheck; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; @@ -134,6 +135,7 @@ public Map buildCheckProperties() { properties.put(JdbcCheckKeys.TABLE_NAME_KEY, getTableName()); properties.put(JdbcCheckKeys.OPERATION_NAME_KEY, getName()); properties.put(JdbcCheckKeys.TABLE_TYPE_KEY, "side"); + properties.put(JdbcCheckKeys.NEED_CHECK, ResourceCheck.NEED_CHECK+""); return properties; } } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java index f89cba731..686203b2b 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java @@ -20,6 +20,7 @@ import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; +import com.dtstack.flink.sql.resource.ResourceCheck; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.google.common.collect.Maps; @@ -89,6 +90,7 @@ public Map buildCheckProperties() { properties.put(JdbcCheckKeys.TABLE_NAME_KEY, getTableName()); properties.put(JdbcCheckKeys.OPERATION_NAME_KEY, "jdbcOutputFormat"); properties.put(JdbcCheckKeys.TABLE_TYPE_KEY, "sink"); + properties.put(JdbcCheckKeys.NEED_CHECK, ResourceCheck.NEED_CHECK+""); return properties; } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java index 10c2742ad..c3ea736a4 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java @@ -20,13 +20,13 @@ import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.core.rdb.JdbcResourceCheck; import com.dtstack.flink.sql.enums.EUpdateMode; +import com.dtstack.flink.sql.resource.ResourceCheck; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import java.util.Map; -import java.util.Properties; /** * Reason: @@ -233,6 +233,7 @@ public Map buildCheckProperties() { properties.put(JdbcCheckKeys.TABLE_NAME_KEY, getTableName()); properties.put(JdbcCheckKeys.OPERATION_NAME_KEY, getName()); properties.put(JdbcCheckKeys.TABLE_TYPE_KEY, "sink"); + properties.put(JdbcCheckKeys.NEED_CHECK, ResourceCheck.NEED_CHECK+""); return properties; } From 26add19c442a8b92d2d3d4f8ec62525d3ff1a1e4 Mon Sep 17 00:00:00 2001 From: chuixue Date: Mon, 18 Jan 2021 16:12:29 +0800 Subject: [PATCH 71/80] [feat-31342][rdb] add lookup rdb resource check on taskmanager --- .../sql/side/rdb/all/AbstractRdbTableFunction.java | 10 +++++++++- .../sql/side/rdb/async/RdbAsyncTableFunction.java | 10 ++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbTableFunction.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbTableFunction.java index 36eec90a9..ae8a14612 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbTableFunction.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbTableFunction.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.rdb.all; +import com.dtstack.flink.sql.core.rdb.JdbcResourceCheck; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; @@ -41,6 +42,7 @@ **/ abstract public class AbstractRdbTableFunction extends BaseTableFunction { private static final Logger LOG = LoggerFactory.getLogger(AbstractRdbTableFunction.class); + private static volatile boolean resourceCheck = true; public AbstractRdbTableFunction(BaseSideInfo sideInfo) { super(sideInfo); @@ -48,8 +50,14 @@ public AbstractRdbTableFunction(BaseSideInfo sideInfo) { @Override public void open(FunctionContext context) throws Exception { - super.open(context); RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideTableInfo; + synchronized (AbstractRdbAllReqRow.class) { + if (resourceCheck) { + resourceCheck = false; + JdbcResourceCheck.getInstance().checkResourceStatus(tableInfo.getCheckProperties()); + } + } + super.open(context); LOG.info("rdb dim table config info: {} ", tableInfo.toString()); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncTableFunction.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncTableFunction.java index a7db919ac..fa2f50765 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncTableFunction.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncTableFunction.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.rdb.async; +import com.dtstack.flink.sql.core.rdb.JdbcResourceCheck; import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.BaseSideInfo; @@ -88,8 +89,17 @@ public class RdbAsyncTableFunction extends BaseAsyncTableFunction { private final static int MAX_TASK_QUEUE_SIZE = 100000; + private static volatile boolean resourceCheck = true; + @Override public void open(FunctionContext context) throws Exception { + RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + synchronized (RdbAsyncReqRow.class) { + if (resourceCheck) { + resourceCheck = false; + JdbcResourceCheck.getInstance().checkResourceStatus(rdbSideTableInfo.getCheckProperties()); + } + } super.open(context); executor = new ThreadPoolExecutor(MAX_DB_CONN_POOL_SIZE_LIMIT, MAX_DB_CONN_POOL_SIZE_LIMIT, 0, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(MAX_TASK_QUEUE_SIZE), new DTThreadFactory("rdbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy()); From c9ddaa9444f316e81253744144e582b5c5f78b4b Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 18 Jan 2021 20:44:50 +0800 Subject: [PATCH 72/80] =?UTF-8?q?[opt-31342][rdb]=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/core/rdb/JdbcResourceCheck.java | 23 ++++++++++--------- .../sql/side/rdb/async/RdbAsyncReqRow.java | 19 ++++++++------- .../sql/side/rdb/table/RdbSideTableInfo.java | 2 +- .../flink/sql/sink/rdb/JDBCOptions.java | 2 +- .../sql/sink/rdb/table/RdbTableInfo.java | 2 +- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java index 514d4c5d9..2b28258a7 100644 --- a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java +++ b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/JdbcResourceCheck.java @@ -20,6 +20,7 @@ import com.dtstack.flink.sql.core.rdb.util.JdbcConnectUtil; import com.dtstack.flink.sql.resource.ResourceCheck; +import org.apache.commons.lang.StringUtils; import org.apache.flink.runtime.execution.SuppressRestartsException; import java.sql.Connection; @@ -43,9 +44,9 @@ public class JdbcResourceCheck extends ResourceCheck { private static final String UPDATE_STR = "update"; private static final String REPLACE_STR = "replace"; - private static final String CHECK_SELECT_SQL = "select 1 from %s where 1=1;"; - private static final String CHECK_DELETE_SQL = "delete from %s where 1 = 3;"; - private static final String CHECK_INSERT_SQL = "insert into %s (select * from %s where 1 = 2);"; + private static final String CHECK_SELECT_SQL = "select 1 from $table where 1=2;"; + private static final String CHECK_DELETE_SQL = "delete from $table where 1=3;"; + private static final String CHECK_INSERT_SQL = "insert into $table (select * from $table where 1=2);"; private static final Map PRIVILEGE_SQL_MAP = new HashMap<>(); private static final JdbcResourceCheck Instance = new JdbcResourceCheck(); @@ -116,13 +117,13 @@ public void checkPrivilege( statement = connection.createStatement(); for (String s : privilegeList) { privilege = s; - if (privilege.startsWith(SELECT_STR)) { - statement.executeQuery( - String.format(PRIVILEGE_SQL_MAP.get(privilege.toLowerCase()), tableInfo)); - } else { - statement.executeUpdate( - String.format(PRIVILEGE_SQL_MAP.get(privilege.toLowerCase()), tableInfo, tableInfo)); - } + statement.execute( + StringUtils.replace( + PRIVILEGE_SQL_MAP.get(privilege.toLowerCase()), + "$table", + tableName + ) + ); } } catch (SQLException sqlException) { if (sqlException.getMessage().contains("command denied")) { @@ -130,7 +131,7 @@ public void checkPrivilege( String.format("user [%s] don't have [%s] privilege of table [%s]", userName, privilege, tableInfo))); } - throw new SuppressRestartsException(new Throwable(sqlException.getMessage())); + throw new SuppressRestartsException(new IllegalArgumentException(sqlException.getMessage())); } finally { JdbcConnectUtil.closeConnectionResource(null, statement, connection, false); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index d4cd6c5b5..daec5d309 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -19,11 +19,9 @@ package com.dtstack.flink.sql.side.rdb.async; -import com.dtstack.flink.sql.core.rdb.JdbcCheckKeys; import com.dtstack.flink.sql.core.rdb.JdbcResourceCheck; import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.factory.DTThreadFactory; -import com.dtstack.flink.sql.resource.ResourceCheck; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.CacheMissVal; @@ -111,7 +109,8 @@ protected void init(BaseSideInfo sideInfo) { } @Override - protected void preInvoke(BaseRow input, ResultFuture resultFuture) { } + protected void preInvoke(BaseRow input, ResultFuture resultFuture) { + } @Override public void handleAsyncInvoke(Map inputParams, BaseRow input, ResultFuture resultFuture) throws Exception { @@ -144,13 +143,13 @@ protected void asyncQueryData(Map inputParams, } final protected void doAsyncQueryData( - Map inputParams, - BaseRow input, - ResultFuture resultFuture, - SQLClient rdbSqlClient, - AtomicLong failCounter, - AtomicBoolean finishFlag, - CountDownLatch latch) { + Map inputParams, + BaseRow input, + ResultFuture resultFuture, + SQLClient rdbSqlClient, + AtomicLong failCounter, + AtomicBoolean finishFlag, + CountDownLatch latch) { rdbSqlClient.getConnection(conn -> { try { if (conn.failed()) { diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java index 6099c583e..19e747c3a 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java @@ -135,7 +135,7 @@ public Map buildCheckProperties() { properties.put(JdbcCheckKeys.TABLE_NAME_KEY, getTableName()); properties.put(JdbcCheckKeys.OPERATION_NAME_KEY, getName()); properties.put(JdbcCheckKeys.TABLE_TYPE_KEY, "side"); - properties.put(JdbcCheckKeys.NEED_CHECK, ResourceCheck.NEED_CHECK+""); + properties.put(JdbcCheckKeys.NEED_CHECK, String.valueOf(ResourceCheck.NEED_CHECK)); return properties; } } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java index 686203b2b..eacdc9b37 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java @@ -90,7 +90,7 @@ public Map buildCheckProperties() { properties.put(JdbcCheckKeys.TABLE_NAME_KEY, getTableName()); properties.put(JdbcCheckKeys.OPERATION_NAME_KEY, "jdbcOutputFormat"); properties.put(JdbcCheckKeys.TABLE_TYPE_KEY, "sink"); - properties.put(JdbcCheckKeys.NEED_CHECK, ResourceCheck.NEED_CHECK+""); + properties.put(JdbcCheckKeys.NEED_CHECK, String.valueOf(ResourceCheck.NEED_CHECK)); return properties; } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java index c3ea736a4..f5ca46f6a 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java @@ -233,7 +233,7 @@ public Map buildCheckProperties() { properties.put(JdbcCheckKeys.TABLE_NAME_KEY, getTableName()); properties.put(JdbcCheckKeys.OPERATION_NAME_KEY, getName()); properties.put(JdbcCheckKeys.TABLE_TYPE_KEY, "sink"); - properties.put(JdbcCheckKeys.NEED_CHECK, ResourceCheck.NEED_CHECK+""); + properties.put(JdbcCheckKeys.NEED_CHECK, String.valueOf(ResourceCheck.NEED_CHECK)); return properties; } From 1abdfff3ebc6a22809389a302e50728e9dd11fc1 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 19 Jan 2021 14:53:11 +0800 Subject: [PATCH 73/80] [hotfix][root][pom] update flink version to 1.11.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a17287ff1..d69fadd1e 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ UTF-8 - 1.11.1 + 1.11.3 2.7.3 4.12 2.21.0 From a8f0afe62199937642d566b1570b95a2c1aef15e Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 19 Jan 2021 15:04:29 +0800 Subject: [PATCH 74/80] [hotfix][pom] update scala binary version to 2.12 --- core/pom.xml | 14 +++++++------- elasticsearch5/elasticsearch5-sink/pom.xml | 4 ++-- elasticsearch6/elasticsearch6-sink/pom.xml | 6 +++--- kafka-base/pom.xml | 2 +- kafka/kafka-source/pom.xml | 2 +- kafka/pom.xml | 2 +- kafka10/pom.xml | 2 +- kafka11/pom.xml | 2 +- launcher/pom.xml | 2 +- pom.xml | 1 + 10 files changed, 19 insertions(+), 18 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index dc827f2ba..117f0d031 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -32,37 +32,37 @@ org.apache.flink - flink-table-api-java-bridge_2.11 + flink-table-api-java-bridge_${scala.binary.version} ${flink.version} org.apache.flink - flink-table-planner-blink_2.11 + flink-table-planner-blink_${scala.binary.version} ${flink.version} org.apache.flink - flink-table-runtime-blink_2.11 + flink-table-runtime-blink_${scala.binary.version} ${flink.version} org.apache.flink - flink-streaming-java_2.11 + flink-streaming-java_${scala.binary.version} ${flink.version} org.apache.flink - flink-streaming-scala_2.11 + flink-streaming-scala_${scala.binary.version} ${flink.version} org.apache.flink - flink-cep-scala_2.11 + flink-cep-scala_${scala.binary.version} ${flink.version} @@ -74,7 +74,7 @@ org.apache.flink - flink-statebackend-rocksdb_2.11 + flink-statebackend-rocksdb_${scala.binary.version} ${flink.version} diff --git a/elasticsearch5/elasticsearch5-sink/pom.xml b/elasticsearch5/elasticsearch5-sink/pom.xml index 40e198279..7fee4b0ba 100644 --- a/elasticsearch5/elasticsearch5-sink/pom.xml +++ b/elasticsearch5/elasticsearch5-sink/pom.xml @@ -45,13 +45,13 @@ org.apache.flink - flink-connector-elasticsearch5_2.11 + flink-connector-elasticsearch5_${scala.binary.version} ${flink.version} org.apache.flink - flink-connector-elasticsearch-base_2.11 + flink-connector-elasticsearch-base_${scala.binary.version} ${flink.version} diff --git a/elasticsearch6/elasticsearch6-sink/pom.xml b/elasticsearch6/elasticsearch6-sink/pom.xml index 59690c251..6219e99dc 100644 --- a/elasticsearch6/elasticsearch6-sink/pom.xml +++ b/elasticsearch6/elasticsearch6-sink/pom.xml @@ -19,7 +19,7 @@ org.apache.flink - flink-streaming-java_2.11 + flink-streaming-java_${scala.binary.version} ${flink.version} provided @@ -50,13 +50,13 @@ org.apache.flink - flink-connector-elasticsearch6_2.11 + flink-connector-elasticsearch6_${scala.binary.version} ${flink.version} org.apache.flink - flink-connector-elasticsearch-base_2.11 + flink-connector-elasticsearch-base_${scala.binary.version} ${flink.version} diff --git a/kafka-base/pom.xml b/kafka-base/pom.xml index 9db588576..21933e70a 100644 --- a/kafka-base/pom.xml +++ b/kafka-base/pom.xml @@ -55,7 +55,7 @@ under the License. org.apache.flink - flink-connector-kafka_2.11 + flink-connector-kafka_${scala.binary.version} ${flink.version} provided diff --git a/kafka/kafka-source/pom.xml b/kafka/kafka-source/pom.xml index 0a337cf87..c817f41c6 100644 --- a/kafka/kafka-source/pom.xml +++ b/kafka/kafka-source/pom.xml @@ -20,7 +20,7 @@ sql.source.kafka-base - flink-connector-kafka_2.11 + flink-connector-kafka_${scala.binary.version} org.apache.flink diff --git a/kafka/pom.xml b/kafka/pom.xml index 772671ff6..fe39e9d11 100644 --- a/kafka/pom.xml +++ b/kafka/pom.xml @@ -20,7 +20,7 @@ org.apache.flink - flink-connector-kafka_2.11 + flink-connector-kafka_${scala.binary.version} ${flink.version} diff --git a/kafka10/pom.xml b/kafka10/pom.xml index d8ad7370f..fb980eb2b 100644 --- a/kafka10/pom.xml +++ b/kafka10/pom.xml @@ -20,7 +20,7 @@ org.apache.flink - flink-connector-kafka-0.10_2.11 + flink-connector-kafka-0.10_${scala.binary.version} ${flink.version} diff --git a/kafka11/pom.xml b/kafka11/pom.xml index 90972fa82..39e7172cd 100644 --- a/kafka11/pom.xml +++ b/kafka11/pom.xml @@ -20,7 +20,7 @@ org.apache.flink - flink-connector-kafka-0.11_2.11 + flink-connector-kafka-0.11_${scala.binary.version} ${flink.version} diff --git a/launcher/pom.xml b/launcher/pom.xml index b51ff8573..215a4c55d 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -19,7 +19,7 @@ org.apache.flink - flink-yarn_2.11 + flink-yarn_${scala.binary.version} ${flink.version} diff --git a/pom.xml b/pom.xml index d69fadd1e..f5d7422e2 100644 --- a/pom.xml +++ b/pom.xml @@ -48,6 +48,7 @@ UTF-8 1.11.3 + 2.12 2.7.3 4.12 2.21.0 From de1df552252e2dd975be951f3b1906183ef67840 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 19 Jan 2021 15:05:25 +0800 Subject: [PATCH 75/80] [hotfix][launcher] remove unuseful file --- launcher/job/kafkaNotNull.txt | 34 ----------------------- launcher/job/mysqlsideSql.txt | 52 ----------------------------------- launcher/job/sideSql.txt | 35 ----------------------- 3 files changed, 121 deletions(-) delete mode 100644 launcher/job/kafkaNotNull.txt delete mode 100644 launcher/job/mysqlsideSql.txt delete mode 100644 launcher/job/sideSql.txt diff --git a/launcher/job/kafkaNotNull.txt b/launcher/job/kafkaNotNull.txt deleted file mode 100644 index 0515f2036..000000000 --- a/launcher/job/kafkaNotNull.txt +++ /dev/null @@ -1,34 +0,0 @@ -CREATE TABLE MyTable( - channel varchar as sss not null, - pv INT not null, - xctime bigint, - CHARACTER_LENGTH(channel) as timeLeng, - WATERMARK FOR xctime AS withOffset(xctime,1000) - )WITH( - type='kafka11', - bootstrapServers='172.16.8.107:9092', - offsetReset='latest', - topic='toutou' - ); - -CREATE TABLE MyResult( - channel varchar, - pv INT - )WITH( - type='mysql', - url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='pv' - ); - - -insert -into - MyResult - select - a.channel, - a.pv - from - MyTable a - diff --git a/launcher/job/mysqlsideSql.txt b/launcher/job/mysqlsideSql.txt deleted file mode 100644 index 8dc9bf9ff..000000000 --- a/launcher/job/mysqlsideSql.txt +++ /dev/null @@ -1,52 +0,0 @@ -CREATE TABLE MyTable( - channel STRING, - pv INT, - xctime bigint, - CHARACTER_LENGTH(channel) as timeLeng, - WATERMARK FOR xctime AS withOffset(xctime,1000) - )WITH( - type='kafka09', - bootstrapServers='172.16.8.198:9092', - offsetReset='latest', - topic='nbTest1' - ); -CREATE TABLE MyResult( - channel STRING, - pv INT - )WITH( - type='mysql', - url='jdbc:mysql://172.16.8.104:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='pv' - ); - -create table sideTable( - channel String, - xccount int, - PRIMARY KEY(channel), - PERIOD FOR SYSTEM_TIME - )WITH( - type='mysql', - url='jdbc:mysql://172.16.8.104:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='sidetest', - cache = 'LRU', - cacheTTLMs='10000' - ); - -insert -into - MyResult - select - a.channel, - b.xccount - from - MyTable a - join - sideTable b - on a.channel=b.channel - where - b.channel = 'xc' - and a.pv=10; \ No newline at end of file diff --git a/launcher/job/sideSql.txt b/launcher/job/sideSql.txt deleted file mode 100644 index c4bb76a32..000000000 --- a/launcher/job/sideSql.txt +++ /dev/null @@ -1,35 +0,0 @@ -CREATE TABLE STREAM_FIELD20( - after.id int AS id, - after.field02 varchar AS field02, - after.field03 varchar AS field03, - after.field04 varchar AS field04, - after.field05 varchar AS field05, - after.field06 varchar AS field06 - )WITH( - type ='kafka10', - bootstrapServers ='172.16.101.247:9092', - zookeeperQuorum ='172.16.101.247:2181,172.16.101.141:2181,172.16.100.214:2181/kafka', - offsetReset ='latest', - topic ='toutiantest', - timezone='Asia/Shanghai', - topicIsPattern ='false', - parallelism ='1' - ); - -CREATE TABLE RESULT_FIELD20( - id INT, - field02 VARCHAR, - PRIMARY KEY(id) - )WITH( - type='console' - ); - - -insert -into - RESULT_FIELD20 - SELECT - a.id AS id, - a.field02 as field02 - from - STREAM_FIELD20 a \ No newline at end of file From e5a03c503e06da1751aa4fd48010c9ea103000e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Mon, 21 Dec 2020 19:32:45 +0800 Subject: [PATCH 76/80] =?UTF-8?q?[github]=20=E4=BF=AE=E6=94=B9readme?= =?UTF-8?q?=E6=8B=9B=E8=81=98=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f804f3f4..4207fd0ca 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,18 @@ FlinkStreamSQL [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) ## 技术交流 -- 招聘**大数据平台开发工程师**，想了解岗位详细信息可以添加本人微信号ysqwhiletrue，注明招聘，如有意者发送简历至[sishu@dtstack.com](mailto:sishu@dtstack.com) +- 招聘**Flink开发工程师**，如果有兴趣，请联系思枢【微信号ysqwhiletrue】，注明招聘
+ Flink开发工程师JD要求：
+ 1.负责袋鼠云基于Flink的衍生框架数据同步flinkx和实时计算flinkstreamsql框架的开发；
+ 2.调研和把握当前最新大数据实时计算技术，将其中的合适技术引入到平台中，改善产品，提升竞争力；
+ 职位要求：
+ 1、本科及以上学历，3年及以上的Flink开发经验，精通Java，熟悉Scala、Python优先考虑；
+ 2、熟悉Flink原理，有基于Flink做过二次源码的开发，在github上贡献者Flink源码者优先；
+ 3、有机器学习、数据挖掘相关经验者优先；
+ 4、对新技术有快速学习和上手能力，对代码有一定的洁癖；
+ 加分项：
+ 1.在GitHub或其他平台上有过开源项目;
+ 可以添加本人微信号ysqwhiletrue，注明招聘，如有意者发送简历至sishu@dtstack.com
- 我们使用[钉钉](https://www.dingtalk.com/)沟通交流，可以搜索群号[**30537511**]或者扫描下面的二维码进入钉钉群
From c9a52951150c5e3f395521cf2b71b81bc9da6642 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 12 Jan 2021 20:39:15 +0800 Subject: [PATCH 77/80] [hotfix][root][pom] comment aws plugin --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f5d7422e2..d95474c65 100644 --- a/pom.xml +++ b/pom.xml @@ -36,7 +36,7 @@ elasticsearch6 cassandra kudu - aws + dirtyData From 7f277ea579b86cdb27d1d78d2170aa7a54ba9a2a Mon Sep 17 00:00:00 2001 From: chuixue Date: Tue, 19 Jan 2021 15:46:39 +0800 Subject: [PATCH 78/80] [feat-31342][rdb] load Driver class use ClassLoaderManager --- .../flink/sql/core/rdb/util/JdbcConnectUtil.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java index 41e01f0f4..a5a33d51b 100644 --- a/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java +++ b/rdb/rdb-core/src/main/java/com/dtstack/flink/sql/core/rdb/util/JdbcConnectUtil.java @@ -136,7 +136,7 @@ public static Connection getConnectWithRetry( + "\nerror message: "); String errorCause = null; - forName(driverName); + ClassLoaderManager.forName(driverName, JdbcConnectUtil.class.getClassLoader()); Preconditions.checkNotNull(url, "url can't be null!"); for (int i = 0; i < DEFAULT_RETRY_NUM; i++) { @@ -157,16 +157,4 @@ public static Connection getConnectWithRetry( } throw new SuppressRestartsException(new Throwable(errorMessage + errorCause)); } - - /** - * @param clazz - */ - public synchronized static void forName(String clazz) { - try { - Class driverClass = Class.forName(clazz); - driverClass.newInstance(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } } From 43a84c7f67a90b38997beb898e764bc31cc557c4 Mon Sep 17 00:00:00 2001 From: chuixue Date: Wed, 20 Jan 2021 09:34:16 +0800 Subject: [PATCH 79/80] [feat-31342][impala] add impala connection check --- .../flink/sql/side/impala/table/ImpalaSideParser.java | 3 +++ .../flink/sql/side/impala/table/ImpalaSideTableInfo.java | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java index 4a35162d5..318ef6066 100644 --- a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java +++ b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.impala.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import com.fasterxml.jackson.databind.ObjectMapper; @@ -61,6 +62,8 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Wed, 20 Jan 2021 15:03:06 +0800 Subject: [PATCH 80/80] [hotfix-34329] optimize sampling print data record. --- .../sink/cassandra/CassandraOutputFormat.java | 8 +-- .../flink/sql/constrant/ConfigConstrant.java | 2 +- .../flink/sql/exec/ExecuteProcessHelper.java | 12 +++- .../format/DeserializationMetricWrapper.java | 14 +++-- .../AbstractDtRichOutputFormat.java | 3 +- .../dtstack/flink/sql/util/SampleUtils.java | 59 +++++++++++++++++++ docs/config.md | 1 + .../sql/sink/hbase/HbaseOutputFormat.java | 12 ++-- .../sql/sink/impala/ImpalaOutputFormat.java | 6 +- .../flink/sql/sink/kudu/KuduOutputFormat.java | 7 +-- .../flink/sql/launcher/LauncherMain.java | 1 - .../sql/sink/mongo/MongoOutputFormat.java | 7 +-- .../rdb/format/JDBCUpsertOutputFormat.java | 10 +--- .../sql/sink/redis/RedisOutputFormat.java | 7 +-- 14 files changed, 104 insertions(+), 45 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/util/SampleUtils.java diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java index 4428fbc76..e5ea13ccc 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java @@ -38,6 +38,7 @@ package com.dtstack.flink.sql.sink.cassandra; +import com.dtstack.flink.sql.util.SampleUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.api.java.tuple.Tuple2; @@ -192,7 +193,7 @@ public void open(int taskNumber, int numTasks) { * @see PreparedStatement */ @Override - public void writeRecord(Tuple2 tuple2) throws IOException { + public void writeRecord(Tuple2 tuple2) { Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); Row row = tupleTrans.getField(1); @@ -209,10 +210,7 @@ public void writeRecord(Tuple2 tuple2) throws IOException { private void insertWrite(Row row) { try { - - if(outRecords.getCount() % ROW_PRINT_FREQUENCY == 0){ - LOG.info("Receive data : {}", row); - } + SampleUtils.samplingSinkPrint(samplingIntervalCount, LOG, outRecords.getCount(), row.toString()); String cql = buildSql(row); if (cql != null) { diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java index 69a4bdcf8..76f01aea6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java @@ -67,7 +67,7 @@ public class ConfigConstrant { public static final String STATE_BACKEND_KEY = "state.backend"; public static final String CHECKPOINTS_DIRECTORY_KEY = "state.checkpoints.dir"; public static final String STATE_BACKEND_INCREMENTAL_KEY = "state.backend.incremental"; - + public static final String SAMPLE_INTERVAL_COUNT = "sample.interval.count"; public static final String RESTOREENABLE = "restore.enable"; public static final String LOG_LEVEL_KEY = "logLevel"; diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index e438587d8..767dda481 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -48,6 +48,7 @@ import com.dtstack.flink.sql.util.DataTypeUtils; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; +import com.dtstack.flink.sql.util.SampleUtils; import com.dtstack.flink.sql.util.SqlFormatterUtil; import com.dtstack.flink.sql.watermarker.WaterMarkerAssigner; import com.fasterxml.jackson.databind.ObjectMapper; @@ -92,6 +93,8 @@ import java.util.Set; import java.util.TimeZone; +import static com.dtstack.flink.sql.constrant.ConfigConstrant.SAMPLE_INTERVAL_COUNT; + /** * 任务执行时的流程方法 * Date: 2020/2/17 @@ -177,7 +180,6 @@ public static boolean checkRemoteSqlPluginPath(String remoteSqlPluginPath, Strin return true; } - public static StreamTableEnvironment getStreamExecution(ParamsInfo paramsInfo) throws Exception { ClassLoader envClassLoader = Thread.currentThread().getContextClassLoader(); @@ -187,6 +189,7 @@ public static StreamTableEnvironment getStreamExecution(ParamsInfo paramsInfo) t StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExeEnv(paramsInfo.getConfProp(), paramsInfo.getDeployMode()); StreamTableEnvironment tableEnv = getStreamTableEnv(env, paramsInfo.getConfProp()); + setSamplingIntervalCount(paramsInfo); ResourceCheck.NEED_CHECK = Boolean.parseBoolean(paramsInfo.getConfProp().getProperty(ResourceCheck.CHECK_STR, "true")); String planner = paramsInfo.getPlanner(); @@ -228,6 +231,13 @@ public static StreamTableEnvironment getStreamExecution(ParamsInfo paramsInfo) t return tableEnv; } + private static void setSamplingIntervalCount(ParamsInfo paramsInfo) { + SampleUtils.setSamplingIntervalCount( + Integer.parseInt( + paramsInfo.getConfProp().getProperty(SAMPLE_INTERVAL_COUNT, "0") + ) + ); + } public static List getExternalJarUrls(String addJarListStr) throws java.io.IOException { List jarUrlList = Lists.newArrayList(); diff --git a/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java b/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java index 5696c1f61..323defe6e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java @@ -20,6 +20,7 @@ import com.dtstack.flink.sql.dirtyManager.manager.DirtyDataManager; import com.dtstack.flink.sql.metric.MetricConstant; +import com.dtstack.flink.sql.util.SampleUtils; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.api.common.serialization.AbstractDeserializationSchema; import org.apache.flink.api.common.serialization.DeserializationSchema; @@ -45,7 +46,7 @@ public class DeserializationMetricWrapper extends AbstractDeserializationSchema< private static final Logger LOG = LoggerFactory.getLogger(DeserializationMetricWrapper.class); - private static int dataPrintFrequency = 1000; + private int samplingIntervalCount = SampleUtils.getSamplingIntervalCount(); private DeserializationSchema deserializationSchema; @@ -96,11 +97,14 @@ public void initMetric() { } @Override - public Row deserialize(byte[] message) throws IOException { + public Row deserialize(byte[] message) { try { - if (numInRecord.getCount() % dataPrintFrequency == 0) { - LOG.info("receive source data:" + new String(message, StandardCharsets.UTF_8)); - } + SampleUtils.samplingSourcePrint( + samplingIntervalCount, + LOG, + numInRecord.getCount(), + new String(message, StandardCharsets.UTF_8) + ); numInRecord.inc(); numInBytes.inc(message.length); beforeDeserialize(); diff --git a/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java b/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java index fbcc86bbd..91b6df46c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java +++ b/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.outputformat; import com.dtstack.flink.sql.metric.MetricConstant; +import com.dtstack.flink.sql.util.SampleUtils; import org.apache.flink.api.common.io.RichOutputFormat; import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.Meter; @@ -32,8 +33,8 @@ public abstract class AbstractDtRichOutputFormat extends RichOutputFormat{ public transient Counter outRecords; public transient Counter outDirtyRecords; public transient Meter outRecordsRate; + protected int samplingIntervalCount = SampleUtils.getSamplingIntervalCount(); - protected static int ROW_PRINT_FREQUENCY = 1000; protected static int DIRTY_PRINT_FREQUENCY = 1000; public void initMetric() { diff --git a/core/src/main/java/com/dtstack/flink/sql/util/SampleUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/SampleUtils.java new file mode 100644 index 000000000..b7de6e2d8 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/util/SampleUtils.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.util; + +import org.slf4j.Logger; + +/** + * @program: flinkStreamSQL + * @author: wuren + * @create: 2021/01/19 + **/ +public class SampleUtils { + + private static int samplingIntervalCount = 0; + + /** + * static变量无法序列化到TaskManager，所以设定此方法，赋值给每个具体使用的类。 + * @return + */ + public static int getSamplingIntervalCount() { + return samplingIntervalCount; + } + + public static void setSamplingIntervalCount(int interval) { + samplingIntervalCount = interval; + } + + public static void samplingSourcePrint(int samplingIntervalCount, Logger logger, long count, String message) { + Runnable func = () -> logger.info("sampling source input data: " + message); + samplingPrint(samplingIntervalCount, count, func); + } + + public static void samplingSinkPrint(int samplingIntervalCount, Logger logger, long count, String message) { + Runnable func = () -> logger.info("sampling sink output data: " + message); + samplingPrint(samplingIntervalCount, count, func); + } + + private static void samplingPrint(int samplingIntervalCount, long count, Runnable func) { + if (samplingIntervalCount > 0 && count % samplingIntervalCount == 0) { + func.run(); + } + } +} diff --git a/docs/config.md b/docs/config.md index aa1860800..f618f86c8 100644 --- a/docs/config.md +++ b/docs/config.md @@ -79,6 +79,7 @@ sh submit.sh -key1 val1 -key2 val2 * savePointPath：任务恢复点的路径（默认无） * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） * logLevel: 日志级别动态配置（默认info） + * sample.interval.count：间隔一定数据条数后，将本次进入Flink的数据抽样打印到日志中。默认为0，不进行抽样打印。可以设置一个整数，例如：1000000。 * [prometheus 相关参数](./prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 434ba694d..433c90666 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -22,6 +22,7 @@ import com.dtstack.flink.sql.dirtyManager.manager.DirtyDataManager; import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.util.SampleUtils; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; @@ -241,11 +242,8 @@ protected synchronized void dealBatchOperation(List records) { outRecords.inc(); } } - // 打印结果 - if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0) { - // 只打印最后一条数据 - LOG.info(records.get(records.size() - 1).toString()); - } + // 只打印最后一条数据 + SampleUtils.samplingSinkPrint(samplingIntervalCount, LOG, outRecords.getCount(), records.get(records.size() - 1).toString()); } catch (IOException | InterruptedException e) { LOG.error("", e); } finally { @@ -271,9 +269,7 @@ protected void dealInsert(Row record) { outDirtyRecords.inc(); } - if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0) { - LOG.info(record.toString()); - } + SampleUtils.samplingSinkPrint(samplingIntervalCount, LOG, outRecords.getCount(), record.toString()); outRecords.inc(); } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index e0b9eafce..70a021e46 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -24,6 +24,7 @@ import com.dtstack.flink.sql.sink.rdb.JDBCTypeConvertUtils; import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.KrbUtils; +import com.dtstack.flink.sql.util.SampleUtils; import com.google.common.collect.Maps; import org.apache.commons.collections.CollectionUtils; import org.apache.flink.api.java.tuple.Tuple2; @@ -78,7 +79,6 @@ public class ImpalaOutputFormat extends AbstractDtRichOutputFormat record) throws IOException { return; } - if (outRecords.getCount() % RECEIVE_DATA_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { - LOG.info("Receive data : {}", record); - } + SampleUtils.samplingSinkPrint(samplingIntervalCount, LOG, outRecords.getCount(), record.toString()); if (updateMode.equalsIgnoreCase(UPDATE_MODE)) { rows.add(Row.copy(record.f1)); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index e8b508f1f..ea5d68416 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -20,6 +20,7 @@ import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.util.KrbUtils; +import com.dtstack.flink.sql.util.SampleUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; @@ -131,7 +132,7 @@ private void establishConnection() throws IOException { } @Override - public void writeRecord(Tuple2 record) throws IOException { + public void writeRecord(Tuple2 record) { Tuple2 tupleTrans = record; Boolean retract = tupleTrans.getField(0); if (!retract) { @@ -148,9 +149,7 @@ public void writeRecord(Tuple2 record) throws IOException { } try { - if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0) { - LOG.info("Receive data : {}", row); - } + SampleUtils.samplingSinkPrint(samplingIntervalCount, LOG, outRecords.getCount(), row.toString()); session.apply(toOperation(writeMode, row)); outRecords.inc(); } catch (KuduException e) { diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 07b2fa4a2..1f354fe35 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -137,7 +137,6 @@ private static String[] parseJson(String[] args) { return list.toArray(new String[0]); } - public static void main(String[] args) throws Exception { JobParamsInfo jobParamsInfo = parseArgs(args); ClusterMode execMode = ClusterMode.valueOf(jobParamsInfo.getMode()); diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index df8293522..3f1bdebb4 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.sink.mongo; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.util.SampleUtils; import com.mongodb.MongoClient; import com.mongodb.MongoClientURI; import com.mongodb.client.MongoCollection; @@ -70,7 +71,7 @@ public void open(int taskNumber, int numTasks) throws IOException { } @Override - public void writeRecord(Tuple2 tuple2) throws IOException { + public void writeRecord(Tuple2 tuple2) { Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); @@ -102,9 +103,7 @@ public void writeRecord(Tuple2 tuple2) throws IOException { dbCollection.insertOne(doc); } - if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0){ - LOG.info(record.toString()); - } + SampleUtils.samplingSinkPrint(samplingIntervalCount, LOG, outRecords.getCount(), record.toString()); outRecords.inc(); } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index 55d9a0ca7..5626def9e 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -27,6 +27,7 @@ import com.dtstack.flink.sql.sink.rdb.writer.AbstractUpsertWriter; import com.dtstack.flink.sql.sink.rdb.writer.AppendOnlyWriter; import com.dtstack.flink.sql.sink.rdb.writer.JDBCWriter; +import com.dtstack.flink.sql.util.SampleUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; @@ -55,8 +56,6 @@ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat tuple2) throws IOException { + public synchronized void writeRecord(Tuple2 tuple2) { checkConnectionOpen(); try { - if (outRecords.getCount() % RECEIVEDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { - LOG.info("Receive data : {}", tuple2); - } - // Receive data + SampleUtils.samplingSinkPrint(samplingIntervalCount, LOG, outRecords.getCount(), tuple2.toString()); outRecords.inc(); jdbcWriter.addRecord(tuple2); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index 801eebc08..311676929 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -20,6 +20,7 @@ import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.sink.redis.enums.RedisType; +import com.dtstack.flink.sql.util.SampleUtils; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; @@ -148,7 +149,7 @@ private void establishConnection() { } @Override - public void writeRecord(Tuple2 record) throws IOException { + public void writeRecord(Tuple2 record) { Tuple2 tupleTrans = record; Boolean retract = tupleTrans.getField(0); if (!retract) { @@ -163,9 +164,7 @@ public void writeRecord(Tuple2 record) throws IOException { refData.put(fieldNames[i], row.getField(i)); } save(refData); - if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0) { - LOG.info(record.toString()); - } + SampleUtils.samplingSinkPrint(samplingIntervalCount, LOG, outRecords.getCount(), row.toString()); outRecords.inc(); }