diff --git a/connectors/rocketmq-connect-jdbc/README.md b/connectors/rocketmq-connect-jdbc/README.md index 003c91e63..1ed4cd29f 100644 --- a/connectors/rocketmq-connect-jdbc/README.md +++ b/connectors/rocketmq-connect-jdbc/README.md @@ -1,20 +1,24 @@ # rocketmq-connect-jdbc -为方便扩展，rocketmq-connect-jdbc目前采用模块化插件的形式进行组织，内部的connector需要用户手动编译成jar包来进行使用。目前支持Mysql、OpenMLDB等数据库 +为方便扩展，rocketmq-connect-jdbc目前采用Spi插件的形式进行扩展，核心扩展api主要有： + ```SPI api + org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialectFactory + org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect + ``` +目前支持Mysql、OpenMLDB数据库, pg、oracle、sqlserver、db2 等关系型数据库还在持续扩展中 ## rocketmq-connect-jdbc使用方法 -1. 进入想要使用的connector - 目录下（以rocketmq-connect-jdbc-mysql目录为例），使用以下指令将此connector打包为jar文件 - +1. 进入想要使用的connectors目录下（以rocketmq-connect-jdbc目录为例），使用以下指令将插件进行打包 ```shell mvn clean package -Dmaven.test.skip=true ``` -2. 打包好的jar文件将出现在`rocketmq-connect-jdbc-mysql/target/`目录下 +2. 打包好的插件以tar.gz的模式出现在`rocketmq-connect-jdbc/target/`目录下 + 3. 在`distribution/conf`目录下找的对应的配置文件进行更新，对于standalone的启动方式，更新`connect-standalone.conf`文件中的`pluginPaths`变量 ```lombok.config - pluginPaths=rocketmq-connect-sample/target/rocketmq-connect-jdbc-mysql-0.0.1-SNAPSHOT-jar-with-dependencies.jar + pluginPaths=(you plugin path) ``` 相应的，使用distributed启动方式，则更新`connect-distributed.conf`中的变量 @@ -40,7 +44,7 @@ mvn clean package -Dmaven.test.skip=true ``` POST http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-jdbc-source-connector-name} { - "connector.class":"org.apache.rocketmq.connect.jdbc.mysql.source.BaseSourceConnector", + "connector.class":"org.apache.rocketmq.connect.jdbc.mysql.source.JdbcSourceConnector", "max.tasks":"2", "connection.url":"jdbc:mysql://XXXXXXXXX:3306", "connection.user":"*****", @@ -92,7 +96,7 @@ http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-jdbc-connector-name}/ * **jdbc-source-connector 参数说明** | KEY | TYPE | Must be filled | Description | Example | -| ------------------------ | ------- | -------------- | ---------------- | --------------------------------------------------------- | +|--------------------------| ------- | -------------- |------------------| --------------------------------------------------------- | | connection.url | String | YES | source端 jdbc连接 | jdbc:mysql://XXXXXXXXX:3306 | | connection.user | String | YES | source端 DB 用户名 | root | | connection.password | String | YES | source端 DB 密码 | root | @@ -104,8 +108,9 @@ http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-jdbc-connector-name}/ | incrementing.column.name | Integer | NO | 增量字段，常用ID | id | | timestamp.column.name | String | YES | 时间增量字段 | modified_time | | table.whitelist | String | YES | 需要扫描的表 | db.table,db.table01 | -| max-task | Integer | YES | 任务数量，最大不能大于表的数量 | 2 | -| source-record-converter | Integer | YES | data转换器 | org.apache.rocketmq.connect.doris.converter.JsonConverter | +| max.tasks | Integer | YES | 任务数量，最大不能大于表的数量 | 2 | +| key.converter | Integer | YES | key转换器 | org.apache.rocketmq.connect.doris.converter.JsonConverter | +| value.converter | Integer | YES | data转换器 | org.apache.rocketmq.connect.doris.converter.JsonConverter | ``` 注：1.source拉取的数据写入到以表名自动创建的topic中，如果需要写入特定的topic中则需要指定"connect-topicname" 参数 @@ -114,14 +119,15 @@ http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-jdbc-connector-name}/ * **jdbc-sink-connector 参数说明** -| KEY | TYPE | Must be filled | Description | Example | -| ----------------------- | ------- | -------------- | --------------- | --------------------------------------------------------- | -| connection.url | String | YES | sink端 jdbc连接 | jdbc:mysql://XXXXXXXXX:3306 | -| connection.user | String | YES | sink端 DB 用户名 | root | -| connection.password | String | YES | sink端 DB 密码 | root | -| host | String | YES | doris host | 192.168.0.1 | -| port | String | YES | doris http port | 8030 | -| user | String | YES | 监听的topic | root | -| passwd | String | YES | 监听的topic | passwd | -| max-task | Integer | NO | 任务数量 | 2 | -| source-record-converter | Integer | YES | data转换器 | org.apache.rocketmq.connect.doris.converter.JsonConverter | +| KEY | TYPE | Must be filled | Description | Example | +|---------------------| ------- | -------------- |-----------------| --------------------------------------------------------- | +| connection.url | String | YES | sink端 jdbc连接 | jdbc:mysql://XXXXXXXXX:3306 | +| connection.user | String | YES | sink端 DB 用户名 | root | +| connection.password | String | YES | sink端 DB 密码 | root | +| host | String | YES | doris host | 192.168.0.1 | +| port | String | YES | doris http port | 8030 | +| user | String | YES | 监听的topic | root | +| passwd | String | YES | 监听的topic | passwd | +| max.tasks | Integer | NO | 任务数量 | 2 | +| key.converter | Integer | YES | key转换器 | org.apache.rocketmq.connect.doris.converter.JsonConverter | +| value.converter | Integer | YES | data转换器 | org.apache.rocketmq.connect.doris.converter.JsonConverter | diff --git a/connectors/rocketmq-connect-jdbc/pom.xml b/connectors/rocketmq-connect-jdbc/pom.xml index de4470878..4d88a902c 100644 --- a/connectors/rocketmq-connect-jdbc/pom.xml +++ b/connectors/rocketmq-connect-jdbc/pom.xml @@ -16,18 +16,12 @@ 4.0.0 org.apache.rocketmq rocketmq-connect-jdbc - pom + jar 0.0.1-SNAPSHOT - - rocketmq-connect-jdbc-core - rocketmq-connect-jdbc-mysql - rocketmq-connect-jdbc-openmldb - rocketmq-connect-jdbc https://github.com/apache/incubator-rocketmq-externals/tree/master/rocketmq-connect-jdbc - The Apache Software License, Version 2.0 @@ -73,56 +67,35 @@ - - org.apache.rocketmq - rocketmq-connect-jdbc-core - ${project.version} - - io.openmessaging openmessaging-connector ${openmessaging-connector.version} + provided io.openmessaging openmessaging-api ${openmessaging-api.version} + provided org.apache.commons commons-lang3 ${commons-lang3.version} + provided commons-codec commons-codec ${commons-codec.version} + provided commons-cli commons-cli ${commons-cli.version} - - - - - junit - junit - ${junit.version} - test - - - org.assertj - assertj-core - ${assertj.version} - test - - - org.mockito - mockito-core - ${mockito.version} - test + provided @@ -130,39 +103,175 @@ com.alibaba fastjson ${fastjson.version} + provided org.slf4j slf4j-api ${slf4j-api.version} + provided ch.qos.logback logback-classic ${logback-classic.version} + provided ch.qos.logback logback-core ${logback-core.version} + provided io.debezium debezium-core ${debezium.version} + provided + + - org.projectlombok - lombok - 1.18.2 + junit + junit + ${junit.version} + test + + + org.assertj + assertj-core + ${assertj.version} + test + + org.mockito + mockito-core + ${mockito.version} + test + + + + + + mysql + mysql-connector-java + 8.0.30 + provided + + + + + com.4paradigm.openmldb + openmldb-native + 0.5.0-macos + provided + + + com.4paradigm.openmldb + openmldb-jdbc + 0.5.0 + provided + + + com.4paradigm.openmldb + openmldb-native + + + org.projectlombok + lombok + + + + + + io.openmessaging + openmessaging-connector + + + io.openmessaging + openmessaging-api + + + org.apache.commons + commons-lang3 + + + commons-codec + commons-codec + + + commons-cli + commons-cli + + + + + com.alibaba + fastjson + + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + + io.debezium + debezium-core + + + + + + junit + junit + test + + + org.assertj + assertj-core + test + + + org.mockito + mockito-core + test + + + + + + + mysql + mysql-connector-java + + + + + com.4paradigm.openmldb + openmldb-native + + + com.4paradigm.openmldb + openmldb-jdbc + + + @@ -263,6 +372,24 @@ findbugs-maven-plugin 3.0.4 + + maven-assembly-plugin + 3.0.0 + + + src/assembly/assembly.xml + + + + + make-assembly + package + + single + + + + maven-checkstyle-plugin 2.17 @@ -271,7 +398,7 @@ verify verify - ../../style/rmq_checkstyle.xml + ../../../style/rmq_checkstyle.xml UTF-8 true true @@ -286,4 +413,5 @@ - + + \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/pom.xml b/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/pom.xml deleted file mode 100644 index 1bb310cf6..000000000 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/pom.xml +++ /dev/null @@ -1,95 +0,0 @@ - - - - rocketmq-connect-jdbc - org.apache.rocketmq - 0.0.1-SNAPSHOT - - 4.0.0 - - rocketmq-connect-jdbc-core - - - 8 - 8 - UTF-8 - - - - - - - io.openmessaging - openmessaging-connector - - - io.openmessaging - openmessaging-api - - - - org.apache.commons - commons-lang3 - - - - - - junit - junit - test - - - org.assertj - assertj-core - test - - - org.mockito - mockito-core - test - - - - - com.alibaba - fastjson - - - - org.slf4j - slf4j-api - - - ch.qos.logback - logback-classic - - - ch.qos.logback - logback-core - - - - commons-codec - commons-codec - - - commons-cli - commons-cli - - - - io.debezium - debezium-core - - - - org.projectlombok - lombok - provided - - - - \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/config/ConfigDef.java b/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/config/ConfigDef.java deleted file mode 100644 index 8bd7ca59f..000000000 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/config/ConfigDef.java +++ /dev/null @@ -1,582 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.connect.jdbc.config; - -import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.connect.jdbc.exception.ConfigException; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.function.BiConsumer; -import java.util.function.Supplier; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -/** - * config def - */ -public class ConfigDef { - - private static final Pattern COMMA_WITH_WHITESPACE = Pattern.compile("\\s*,\\s*"); - /** - * A unique Java object which represents the lack of a default value. - */ - public static final Object NO_DEFAULT_VALUE = new Object(); - - private final Map configKeys; - - public ConfigDef() { - configKeys = new LinkedHashMap<>(); - } - - - public ConfigDef define(ConfigKey key) { - if (configKeys.containsKey(key.name)) { - throw new ConfigException("Configuration " + key.name + " is defined twice."); - } - configKeys.put(key.name, key); - return this; - } - - /** - * no validate - * - * @param name - * @param type - * @param defaultValue - * @param validator - * @param documentation - * @return - */ - public ConfigDef define(String name, Type type, Object defaultValue, Validator validator, String documentation) { - return define(new ConfigKey(name, type, defaultValue, validator, documentation)); - } - - /** - * validate config - * - * @param name - * @param type - * @param defaultValue - * @param documentation - * @return - */ - public ConfigDef define(String name, Type type, Object defaultValue, String documentation) { - return define(new ConfigKey(name, type, defaultValue, null, documentation)); - } - - - /** - * Get the configuration keys - * - * @return a map containing all configuration keys - */ - public Map configKeys() { - return configKeys; - } - - /** - * parse props - * - * @param props - * @return - */ - public Map parse(Map props) { - // parse all known keys - Map values = new HashMap<>(); - for (ConfigKey key : configKeys.values()) { - values.put(key.name, parseValue(key, props.get(key.name), props.containsKey(key.name))); - } - return values; - } - - /** - * convert + validate - * - * @param key - * @param value - * @param isSet - * @return - */ - Object parseValue(ConfigKey key, Object value, boolean isSet) { - Object parsedValue; - if (isSet) { - parsedValue = parseType(key.name, value, key.type); - // props map doesn't contain setting, the key is required because no default value specified - its an error - } else if (NO_DEFAULT_VALUE.equals(key.defaultValue)) { - throw new ConfigException("Missing required configuration \"" + key.name + "\" which has no default value."); - } else { - parsedValue = key.defaultValue; - } - if (key.validator != null) { - key.validator.ensureValid(key.name, parsedValue); - } - return parsedValue; - } - - - /** - * Parse a value according to its expected type. - * - * @param name The config name - * @param value The config value - * @param type The expected type - * @return The parsed object - */ - public static Object parseType(String name, Object value, Type type) { - try { - if (value == null) { - return null; - } - String trimmed = null; - if (value instanceof String) { - trimmed = ((String) value).trim(); - } - switch (type) { - case BOOLEAN: - if (value instanceof String) { - if (trimmed.equalsIgnoreCase("true")) { - return true; - } else if (trimmed.equalsIgnoreCase("false")) { - return false; - } else { - throw new ConfigException(name, value, "Expected value to be either true or false"); - } - } else if (value instanceof Boolean) { - return value; - } else { - throw new ConfigException(name, value, "Expected value to be either true or false"); - } - case STRING: - if (value instanceof String) { - return trimmed; - } else { - throw new ConfigException(name, value, "Expected value to be a string, but it was a " + value.getClass().getName()); - } - case INT: - if (value instanceof Integer) { - return value; - } else if (value instanceof String) { - return Integer.parseInt(trimmed); - } else { - throw new ConfigException(name, value, "Expected value to be a 32-bit integer, but it was a " + value.getClass().getName()); - } - case SHORT: - if (value instanceof Short) { - return value; - } else if (value instanceof String) { - return Short.parseShort(trimmed); - } else { - throw new ConfigException(name, value, "Expected value to be a 16-bit integer (short), but it was a " + value.getClass().getName()); - } - case LONG: - if (value instanceof Integer) { - return ((Integer) value).longValue(); - } - if (value instanceof Long) { - return value; - } else if (value instanceof String) { - return Long.parseLong(trimmed); - } else { - throw new ConfigException(name, value, "Expected value to be a 64-bit integer (long), but it was a " + value.getClass().getName()); - } - case DOUBLE: - if (value instanceof Number) { - return ((Number) value).doubleValue(); - } else if (value instanceof String) { - return Double.parseDouble(trimmed); - } else { - throw new ConfigException(name, value, "Expected value to be a double, but it was a " + value.getClass().getName()); - } - case LIST: - if (value instanceof List) { - return value; - } else if (value instanceof String) { - if (trimmed.isEmpty()) { - return Collections.emptyList(); - } else { - return Arrays.asList(COMMA_WITH_WHITESPACE.split(trimmed, -1)); - } - } else { - throw new ConfigException(name, value, "Expected a comma separated list."); - } - default: - throw new IllegalStateException("Unknown type."); - } - } catch (NumberFormatException e) { - throw new ConfigException(name, value, "Not a number of type " + type); - } - } - - public static String convertToString(Object parsedValue, Type type) { - if (parsedValue == null) { - return null; - } - - if (type == null) { - return parsedValue.toString(); - } - - switch (type) { - case BOOLEAN: - case SHORT: - case INT: - case LONG: - case DOUBLE: - case STRING: - case PASSWORD: - return parsedValue.toString(); - case LIST: - List valueList = (List) parsedValue; - return StringUtils.join(valueList, ","); - case CLASS: - Class clazz = (Class) parsedValue; - return clazz.getName(); - default: - throw new IllegalStateException("Unknown type."); - } - } - - /** - * The config types - */ - public enum Type { - BOOLEAN, STRING, INT, SHORT, LONG, DOUBLE, LIST, CLASS, PASSWORD - } - - - /** - * Validation logic the user may provide to perform single configuration validation. - */ - public interface Validator { - /** - * Perform single configuration validation. - * - * @param name The name of the configuration - * @param value The value of the configuration - * @throws ConfigException if the value is invalid. - */ - void ensureValid(String name, Object value); - } - - /** - * Validation logic for numeric ranges - */ - public static class Range implements Validator { - private final Number min; - private final Number max; - - /** - * A numeric range with inclusive upper bound and inclusive lower bound - * - * @param min the lower bound - * @param max the upper bound - */ - private Range(Number min, Number max) { - this.min = min; - this.max = max; - } - - /** - * A numeric range that checks only the lower bound - * - * @param min The minimum acceptable value - */ - public static Range atLeast(Number min) { - return new Range(min, null); - } - - /** - * A numeric range that checks both the upper (inclusive) and lower bound - */ - public static Range between(Number min, Number max) { - return new Range(min, max); - } - - @Override - public void ensureValid(String name, Object o) { - if (o == null) { - throw new ConfigException(name, null, "Value must be non-null"); - } - Number n = (Number) o; - if (min != null && n.doubleValue() < min.doubleValue()) { - throw new ConfigException(name, o, "Value must be at least " + min); - } - if (max != null && n.doubleValue() > max.doubleValue()) { - throw new ConfigException(name, o, "Value must be no more than " + max); - } - } - - @Override - public String toString() { - if (min == null && max == null) { - return "[...]"; - } else if (min == null) { - return "[...," + max + "]"; - } else if (max == null) { - return "[" + min + ",...]"; - } else { - return "[" + min + ",...," + max + "]"; - } - } - } - - public static class ValidList implements Validator { - - final ValidString validString; - - private ValidList(List validStrings) { - this.validString = new ValidString(validStrings); - } - - public static ValidList in(String... validStrings) { - return new ValidList(Arrays.asList(validStrings)); - } - - @Override - public void ensureValid(final String name, final Object value) { - @SuppressWarnings("unchecked") - List values = (List) value; - for (String string : values) { - validString.ensureValid(name, string); - } - } - - @Override - public String toString() { - return validString.toString(); - } - } - - public static class ValidString implements Validator { - final List validStrings; - - private ValidString(List validStrings) { - this.validStrings = validStrings; - } - - public static ValidString in(String... validStrings) { - return new ValidString(Arrays.asList(validStrings)); - } - - @Override - public void ensureValid(String name, Object o) { - String s = (String) o; - if (!validStrings.contains(s)) { - throw new ConfigException(name, o, "String must be one of: " + StringUtils.join(validStrings, ", ")); - } - - } - - @Override - public String toString() { - return "[" + StringUtils.join(validStrings, ", ") + "]"; - } - } - - public static class CaseInsensitiveValidString implements Validator { - - final Set validStrings; - - private CaseInsensitiveValidString(List validStrings) { - this.validStrings = validStrings.stream() - .map(s -> s.toUpperCase(Locale.ROOT)) - .collect(Collectors.toSet()); - } - - public static CaseInsensitiveValidString in(String... validStrings) { - return new CaseInsensitiveValidString(Arrays.asList(validStrings)); - } - - @Override - public void ensureValid(String name, Object o) { - String s = (String) o; - if (s == null || !validStrings.contains(s.toUpperCase(Locale.ROOT))) { - throw new ConfigException(name, o, "String must be one of (case insensitive): " + StringUtils.join(validStrings, ", ")); - } - } - - @Override - public String toString() { - return "(case insensitive) [" + StringUtils.join(validStrings, ", ") + "]"; - } - } - - public static class NonNullValidator implements Validator { - @Override - public void ensureValid(String name, Object value) { - if (value == null) { - // Pass in the string null to avoid the spotbugs warning - throw new ConfigException(name, "null", "entry must be non null"); - } - } - - @Override - public String toString() { - return "non-null string"; - } - } - - public static class LambdaValidator implements Validator { - BiConsumer ensureValid; - Supplier toStringFunction; - - private LambdaValidator(BiConsumer ensureValid, - Supplier toStringFunction) { - this.ensureValid = ensureValid; - this.toStringFunction = toStringFunction; - } - - public static LambdaValidator with(BiConsumer ensureValid, - Supplier toStringFunction) { - return new LambdaValidator(ensureValid, toStringFunction); - } - - @Override - public void ensureValid(String name, Object value) { - ensureValid.accept(name, value); - } - - @Override - public String toString() { - return toStringFunction.get(); - } - } - - public static class CompositeValidator implements Validator { - private final List validators; - - private CompositeValidator(List validators) { - this.validators = Collections.unmodifiableList(validators); - } - - public static CompositeValidator of(Validator... validators) { - return new CompositeValidator(Arrays.asList(validators)); - } - - @Override - public void ensureValid(String name, Object value) { - for (Validator validator : validators) { - validator.ensureValid(name, value); - } - } - - @Override - public String toString() { - if (validators == null) { - return ""; - } - StringBuilder desc = new StringBuilder(); - for (Validator v : validators) { - if (desc.length() > 0) { - desc.append(',').append(' '); - } - desc.append(String.valueOf(v)); - } - return desc.toString(); - } - } - - public static class NonEmptyString implements Validator { - - @Override - public void ensureValid(String name, Object o) { - String s = (String) o; - if (s != null && s.isEmpty()) { - throw new ConfigException(name, o, "String must be non-empty"); - } - } - - @Override - public String toString() { - return "non-empty string"; - } - } - - public static class NonEmptyStringWithoutControlChars implements Validator { - - public static NonEmptyStringWithoutControlChars nonEmptyStringWithoutControlChars() { - return new NonEmptyStringWithoutControlChars(); - } - - @Override - public void ensureValid(String name, Object value) { - String s = (String) value; - - if (s == null) { - // This can happen during creation of the config object due to no default value being defined for the - // name configuration - a missing name parameter is caught when checking for mandatory parameters, - // thus we can ok a null value here - return; - } else if (s.isEmpty()) { - throw new ConfigException(name, value, "String may not be empty"); - } - - // Check name string for illegal characters - ArrayList foundIllegalCharacters = new ArrayList<>(); - - for (int i = 0; i < s.length(); i++) { - if (Character.isISOControl(s.codePointAt(i))) { - foundIllegalCharacters.add(s.codePointAt(i)); - } - } - if (!foundIllegalCharacters.isEmpty()) { - throw new ConfigException(name, value, "String may not contain control sequences but had the following ASCII chars: " + StringUtils.join(foundIllegalCharacters, ",")); - } - } - - @Override - public String toString() { - return "non-empty string without ISO control characters"; - } - } - - public static class ConfigKey { - public final String name; - public final Type type; - public final String documentation; - public final Object defaultValue; - public final Validator validator; - - public ConfigKey(String name, Type type, Object defaultValue, Validator validator, String documentation) { - this.name = name; - this.type = type; - this.defaultValue = NO_DEFAULT_VALUE.equals(defaultValue) ? NO_DEFAULT_VALUE : parseType(name, defaultValue, type); - this.validator = validator; - if (this.validator != null && hasDefault()) { - this.validator.ensureValid(name, this.defaultValue); - } - this.documentation = documentation; - } - - public boolean hasDefault() { - return !NO_DEFAULT_VALUE.equals(this.defaultValue); - } - - public Type type() { - return type; - } - } - -} diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DatabaseDialect.java b/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DatabaseDialect.java deleted file mode 100644 index c2152cbbb..000000000 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DatabaseDialect.java +++ /dev/null @@ -1,380 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.connect.jdbc.dialect; - -import io.openmessaging.connector.api.data.ConnectRecord; -import io.openmessaging.connector.api.data.Schema; -import io.openmessaging.connector.api.data.SchemaBuilder; -import org.apache.rocketmq.connect.jdbc.dialect.provider.ConnectionProvider; -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; -import org.apache.rocketmq.connect.jdbc.schema.table.TableDefinition; -import org.apache.rocketmq.connect.jdbc.schema.table.TableId; -import org.apache.rocketmq.connect.jdbc.sink.JdbcSinkConfig; -import org.apache.rocketmq.connect.jdbc.sink.metadata.FieldsMetadata; -import org.apache.rocketmq.connect.jdbc.sink.metadata.SchemaPair; -import org.apache.rocketmq.connect.jdbc.sink.metadata.SinkRecordField; -import org.apache.rocketmq.connect.jdbc.source.TimestampIncrementingCriteria; -import org.apache.rocketmq.connect.jdbc.source.metadata.ColumnMapping; -import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; -import org.apache.rocketmq.connect.jdbc.util.IdentifierRules; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.util.Calendar; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * database dialect - */ -public interface DatabaseDialect extends ConnectionProvider { - - /** - * dialect name - * - * @return - */ - String name(); - - /** - * get dialect class - * - * @return - */ - default Class getDialectClass() { - return this.getClass(); - } - - /** - * create jdbc prepared statement - * - * @param connection - * @param query - * @return - * @throws SQLException - */ - PreparedStatement createPreparedStatement(Connection connection, String query) throws SQLException; - - /** - * parse to Table Id - * - * @param fqn - * @return - */ - TableId parseToTableId(String fqn); - - /** - * Get the identifier rules for this database. - * - * @return the identifier rules - */ - IdentifierRules identifierRules(); - - /** - * Get a new expression builder that can be used to build expressions with quoted identifiers. - * - * @return - */ - ExpressionBuilder expressionBuilder(); - - Timestamp currentTimeOnDB(Connection connection, Calendar cal) throws SQLException; - - /** - * Get a list of identifiers of the non-system tables in the database. - * - * @param connection - * @return - * @throws SQLException - */ - List tableIds(Connection connection) throws SQLException; - - /** - * table exists - * - * @param connection - * @param tableId - * @return - * @throws SQLException - */ - boolean tableExists(Connection connection, TableId tableId) throws SQLException; - - - /** - * Create the definition for the columns described by the database metadata. - */ - Map describeColumns(Connection connection, String tablePattern, String columnPattern) throws SQLException; - - /** - * Create the definition for the columns described by the database metadata. - */ - Map describeColumns(Connection connection, String catalogPattern, String schemaPattern, String tablePattern, String columnPattern) throws SQLException; - - /** - * Create the definition for the columns in the result set. - */ - Map describeColumns(Connection conn, TableId tableId, ResultSetMetaData rsMetadata) throws SQLException; - - /** - * describe table info - * - * @param connection - * @param tableId - * @return - * @throws SQLException - */ - TableDefinition describeTable(Connection connection, TableId tableId) throws SQLException; - - /** - * describe columns by query sql - * - * @param connection - * @param tableId - * @return - * @throws SQLException - */ - Map describeColumnsByQuerying(Connection connection, TableId tableId) throws SQLException; - - /** - * add field to schema - * - * @param column - * @return - */ - String addFieldToSchema(ColumnDefinition column, SchemaBuilder schemaBuilder); - - /** - * Apply the supplied DDL statements using the given connection. This gives the dialect the - * opportunity to execute the statements with a different autocommit setting. - */ - void applyDdlStatements(Connection connection, List statements) throws SQLException; - - /** - * build dml statement - * - * @param table - * @param keyColumns - * @param nonKeyColumns - * @return - */ - String buildInsertStatement(TableId table, Collection keyColumns, Collection nonKeyColumns); - - /** - * update statement - * - * @param table - * @param keyColumns - * @param nonKeyColumns - * @return - */ - String buildUpdateStatement(TableId table, Collection keyColumns, Collection nonKeyColumns); - - /** - * upsert statment - * - * @param table - * @param keyColumns - * @param nonKeyColumns - * @return - */ - String buildUpsertQueryStatement(TableId table, Collection keyColumns, Collection nonKeyColumns); - - /** - * delete statement - * - * @param table - * @param keyColumns - * @return - */ - default String buildDeleteStatement(TableId table, Collection keyColumns) { - throw new UnsupportedOperationException(); - } - - /** - * Get insert sql - * - * @return - */ - String getInsertSql(JdbcSinkConfig config, FieldsMetadata fieldsMetadata, TableId tableId); - - /** - * Get insert sql - * - * @return - */ - String getDeleteSql(JdbcSinkConfig config, FieldsMetadata fieldsMetadata, TableId tableId); - - - /** - * build select table - */ - String buildSelectTableMode(); - - void buildSelectTable(ExpressionBuilder builder, TableId tableId); - - - /** - * drop table - * - * @param table - * @param options - * @return - */ - String buildDropTableStatement(TableId table, DropOptions options); - - /** - * create table - * - * @param table - * @param fields - * @return - */ - String buildCreateTableStatement(TableId table, Collection fields); - - /** - * build alter table - * - * @param table - * @param fields - * @return - */ - List buildAlterTable(TableId table, Collection fields); - - /** - * Create a component that can bind record values into the supplied prepared statement. - * - * @param statement the prepared statement - * @param pkMode the primary key mode; may not be null - * @param schemaPair the key and value schemas; may not be null - * @param fieldsMetadata the field metadata; may not be null - * @param tableDefinition the table definition; may be null - * @param insertMode the insert mode; may not be null - * @return the statement binder; may not be null - */ - StatementBinder statementBinder( - PreparedStatement statement, - JdbcSinkConfig.PrimaryKeyMode pkMode, - SchemaPair schemaPair, - FieldsMetadata fieldsMetadata, - TableDefinition tableDefinition, - JdbcSinkConfig.InsertMode insertMode - ); - - /** - * value column types - * - * @param rsMetadata - * @param columns - * @throws io.openmessaging.connector.api.errors.ConnectException - */ - void validateColumnTypes( - ResultSetMetaData rsMetadata, - List columns - ) throws io.openmessaging.connector.api.errors.ConnectException; - - - /** - * bind field - * - * @param statement - * @param index - * @param schema - * @param value - * @param colDef - * @throws SQLException - */ - void bindField(PreparedStatement statement, int index, Schema schema, Object value, ColumnDefinition colDef) throws SQLException; - - /** - * criteria for - * - * @param incrementingColumn - * @param timestampColumns - * @return - */ - TimestampIncrementingCriteria criteriaFor( - ColumnId incrementingColumn, - List timestampColumns - ); - - /** - * get min timestamp value - * - * @param con - * @param tableOrQuery - * @param timestampColumns - * @return - * @throws SQLException - */ - Long getMinTimestampValue(Connection con, String tableOrQuery, List timestampColumns) throws SQLException; - - /** - * A function to bind the values from a sink record into a prepared statement. - */ - @FunctionalInterface - interface StatementBinder { - /** - * bind record - * - * @param record - * @throws SQLException - */ - void bindRecord(ConnectRecord record) throws SQLException; - - - default Optional executeUpdates(PreparedStatement preparedStatement) throws SQLException { - // no-op - return Optional.empty(); - } - - default long executeDeletes(PreparedStatement preparedStatement) throws SQLException { - // no-op - return 0; - } - } - - - /** - * Create a function that converts column values for the column defined by the specified mapping. - * - * @param mapping - * @return - */ - ColumnConverter createColumnConverter(ColumnMapping mapping); - - /** - * A function that obtains a column value from the current row of the specified result set. - */ - @FunctionalInterface - interface ColumnConverter { - /** - * convert - * @param resultSet - * @return - * @throws SQLException - * @throws IOException - */ - Object convert(ResultSet resultSet) throws SQLException, IOException; - } -} \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DropOptions.java b/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DropOptions.java deleted file mode 100644 index 11d66747f..000000000 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DropOptions.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.connect.jdbc.dialect; - -import java.util.Objects; - -public class DropOptions { - - private final boolean ifExists; - private final boolean cascade; - - /** - * Create a new instance with the default settings. - */ - public DropOptions() { - this(false, false); - } - - protected DropOptions( - boolean ifExists, - boolean cascade - ) { - this.ifExists = ifExists; - this.cascade = cascade; - } - - /** - * Get whether the 'IF EXISTS' clause should be used with the 'DROP' statement. - * - * @return true if the object should be dropped only if it already exists, or false otherwise - */ - public boolean ifExists() { - return ifExists; - } - - /** - * Get whether the 'DROP' statement should cascade to dependent objects. - * - * @return true if dependent objects should also be dropped, or false otherwise - */ - public boolean cascade() { - return cascade; - } - - /** - * Set whether the 'IF EXISTS' clause should be used with the 'DROP' statement. - * - * @param ifExists true if the object should be dropped only if it already exists - * @return a new options object with the current state plus the new if-exists state; never null - */ - public DropOptions setIfExists(boolean ifExists) { - return new DropOptions(ifExists, cascade); - } - - /** - * Set whether the 'DROP' statement should cascade to dependent objects. - * - * @param cascade true if dependent objects should also be dropped, or false otherwise - * @return a new options object with the current state plus the new cascade state; never null - */ - public DropOptions setCascade(boolean cascade) { - return new DropOptions(ifExists, cascade); - } - - @Override - public String toString() { - return "DropOptions{ifExists=" + ifExists + ", cascade=" + cascade + "}"; - } - - @Override - public int hashCode() { - return Objects.hash(ifExists, cascade); - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - if (obj instanceof DropOptions) { - DropOptions that = (DropOptions) obj; - return this.ifExists() == that.ifExists() && this.cascade() == that.cascade(); - } - return false; - } -} diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/provider/ConnectionProvider.java b/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/provider/ConnectionProvider.java deleted file mode 100644 index 6e6c7a7e7..000000000 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/provider/ConnectionProvider.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.connect.jdbc.dialect.provider; - -import java.sql.Connection; -import java.sql.SQLException; - -/** - * A provider of JDBC {@link Connection} instances. - */ -public interface ConnectionProvider extends AutoCloseable { - - /** - * Create a connection. - * - * @return the connection; never null - * @throws SQLException if there is a problem getting the connection - */ - Connection getConnection() throws SQLException; - - /** - * Determine if the specified connection is valid. - * - * @param connection the database connection; may not be null - * @param timeout The time in seconds to wait for the database operation used to validate - * the connection to complete. If the timeout period expires before the - * operation completes, this method returns false. A value of 0 indicates a - * timeout is not applied to the database operation. - * @return true if it is valid, or false otherwise - * @throws SQLException if there is an error with the database connection - */ - boolean isConnectionValid( - Connection connection, - int timeout - ) throws SQLException; - - /** - * Close this connection provider. - */ - @Override - void close(); - - /** - * Get the publicly viewable identifier for this connection provider and / or the database. - * The resulting value should not contain any secrets or passwords. - * - * @return the identifier; never null - */ - default String identifier() { - return toString(); - } -} diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/metadata/SchemaMapping.java b/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/metadata/SchemaMapping.java deleted file mode 100644 index 9affae6e2..000000000 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/metadata/SchemaMapping.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.connect.jdbc.source.metadata; - -import io.openmessaging.connector.api.data.Field; -import io.openmessaging.connector.api.data.Schema; -import io.openmessaging.connector.api.data.SchemaBuilder; -import io.openmessaging.connector.api.data.Struct; -import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; -import org.apache.rocketmq.connect.jdbc.schema.table.TableId; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * schema mapping - */ -public final class SchemaMapping { - private static final Logger log = LoggerFactory.getLogger(SchemaMapping.class); - - public static SchemaMapping create( - Connection conn, - TableId tableId, - ResultSetMetaData metadata, - DatabaseDialect dialect - ) throws SQLException { - // backwards compatible - String schemaName = tableId != null ? tableId.tableName() : null; - // describe columns - Map colDefins = dialect.describeColumns(conn, tableId, metadata); - Map colConvertersByFieldName = new LinkedHashMap<>(); - SchemaBuilder builder = SchemaBuilder.struct().name(schemaName); - - int columnNumber = 0; - for (ColumnDefinition colDefn : colDefins.values()) { - ++columnNumber; - String fieldName = dialect.addFieldToSchema(colDefn, builder); - if (fieldName == null) { - continue; - } - Field field = builder.field(fieldName); - ColumnMapping mapping = new ColumnMapping(colDefn, columnNumber, field); - DatabaseDialect.ColumnConverter converter = dialect.createColumnConverter(mapping); - colConvertersByFieldName.put(fieldName, converter); - } - return new SchemaMapping(builder.build(), colConvertersByFieldName); - } - - private final Schema schema; - private final List fieldSetters; - - private SchemaMapping( - Schema schema, - Map convertersByFieldName - ) { - assert schema != null; - assert convertersByFieldName != null; - assert !convertersByFieldName.isEmpty(); - this.schema = schema; - List fieldSetters = new ArrayList<>(convertersByFieldName.size()); - for (Map.Entry entry : convertersByFieldName.entrySet()) { - DatabaseDialect.ColumnConverter converter = entry.getValue(); - Field field = schema.getField(entry.getKey()); - assert field != null; - fieldSetters.add(new FieldSetter(converter, field)); - } - this.fieldSetters = Collections.unmodifiableList(fieldSetters); - } - - /** - * schema - * - * @return - */ - public Schema schema() { - return schema; - } - - /** - * field setters - * - * @return - */ - public List fieldSetters() { - return fieldSetters; - } - - @Override - public String toString() { - return "Mapping for " + schema.getName(); - } - - public static final class FieldSetter { - - private final DatabaseDialect.ColumnConverter converter; - private final Field field; - - private FieldSetter( - DatabaseDialect.ColumnConverter converter, - Field field - ) { - this.converter = converter; - this.field = field; - } - - /** - * Get the {@link Field} that this setter function sets. - * - * @return the field; never null - */ - public Field field() { - return field; - } - - /** - * set field - * - * @param payload - * @param resultSet - * @throws SQLException - * @throws IOException - */ - public void setField( - Struct payload, - ResultSet resultSet - ) throws SQLException, IOException { - Object value = this.converter.convert(resultSet); - payload.put(field, value); - } - - @Override - public String toString() { - return field.getName(); - } - } -} diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/ExpressionBuilder.java b/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/ExpressionBuilder.java deleted file mode 100644 index cc90f1009..000000000 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/ExpressionBuilder.java +++ /dev/null @@ -1,597 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.connect.jdbc.util; - -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; - -public class ExpressionBuilder { - - /** - * A functional interface for anything that can be appended to an expression builder. - * This makes use of double-dispatch to allow implementations to customize the behavior, - * yet have callers not care about the differences in behavior. - */ - @FunctionalInterface - public interface Expressable { - - /** - * Append this object to the specified builder. - * - * @param builder the builder to use; may not be null - * @param useQuotes whether quotes should be used for this object - */ - void appendTo( - ExpressionBuilder builder, - boolean useQuotes - ); - - /** - * Append this object to the specified builder. - * - * @param builder the builder to use; may not be null - * @param useQuotes whether quotes should be used for this object - */ - default void appendTo( - ExpressionBuilder builder, - QuoteMethod useQuotes - ) { - switch (useQuotes) { - case ALWAYS: - appendTo(builder, true); - break; - case NEVER: - default: - // do nothing - break; - } - } - } - - /** - * A functional interface for a transformation that an expression builder might use when - * appending one or more other objects. - * - * @param the type of object to transform before appending. - */ - @FunctionalInterface - public interface Transform { - void apply( - ExpressionBuilder builder, - T input - ); - } - - /** - * A fluent API interface returned by the {@link ExpressionBuilder#appendList()} method that - * allows a caller to easily define a custom delimiter to be used between items in the list, - * an optional transformation that should be applied to each item in the list, and the - * items in the list. This is very handle when the number of items is not known a priori. - * - * @param the type of object to be appended to the expression builder - */ - public interface ListBuilder { - - /** - * Define the delimiter to appear between items in the list. If not specified, a comma - * is used as the default delimiter. - * - * @param delimiter the delimiter; may not be null - * @return this builder to enable methods to be chained; never null - */ - ListBuilder delimitedBy(String delimiter); - - /** - * Define a {@link Transform} that should be applied to every item in the list as it is - * appended. - * - * @param transform the transform; may not be null - * @param the type of item to be transformed - * @return this builder to enable methods to be chained; never null - */ - ListBuilder transformedBy(Transform transform); - - /** - * Append to this list all of the items in the specified {@link Iterable}. - * - * @param objects the objects to be appended to the list - * @return this builder to enable methods to be chained; never null - */ - ExpressionBuilder of(Iterable objects); - - /** - * Append to this list all of the items in the specified {@link Iterable} objects. - * - * @param objects1 the first collection of objects to be added to the list - * @param objects2 a second collection of objects to be added to the list - * @return this builder to enable methods to be chained; never null - */ - default ExpressionBuilder of(Iterable objects1, Iterable objects2) { - of(objects1); - return of(objects2); - } - - /** - * Append to this list all of the items in the specified {@link Iterable} objects. - * - * @param objects1 the first collection of objects to be added to the list - * @param objects2 a second collection of objects to be added to the list - * @param objects3 a third collection of objects to be added to the list - * @return this builder to enable methods to be chained; never null - */ - default ExpressionBuilder of( - Iterable objects1, - Iterable objects2, - Iterable objects3 - ) { - of(objects1); - of(objects2); - return of(objects3); - } - } - - /** - * Get a {@link Transform} that will surround the inputs with quotes. - * - * @return the transform; never null - */ - public static Transform quote() { - return (builder, input) -> builder.appendColumnName(input); - } - - /** - * Get a {@link Transform} that will quote just the column names. - * - * @return the transform; never null - */ - public static Transform columnNames() { - return (builder, input) -> builder.appendColumnName(input.name()); - } - - /** - * Get a {@link Transform} that will quote just the column names and append the given string. - * - * @param appended the string to append after the quoted column names - * @return the transform; never null - */ - public static Transform columnNamesWith(final String appended) { - return (builder, input) -> { - builder.appendColumnName(input.name()); - builder.append(appended); - }; - } - - /** - * Get a {@link Transform} that will append a placeholder rather than each of the column names. - * - * @param str the string to output instead the each column name - * @return the transform; never null - */ - public static Transform placeholderInsteadOfColumnNames(final String str) { - return (builder, input) -> builder.append(str); - } - - /** - * Get a {@link Transform} that will append the prefix and then the quoted column name. - * - * @param prefix the string to output before the quoted column names - * @return the transform; never null - */ - public static Transform columnNamesWithPrefix(final String prefix) { - return (builder, input) -> { - builder.append(prefix); - builder.appendColumnName(input.name()); - }; - } - - /** - * Create a new ExpressionBuilder using the default {@link IdentifierRules}. - * - * @return the expression builder - */ - public static ExpressionBuilder create() { - return new ExpressionBuilder(); - } - - protected static final QuoteMethod DEFAULT_QUOTE_METHOD = QuoteMethod.ALWAYS; - - private final IdentifierRules rules; - private final StringBuilder sb = new StringBuilder(); - private QuoteMethod quoteSqlIdentifiers = DEFAULT_QUOTE_METHOD; - - /** - * Create a new expression builder with the default {@link IdentifierRules}. - */ - public ExpressionBuilder() { - this(null); - } - - /** - * Create a new expression builder that uses the specified {@link IdentifierRules}. - * - * @param rules the rules; may be null if the default rules are to be used - */ - public ExpressionBuilder(IdentifierRules rules) { - this.rules = rules != null ? rules : IdentifierRules.DEFAULT; - } - - /** - * Set when this expression builder should quote identifiers, such as table and column names. - * - * @param method the quoting method; may be null if the default method - * ({@link QuoteMethod#ALWAYS always}) should be used - * @return this expression builder; never null - */ - public ExpressionBuilder setQuoteIdentifiers(QuoteMethod method) { - this.quoteSqlIdentifiers = method != null ? method : DEFAULT_QUOTE_METHOD; - return this; - } - - /** - * Return a new ExpressionBuilder that escapes quotes with the specified prefix. - * This builder remains unaffected. - * - * @param prefix the prefix - * @return the new ExpressionBuilder, or this builder if the prefix is null or empty - */ - public ExpressionBuilder escapeQuotesWith(String prefix) { - if (prefix == null || prefix.isEmpty()) { - return this; - } - return new ExpressionBuilder(this.rules.escapeQuotesWith(prefix)); - } - - /** - * Append to this builder's expression the delimiter defined by this builder's - * {@link IdentifierRules}. - * - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder appendIdentifierDelimiter() { - sb.append(rules.identifierDelimiter()); - return this; - } - - /** - * Always append to this builder's expression the leading quote character(s) defined by this - * builder's {@link IdentifierRules}. - * - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder appendLeadingQuote() { - return appendLeadingQuote(QuoteMethod.ALWAYS); - } - - - protected ExpressionBuilder appendLeadingQuote(QuoteMethod method) { - switch (method) { - case ALWAYS: - sb.append(rules.leadingQuoteString()); - break; - case NEVER: - default: - break; - } - return this; - } - - /** - * Always append to this builder's expression the trailing quote character(s) defined by this - * builder's {@link IdentifierRules}. - * - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder appendTrailingQuote() { - return appendTrailingQuote(QuoteMethod.ALWAYS); - } - - protected ExpressionBuilder appendTrailingQuote(QuoteMethod method) { - switch (method) { - case ALWAYS: - sb.append(rules.trailingQuoteString()); - break; - case NEVER: - default: - break; - } - return this; - } - - /** - * Append to this builder's expression the string quote character ({@code '}). - * - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder appendStringQuote() { - sb.append("'"); - return this; - } - - /** - * Append to this builder's expression a string surrounded by single quote characters ({@code '}). - * Use {@link #appendIdentifier(String, QuoteMethod)} for identifiers, - * {@link #appendColumnName(String, QuoteMethod)} for column names, or - * {@link #appendTableName(String, QuoteMethod)} for table names. - * - * @param name the object whose string representation is to be appended - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder appendStringQuoted(Object name) { - appendStringQuote(); - sb.append(name); - appendStringQuote(); - return this; - } - - /** - * Append to this builder's expression the identifier. - * - * @param name the name to be appended - * @param quoted true if the name should be quoted, or false otherwise - * @return this builder to enable methods to be chained; never null - * @deprecated use {@link #appendIdentifier(String, QuoteMethod)} instead - */ - @Deprecated - public ExpressionBuilder appendIdentifier( - String name, - boolean quoted - ) { - return appendIdentifier(name, quoted ? QuoteMethod.ALWAYS : QuoteMethod.NEVER); - } - - /** - * Append to this builder's expression the identifier. - * - * @param name the name to be appended - * @param quoted true if the name should be quoted, or false otherwise - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder appendIdentifier( - String name, - QuoteMethod quoted - ) { - appendLeadingQuote(quoted); - sb.append(name); - appendTrailingQuote(quoted); - return this; - } - - /** - * Append to this builder's expression the specified Column identifier, possibly surrounded by - * the leading and trailing quotes based upon {@link #setQuoteIdentifiers(QuoteMethod)}. - * - * @param name the name to be appended - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder appendTableName(String name) { - return appendTableName(name, quoteSqlIdentifiers); - } - - /** - * Append to this builder's expression the specified Column identifier, possibly surrounded by - * the leading and trailing quotes based upon {@link #setQuoteIdentifiers(QuoteMethod)}. - * - * @param name the name to be appended - * @param quote the quote method to be used - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder appendTableName(String name, QuoteMethod quote) { - appendLeadingQuote(quote); - sb.append(name); - appendTrailingQuote(quote); - return this; - } - - /** - * Append to this builder's expression the specified Column identifier, possibly surrounded by - * the leading and trailing quotes based upon {@link #setQuoteIdentifiers(QuoteMethod)}. - * - * @param name the name to be appended - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder appendColumnName(String name) { - return appendColumnName(name, quoteSqlIdentifiers); - } - - /** - * Append to this builder's expression the specified Column identifier, possibly surrounded by - * the leading and trailing quotes based upon {@link #setQuoteIdentifiers(QuoteMethod)}. - * - * @param name the name to be appended - * @param quote whether to quote the column name; may not be null - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder appendColumnName(String name, QuoteMethod quote) { - appendLeadingQuote(quote); - sb.append(name); - appendTrailingQuote(quote); - return this; - } - - /** - * Append to this builder's expression the specified identifier, surrounded by the leading and - * trailing quotes. - * - * @param name the name to be appended - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder appendIdentifierQuoted(String name) { - appendLeadingQuote(); - sb.append(name); - appendTrailingQuote(); - return this; - } - - /** - * Append to this builder's expression the binary value as a hex string, prefixed and - * suffixed by a single quote character. - * - * @param value the value to be appended - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder appendBinaryLiteral(byte[] value) { - return append("x'").append(BytesUtil.toHex(value)).append("'"); - } - - /** - * Append to this builder's expression a new line. - * - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder appendNewLine() { - sb.append(System.lineSeparator()); - return this; - } - - /** - * Append to this builder's expression the specified object. If the object is {@link Expressable}, - * then this builder delegates to the object's - * {@link Expressable#appendTo(ExpressionBuilder, boolean)} method. Otherwise, the string - * representation of the object is appended to the expression. - * - * @param obj the object to be appended - * @param useQuotes true if the object should be surrounded by quotes, or false otherwise - * @return this builder to enable methods to be chained; never null - * @deprecated use {@link #append(Object, QuoteMethod)} instead - */ - @Deprecated - public ExpressionBuilder append( - Object obj, - boolean useQuotes - ) { - return append(obj, useQuotes ? QuoteMethod.ALWAYS : QuoteMethod.NEVER); - } - - /** - * Append to this builder's expression the specified object. If the object is {@link Expressable}, - * then this builder delegates to the object's - * {@link Expressable#appendTo(ExpressionBuilder, boolean)} method. Otherwise, the string - * representation of the object is appended to the expression. - * - * @param obj the object to be appended - * @param useQuotes true if the object should be surrounded by quotes, or false otherwise - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder append( - Object obj, - QuoteMethod useQuotes - ) { - if (obj instanceof Expressable) { - ((Expressable) obj).appendTo(this, useQuotes); - } else if (obj != null) { - sb.append(obj); - } - return this; - } - - /** - * Append to this builder's expression the specified object surrounded by quotes. If the object - * is {@link Expressable}, then this builder delegates to the object's - * {@link Expressable#appendTo(ExpressionBuilder, boolean)} method. Otherwise, the string - * representation of the object is appended to the expression. - * - * @param obj the object to be appended - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder append(Object obj) { - return append(obj, quoteSqlIdentifiers); - } - - /** - * Append to this builder's expression the specified object surrounded by quotes. If the object - * is {@link Expressable}, then this builder delegates to the object's - * {@link Expressable#appendTo(ExpressionBuilder, boolean)} method. Otherwise, the string - * representation of the object is appended to the expression. - * - * @param obj the object to be appended - * @param transform the transform that should be used on the supplied object to obtain the - * representation that is appended to the expression; may be null - * @param the type of object to transform before appending. - * @return this builder to enable methods to be chained; never null - */ - public ExpressionBuilder append( - T obj, - Transform transform - ) { - if (transform != null) { - transform.apply(this, obj); - } else { - append(obj); - } - return this; - } - - protected class BasicListBuilder implements ListBuilder { - private final String delimiter; - private final Transform transform; - private boolean first = true; - - BasicListBuilder() { - this(", ", null); - } - - BasicListBuilder(String delimiter, Transform transform) { - this.delimiter = delimiter; - this.transform = transform != null ? transform : ExpressionBuilder::append; - } - - @Override - public ListBuilder delimitedBy(String delimiter) { - return new BasicListBuilder(delimiter, transform); - } - - @Override - public ListBuilder transformedBy(Transform transform) { - return new BasicListBuilder<>(delimiter, transform); - } - - @Override - public ExpressionBuilder of(Iterable objects) { - for (T obj : objects) { - if (first) { - first = false; - } else { - append(delimiter); - } - append(obj, transform); - } - return ExpressionBuilder.this; - } - } - - public ListBuilder appendList() { - return new BasicListBuilder<>(); - } - - public ExpressionBuilder appendMultiple( - String delimiter, - String expression, - int times - ) { - for (int i = 0; i < times; i++) { - if (i > 0) { - append(delimiter); - } - append(expression); - } - return this; - } - - @Override - public String toString() { - return sb.toString(); - } -} diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/resources/META-INF/services/org.apache.rocketmq.connect.jdbc.dialect.provider.DatabaseDialectProvider b/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/resources/META-INF/services/org.apache.rocketmq.connect.jdbc.dialect.provider.DatabaseDialectProvider deleted file mode 100644 index 4b79c5373..000000000 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/resources/META-INF/services/org.apache.rocketmq.connect.jdbc.dialect.provider.DatabaseDialectProvider +++ /dev/null @@ -1,3 +0,0 @@ -org.apache.rocketmq.connect.jdbc.dialect.GenericDatabaseDialect$Provider -org.apache.rocketmq.connect.jdbc.dialect.impl.MySqlDatabaseDialect$Provider -org.apache.rocketmq.connect.jdbc.dialect.impl.OpenMLDBDatabaseDialect$Provider \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/pom.xml b/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/pom.xml deleted file mode 100644 index 72f48347a..000000000 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/pom.xml +++ /dev/null @@ -1,169 +0,0 @@ - - - - rocketmq-connect-jdbc - org.apache.rocketmq - 0.0.1-SNAPSHOT - - 4.0.0 - - rocketmq-connect-jdbc-mysql - - - 8 - 8 - UTF-8 - - - - - - mysql - mysql-connector-java - 8.0.30 - - - org.apache.rocketmq - rocketmq-connect-jdbc-core - compile - - - - - - - - org.codehaus.mojo - versions-maven-plugin - 2.3 - - - org.codehaus.mojo - clirr-maven-plugin - 2.7 - - - maven-dependency-plugin - - ${project.build.directory}/lib - false - true - - - - maven-compiler-plugin - 3.6.1 - - ${maven.compiler.source} - ${maven.compiler.target} - ${maven.compiler.source} - true - true - - - - maven-surefire-plugin - 2.19.1 - - -Xms512m -Xmx1024m - always - - **/*Test.java - - - - - maven-site-plugin - 3.6 - - en_US - UTF-8 - UTF-8 - - - - maven-source-plugin - 3.0.1 - - - attach-sources - - jar - - - - - - maven-javadoc-plugin - 2.10.4 - - UTF-8 - en_US - io.openmessaging.internal - - - - aggregate - - aggregate - - site - - - - - maven-resources-plugin - 3.0.2 - - ${project.build.sourceEncoding} - - - - org.codehaus.mojo - findbugs-maven-plugin - 3.0.4 - - - maven-assembly-plugin - 3.0.0 - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - maven-checkstyle-plugin - 2.17 - - - verify - verify - - ../../../style/rmq_checkstyle.xml - UTF-8 - true - true - false - false - - - check - - - - - - - \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-openmldb/pom.xml b/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-openmldb/pom.xml deleted file mode 100644 index 01af1627b..000000000 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-openmldb/pom.xml +++ /dev/null @@ -1,185 +0,0 @@ - - - - rocketmq-connect-jdbc - org.apache.rocketmq - 0.0.1-SNAPSHOT - - 4.0.0 - - rocketmq-connect-jdbc-openmldb - - - 8 - 8 - UTF-8 - - - - - org.apache.rocketmq - rocketmq-connect-jdbc-core - compile - - - - com.4paradigm.openmldb - openmldb-native - 0.5.0-macos - - - com.4paradigm.openmldb - openmldb-jdbc - 0.5.0 - - - com.4paradigm.openmldb - openmldb-native - - - org.projectlombok - lombok - - - - - - - - - - org.codehaus.mojo - versions-maven-plugin - 2.3 - - - org.codehaus.mojo - clirr-maven-plugin - 2.7 - - - maven-dependency-plugin - - ${project.build.directory}/lib - false - true - - - - maven-compiler-plugin - 3.6.1 - - ${maven.compiler.source} - ${maven.compiler.target} - ${maven.compiler.source} - true - true - - - - maven-surefire-plugin - 2.19.1 - - -Xms512m -Xmx1024m - always - - **/*Test.java - - - - - maven-site-plugin - 3.6 - - en_US - UTF-8 - UTF-8 - - - - maven-source-plugin - 3.0.1 - - - attach-sources - - jar - - - - - - maven-javadoc-plugin - 2.10.4 - - UTF-8 - en_US - io.openmessaging.internal - - - - aggregate - - aggregate - - site - - - - - maven-resources-plugin - 3.0.2 - - ${project.build.sourceEncoding} - - - - org.codehaus.mojo - findbugs-maven-plugin - 3.0.4 - - - maven-assembly-plugin - 3.0.0 - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - maven-checkstyle-plugin - 2.17 - - - verify - verify - - ../../../style/rmq_checkstyle.xml - UTF-8 - true - true - false - false - - - check - - - - - - - - \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-openmldb/src/main/resources/openmldb-jdbc-sink.conf b/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-openmldb/src/main/resources/openmldb-jdbc-sink.conf deleted file mode 100644 index e69de29bb..000000000 diff --git a/connectors/rocketmq-connect-jdbc/src/assembly/assembly.xml b/connectors/rocketmq-connect-jdbc/src/assembly/assembly.xml new file mode 100644 index 000000000..430fbb866 --- /dev/null +++ b/connectors/rocketmq-connect-jdbc/src/assembly/assembly.xml @@ -0,0 +1,52 @@ + + + + package + false + + dir + tar.gz + + + + + false + share/java/rocketmq-connect-jdbc/ + true + true + system + + + + false + share/java/rocketmq-connect-jdbc/ + true + true + runtime + + + + false + share/java/rocketmq-connect-jdbc/ + true + true + provided + + + + diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/PreparedStatementBinder.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/binder/AbstractJdbcRecordBinder.java similarity index 55% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/PreparedStatementBinder.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/binder/AbstractJdbcRecordBinder.java index 6bf2cd082..f5410e1d3 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/PreparedStatementBinder.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/binder/AbstractJdbcRecordBinder.java @@ -14,14 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.connect.jdbc.dialect; +package org.apache.rocketmq.connect.jdbc.binder; import io.openmessaging.connector.api.data.ConnectRecord; import io.openmessaging.connector.api.data.Field; import io.openmessaging.connector.api.data.Schema; import io.openmessaging.connector.api.data.Struct; import io.openmessaging.connector.api.errors.ConnectException; -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Objects; import org.apache.rocketmq.connect.jdbc.schema.table.TableDefinition; import org.apache.rocketmq.connect.jdbc.sink.JdbcSinkConfig; import org.apache.rocketmq.connect.jdbc.sink.metadata.FieldsMetadata; @@ -29,45 +31,32 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Objects; -import java.util.Optional; - -import static java.util.Objects.nonNull; - /** * prepared statement binder */ -public class PreparedStatementBinder implements DatabaseDialect.StatementBinder { - - private final static Logger log = LoggerFactory.getLogger(PreparedStatementBinder.class); - +public abstract class AbstractJdbcRecordBinder implements JdbcRecordBinder { + private final static Logger log = LoggerFactory.getLogger(AbstractJdbcRecordBinder.class); private final JdbcSinkConfig.PrimaryKeyMode pkMode; - private final PreparedStatement statement; + private final JdbcSinkConfig.InsertMode insertMode; + private final PreparedStatement preparedStatement; private final SchemaPair schemaPair; private final FieldsMetadata fieldsMetadata; - private final JdbcSinkConfig.InsertMode insertMode; - private final DatabaseDialect dialect; - private final TableDefinition tabDef; + private final TableDefinition tableDefinition; - public PreparedStatementBinder( - DatabaseDialect dialect, - PreparedStatement statement, - JdbcSinkConfig.PrimaryKeyMode pkMode, - SchemaPair schemaPair, - FieldsMetadata fieldsMetadata, - TableDefinition tabDef, - JdbcSinkConfig.InsertMode insertMode + public AbstractJdbcRecordBinder( + PreparedStatement statement, + TableDefinition tableDefinition, + FieldsMetadata fieldsMetadata, + SchemaPair schemaPair, + JdbcSinkConfig.PrimaryKeyMode pkMode, + JdbcSinkConfig.InsertMode insertMode ) { - this.dialect = dialect; this.pkMode = pkMode; - this.statement = statement; + this.preparedStatement = statement; + this.tableDefinition = tableDefinition; this.schemaPair = schemaPair; this.fieldsMetadata = fieldsMetadata; this.insertMode = insertMode; - this.tabDef = tabDef; } @Override @@ -91,50 +80,9 @@ public void bindRecord(ConnectRecord record) throws SQLException { throw new AssertionError(); } } - statement.addBatch(); + preparedStatement.addBatch(); } - public long executeDeletes(PreparedStatement deletePreparedStatement) throws SQLException { - long totalDeleteCount = 0; - if (nonNull(deletePreparedStatement)) { - try { - for (int updateCount : deletePreparedStatement.executeBatch()) { - if (updateCount != Statement.SUCCESS_NO_INFO) { - totalDeleteCount += updateCount; - } - } - } catch (SQLException e) { - log.error("deletePreparedStatement.executeBatch failed, errCode={}, sqlState={}, error msg={}, cause={}, sql={}", - e.getErrorCode(), e.getSQLState(), e.getMessage(), e.getCause(), deletePreparedStatement); - throw new SQLException(e); - } - } - return totalDeleteCount; - } - - @Override - public Optional executeUpdates(PreparedStatement updatePreparedStatement) throws SQLException { - Optional count = Optional.empty(); - if (nonNull(updatePreparedStatement)) { - try { - for (int updateCount : updatePreparedStatement.executeBatch()) { - if (updateCount != Statement.SUCCESS_NO_INFO) { - count = count.isPresent() - ? count.map(total -> total + updateCount) - : Optional.of((long) updateCount); - } - } - } catch (SQLException e) { - log.error("updatePreparedStatement.executeBatch failed, errCode={}, sqlState={}, error msg={}, " + - "cause={}, sql={}", - e.getErrorCode(), e.getSQLState(), e.getMessage(), e.getCause(), updatePreparedStatement); - throw new SQLException(e); - } - } - return count; - } - - protected int bindKeyFields(ConnectRecord record, int index) throws SQLException { switch (pkMode) { case NONE: @@ -142,27 +90,25 @@ protected int bindKeyFields(ConnectRecord record, int index) throws SQLException throw new AssertionError(); } break; - case RECORD_KEY: { + case RECORD_KEY: if (schemaPair.keySchema.getFieldType().isPrimitive()) { assert fieldsMetadata.keyFieldNames.size() == 1; bindField(index++, schemaPair.keySchema, record.getKey(), - fieldsMetadata.keyFieldNames.iterator().next()); + fieldsMetadata.keyFieldNames.iterator().next()); } else { for (String fieldName : fieldsMetadata.keyFieldNames) { final Field field = schemaPair.keySchema.getField(fieldName); bindField(index++, field.getSchema(), ((Struct) record.getKey()).get(field), fieldName); } } - } - break; - case RECORD_VALUE: { + break; + case RECORD_VALUE: Struct struct = (Struct) record.getData(); for (String fieldName : fieldsMetadata.keyFieldNames) { - final Field field = schemaPair.schema.getField(fieldName); + final Field field = schemaPair.valueSchema.getField(fieldName); bindField(index++, field.getSchema(), struct.get(fieldName), fieldName); } - } - break; + break; default: throw new ConnectException("Unknown primary key mode: " + pkMode); } @@ -171,7 +117,7 @@ protected int bindKeyFields(ConnectRecord record, int index) throws SQLException protected int bindNonKeyFields( ConnectRecord record, - int index + int index ) throws SQLException { Struct struct = (Struct) record.getData(); for (final String fieldName : fieldsMetadata.nonKeyFieldNames) { @@ -181,9 +127,14 @@ protected int bindNonKeyFields( return index; } - protected void bindField(int index, Schema schema, Object value, String fieldName) - throws SQLException { - ColumnDefinition colDef = tabDef == null ? null : tabDef.definitionForColumn(fieldName); - dialect.bindField(statement, index, schema, value, colDef); + protected PreparedStatement getPreparedStatement() { + return preparedStatement; + } + + protected TableDefinition getTableDefinition() { + return tableDefinition; } + + // Bind field + protected abstract void bindField(int index, Schema schema, Object value, String fieldName) throws SQLException; } diff --git a/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/binder/DefaultJdbcRecordBinder.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/binder/DefaultJdbcRecordBinder.java new file mode 100644 index 000000000..058a71b4c --- /dev/null +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/binder/DefaultJdbcRecordBinder.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.connect.jdbc.binder; + +import io.openmessaging.connector.api.data.Schema; +import io.openmessaging.connector.api.data.logical.Date; +import io.openmessaging.connector.api.data.logical.Decimal; +import io.openmessaging.connector.api.data.logical.Time; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.ZoneId; +import java.util.Objects; +import java.util.TimeZone; +import org.apache.rocketmq.connect.jdbc.common.DebeziumTimeTypes; +import org.apache.rocketmq.connect.jdbc.schema.table.TableDefinition; +import org.apache.rocketmq.connect.jdbc.sink.JdbcSinkConfig; +import org.apache.rocketmq.connect.jdbc.sink.metadata.FieldsMetadata; +import org.apache.rocketmq.connect.jdbc.sink.metadata.SchemaPair; +import org.apache.rocketmq.connect.jdbc.util.DateTimeUtils; + +/** + * Jdbc record binder + */ +public class DefaultJdbcRecordBinder extends AbstractJdbcRecordBinder { + + private TimeZone timeZone = TimeZone.getTimeZone(ZoneId.of(JdbcSinkConfig.DB_TIMEZONE_DEFAULT)); + + public DefaultJdbcRecordBinder(PreparedStatement statement, TableDefinition tableDefinition, + FieldsMetadata fieldsMetadata, + SchemaPair schemaPair, JdbcSinkConfig.PrimaryKeyMode pkMode, JdbcSinkConfig.InsertMode insertMode, + TimeZone timeZone) { + super(statement, tableDefinition, fieldsMetadata, schemaPair, pkMode, insertMode); + if (Objects.nonNull(this.timeZone)) { + this.timeZone = timeZone; + } + } + + @Override + public void bindField(int index, Schema schema, Object value, String fieldName) throws SQLException { + PreparedStatement statement = getPreparedStatement(); + if (Objects.isNull(value)) { + Integer type = getSqlTypeForSchema(schema); + if (type != null) { + statement.setNull(index, type); + } else { + statement.setObject(index, null); + } + } else { + boolean bound = maybeBindLogical(statement, index, schema, value); + if (!bound) { + bound = maybeBindDebeziumLogical(statement, index, schema, value); + } + if (!bound) { + bound = maybeBindPrimitive(statement, index, schema, value); + } + if (!bound) { + throw new io.openmessaging.connector.api.errors.ConnectException("Unsupported source data type: " + schema.getFieldType()); + } + } + } + + protected boolean maybeBindLogical(PreparedStatement statement, int index, Schema schema, + Object value) throws SQLException { + if (schema.getName() != null) { + switch (schema.getName()) { + case Decimal.LOGICAL_NAME: + statement.setBigDecimal(index, (BigDecimal) value); + return true; + case Date.LOGICAL_NAME: + java.sql.Date date; + if (value instanceof java.util.Date) { + date = new java.sql.Date(((java.util.Date) value).getTime()); + } else { + date = new java.sql.Date((int) value); + } + statement.setDate( + index, date, + DateTimeUtils.getTimeZoneCalendar(timeZone) + ); + return true; + case Time.LOGICAL_NAME: + java.sql.Time time; + if (value instanceof java.util.Date) { + time = new java.sql.Time(((java.util.Date) value).getTime()); + } else { + time = new java.sql.Time((int) value); + } + statement.setTime( + index, time, + DateTimeUtils.getTimeZoneCalendar(timeZone) + ); + return true; + case io.openmessaging.connector.api.data.logical.Timestamp.LOGICAL_NAME: + Timestamp timestamp; + if (value instanceof java.util.Date) { + timestamp = new Timestamp(((java.util.Date) value).getTime()); + } else { + timestamp = new Timestamp((long) value); + } + statement.setTimestamp( + index, timestamp, + DateTimeUtils.getTimeZoneCalendar(timeZone) + ); + return true; + default: + return false; + } + } + return false; + } + + /** + * Dialects not supporting `setObject(index, null)` can override this method + * to provide a specific sqlType, as per the JDBC documentation + * + * @param schema the schema + * @return the SQL type + */ + protected Integer getSqlTypeForSchema(Schema schema) { + return null; + } + + protected boolean maybeBindDebeziumLogical( + PreparedStatement statement, + int index, + Schema schema, + Object value + ) throws SQLException { + return DebeziumTimeTypes.maybeBindDebeziumLogical(statement, index, schema, value, timeZone); + } + + protected boolean maybeBindPrimitive( + PreparedStatement statement, + int index, + Schema schema, + Object value + ) throws SQLException { + switch (schema.getFieldType()) { + case INT8: + statement.setByte(index, Byte.parseByte(value.toString())); + break; + case INT32: + statement.setInt(index, Integer.parseInt(value.toString())); + break; + case INT64: + statement.setLong(index, Long.parseLong(value.toString())); + break; + case FLOAT32: + statement.setFloat(index, Float.parseFloat(value.toString())); + break; + case FLOAT64: + statement.setDouble(index, Double.parseDouble(value.toString())); + break; + case BOOLEAN: + statement.setBoolean(index, Boolean.parseBoolean(value.toString())); + break; + case STRING: + statement.setString(index, (String) value); + break; + case BYTES: + final byte[] bytes; + if (value instanceof ByteBuffer) { + final ByteBuffer buffer = ((ByteBuffer) value).slice(); + bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + } else { + bytes = (byte[]) value; + } + statement.setBytes(index, bytes); + break; + case DATETIME: + java.sql.Date date; + if (value instanceof java.util.Date) { + date = new java.sql.Date(((java.util.Date) value).getTime()); + } else { + date = new java.sql.Date((int) value); + } + statement.setDate( + index, date, + DateTimeUtils.getTimeZoneCalendar(timeZone) + ); + break; + default: + return false; + } + return true; + } +} diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/java/org/apache/rocketmq/connect/jdbc/mysql/sink/MysqlJdbcSinkConnector.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/binder/JdbcRecordBinder.java similarity index 67% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/java/org/apache/rocketmq/connect/jdbc/mysql/sink/MysqlJdbcSinkConnector.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/binder/JdbcRecordBinder.java index efe131153..248eb1b70 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/java/org/apache/rocketmq/connect/jdbc/mysql/sink/MysqlJdbcSinkConnector.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/binder/JdbcRecordBinder.java @@ -14,17 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.connect.jdbc.mysql.sink; -import io.openmessaging.connector.api.component.task.Task; -import org.apache.rocketmq.connect.jdbc.sink.BaseSinkConnector; +package org.apache.rocketmq.connect.jdbc.binder; -/** - * mysql jdbc sink connector - */ -public class MysqlJdbcSinkConnector extends BaseSinkConnector { - @Override - public Class taskClass() { - return MysqlJdbcSinkTask.class; - } -} +import io.openmessaging.connector.api.data.ConnectRecord; +import java.sql.SQLException; + +public interface JdbcRecordBinder { + /** + * bind record + * + * @param record + * @throws SQLException + */ + void bindRecord(ConnectRecord record) throws SQLException; +} \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/common/DebeziumTimeTypes.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/DebeziumTimeTypes.java similarity index 99% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/common/DebeziumTimeTypes.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/DebeziumTimeTypes.java index ffd4cf959..b685b9cd5 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/common/DebeziumTimeTypes.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/DebeziumTimeTypes.java @@ -20,8 +20,6 @@ import io.debezium.time.Date; import io.debezium.time.ZonedTimestamp; import io.openmessaging.connector.api.data.Schema; -import org.apache.rocketmq.connect.jdbc.util.DateTimeUtils; - import java.sql.PreparedStatement; import java.sql.SQLException; import java.time.LocalDate; @@ -29,6 +27,7 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.TimeZone; +import org.apache.rocketmq.connect.jdbc.util.DateTimeUtils; /** diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/common/HeaderField.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/HeaderField.java similarity index 92% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/common/HeaderField.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/HeaderField.java index f79c688da..f4829e931 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/common/HeaderField.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/HeaderField.java @@ -21,9 +21,6 @@ * header field */ public interface HeaderField { - String SOURCE_TABLE_KEY = "__source_table"; String SOURCE_DB_KEY = "__source_db"; - String SINK_TABLE_KEY = "__sink_table"; - String SINK_DB_KEY = "__sink_db"; } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/common/JdbcSourceConfigConstants.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/JdbcSourceConfigConstants.java similarity index 100% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/common/JdbcSourceConfigConstants.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/JdbcSourceConfigConstants.java diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/config/AbstractConfig.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/AbstractConfig.java similarity index 86% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/config/AbstractConfig.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/AbstractConfig.java index 3d7c52daf..2f0c04036 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/config/AbstractConfig.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/AbstractConfig.java @@ -18,45 +18,37 @@ import com.google.common.collect.Lists; import io.openmessaging.KeyValue; -import org.apache.rocketmq.connect.jdbc.util.QuoteMethod; - import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.regex.Pattern; +import org.apache.rocketmq.connect.jdbc.util.QuoteMethod; /** * abstract config */ public abstract class AbstractConfig { - private static final Pattern COMMA_WITH_WHITESPACE = Pattern.compile("\\s*,\\s*"); - // connection url public static final String CONNECTION_PREFIX = "connection."; public static final String CONNECTION_URL_CONFIG = CONNECTION_PREFIX + "url"; // connection user public static final String CONNECTION_USER_CONFIG = CONNECTION_PREFIX + "user"; - private static final String CONNECTION_USER_DOC = "JDBC connection user."; // connection password public static final String CONNECTION_PASSWORD_CONFIG = CONNECTION_PREFIX + "password"; - private static final String CONNECTION_PASSWORD_DOC = "JDBC connection password."; // connection attempts public static final String CONNECTION_ATTEMPTS_CONFIG = CONNECTION_PREFIX + "attempts"; - public static final String CONNECTION_ATTEMPTS_DOC = "Maximum number of attempts to retrieve a valid JDBC connection.Must be a positive integer."; + public static final int CONNECTION_ATTEMPTS_DEFAULT = 3; // backoff ms public static final String CONNECTION_BACKOFF_CONFIG = CONNECTION_PREFIX + "backoff.ms"; - public static final String CONNECTION_BACKOFF_DOC = "Backoff time in milliseconds between connection attempts."; + public static final long CONNECTION_BACKOFF_DEFAULT = 10000L; /** * quote.sql.identifiers */ public static final String QUOTE_SQL_IDENTIFIERS_CONFIG = "quote.sql.identifiers"; public static final String QUOTE_SQL_IDENTIFIERS_DEFAULT = QuoteMethod.ALWAYS.name().toString(); - public static final String QUOTE_SQL_IDENTIFIERS_DOC = - "When to quote table names, column names, and other identifiers in SQL statements. " - + "For backward compatibility, the default is ``always``."; private String connectionDbUrl; diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/provider/CachedConnectionProvider.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connection/CachedConnectionProvider.java similarity index 96% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/provider/CachedConnectionProvider.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connection/CachedConnectionProvider.java index de6716d4e..0b1f1ce28 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/provider/CachedConnectionProvider.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connection/CachedConnectionProvider.java @@ -14,14 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.connect.jdbc.dialect.provider; +package org.apache.rocketmq.connect.jdbc.connection; import io.openmessaging.connector.api.errors.ConnectException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.sql.Connection; import java.sql.SQLException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * cached connection provider @@ -115,11 +114,6 @@ public synchronized void close() { } } - @Override - public String identifier() { - return provider.identifier(); - } - protected void onConnect(Connection connection) throws SQLException { } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/java/org/apache/rocketmq/connect/jdbc/mysql/sink/MysqlJdbcSinkTask.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connection/ConnectionProvider.java similarity index 61% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/java/org/apache/rocketmq/connect/jdbc/mysql/sink/MysqlJdbcSinkTask.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connection/ConnectionProvider.java index 1bd1f3233..3197a4137 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/java/org/apache/rocketmq/connect/jdbc/mysql/sink/MysqlJdbcSinkTask.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connection/ConnectionProvider.java @@ -14,19 +14,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.connect.jdbc.mysql.sink; +package org.apache.rocketmq.connect.jdbc.connection; -import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; -import org.apache.rocketmq.connect.jdbc.mysql.dialect.MySqlDatabaseDialect; -import org.apache.rocketmq.connect.jdbc.sink.BaseSinkTask; -import org.apache.rocketmq.connect.jdbc.sink.JdbcSinkConfig; +import java.sql.Connection; +import java.sql.SQLException; /** - * mysql jdbc sink task + * A provider of JDBC {@link Connection} instances. */ -public class MysqlJdbcSinkTask extends BaseSinkTask { +public interface ConnectionProvider extends AutoCloseable { + + /** + * Create connection + * + * @return + * @throws SQLException + */ + Connection getConnection() throws SQLException; + + boolean isConnectionValid(Connection connection, int timeout) throws SQLException; + + /** + * Close this connection provider. + */ @Override - protected DatabaseDialect newDialect(JdbcSinkConfig config) { - return new MySqlDatabaseDialect(config); - } + void close(); } diff --git a/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/converter/DefaultColumnConverter.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/converter/DefaultColumnConverter.java new file mode 100644 index 000000000..a16233fb4 --- /dev/null +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/converter/DefaultColumnConverter.java @@ -0,0 +1,451 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.connect.jdbc.converter; + +import io.openmessaging.connector.api.data.Schema; +import io.openmessaging.connector.api.data.SchemaBuilder; +import io.openmessaging.connector.api.data.logical.Date; +import io.openmessaging.connector.api.data.logical.Decimal; +import io.openmessaging.connector.api.data.logical.Time; +import java.io.IOException; +import java.net.URL; +import java.sql.Blob; +import java.sql.Clob; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLXML; +import java.sql.Types; +import java.time.ZoneOffset; +import java.util.TimeZone; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; +import org.apache.rocketmq.connect.jdbc.util.DateTimeUtils; +import org.apache.rocketmq.connect.jdbc.util.NumericMapping; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DefaultColumnConverter implements JdbcColumnConverter { + + protected static final int NUMERIC_TYPE_SCALE_LOW = -84; + protected static final int NUMERIC_TYPE_SCALE_HIGH = 127; + protected static final int NUMERIC_TYPE_SCALE_UNSET = -127; + private static final Logger log = LoggerFactory.getLogger(DefaultColumnConverter.class); + // The maximum precision that can be achieved in a signed 64-bit integer is 2^63 ~= 9.223372e+18 + private static final int MAX_INTEGER_TYPE_PRECISION = 18; + + private final NumericMapping numericMapping; + private final boolean isJdbc4; + private final TimeZone timeZone; + + public DefaultColumnConverter(NumericMapping numericMapping, boolean isJdbc4, TimeZone timeZone) { + this.numericMapping = numericMapping; + this.isJdbc4 = isJdbc4; + this.timeZone = timeZone; + } + + @Override + public Object convertToConnectFieldValue(ResultSet rs, ColumnDefinition columnDefinition, int columnNumber) throws SQLException, IOException { + switch (columnDefinition.type()) { + case Types.BOOLEAN: + return rs.getBoolean(columnNumber); + + case Types.BIT: + return rs.getByte(columnNumber); + + // 8 bits int + case Types.TINYINT: + if (columnDefinition.isSignedNumber()) { + return rs.getByte(columnNumber); + } else { + return rs.getShort(columnNumber); + } + + // 16 bits int + case Types.SMALLINT: + if (columnDefinition.isSignedNumber()) { + return rs.getShort(columnNumber); + } else { + return rs.getInt(columnNumber); + } + + // 32 bits int + case Types.INTEGER: + if (columnDefinition.isSignedNumber()) { + return rs.getInt(columnNumber); + } else { + return rs.getLong(columnNumber); + } + + // 64 bits int + case Types.BIGINT: + return rs.getLong(columnNumber); + + // REAL is a single precision floating point value, i.e. a Java float + case Types.REAL: + return rs.getFloat(columnNumber); + + // FLOAT is, confusingly, double precision and effectively the same as DOUBLE. See REAL + // for single precision + case Types.FLOAT: + case Types.DOUBLE: + return rs.getDouble(columnNumber); + + case Types.NUMERIC: + if (numericMapping == NumericMapping.PRECISION_ONLY) { + int precision = columnDefinition.precision(); + int scale = columnDefinition.scale(); + log.trace("NUMERIC with precision: '{}' and scale: '{}'", precision, scale); + if (scale == 0 && precision <= MAX_INTEGER_TYPE_PRECISION) { // integer + if (precision > 9) { + return rs.getLong(columnNumber); + } else if (precision > 4) { + return rs.getInt(columnNumber); + } else if (precision > 2) { + return rs.getShort(columnNumber); + } else { + return rs.getByte(columnNumber); + } + } + } else if (numericMapping == NumericMapping.BEST_FIT) { + int precision = columnDefinition.precision(); + int scale = columnDefinition.scale(); + log.trace("NUMERIC with precision: '{}' and scale: '{}'", precision, scale); + if (precision <= MAX_INTEGER_TYPE_PRECISION) { // fits in primitive data types. + if (scale < 1 && scale >= NUMERIC_TYPE_SCALE_LOW) { // integer + if (precision > 9) { + return rs.getLong(columnNumber); + } else if (precision > 4) { + return rs.getInt(columnNumber); + } else if (precision > 2) { + return rs.getShort(columnNumber); + } else { + return rs.getByte(columnNumber); + } + } else if (scale > 0) { // floating point - use double in all cases + return rs.getDouble(columnNumber); + } + } + } else if (numericMapping == NumericMapping.BEST_FIT_EAGER_DOUBLE) { + int precision = columnDefinition.precision(); + int scale = columnDefinition.scale(); + log.trace("NUMERIC with precision: '{}' and scale: '{}'", precision, scale); + if (scale < 1 && scale >= NUMERIC_TYPE_SCALE_LOW) { // integer + if (precision <= MAX_INTEGER_TYPE_PRECISION) { // fits in primitive data types. + if (precision > 9) { + return rs.getLong(columnNumber); + } else if (precision > 4) { + return rs.getInt(columnNumber); + } else if (precision > 2) { + return rs.getShort(columnNumber); + } else { + return rs.getByte(columnNumber); + } + } + } else if (scale > 0) { // floating point - use double in all cases + return rs.getDouble(columnNumber); + } + } + // fallthrough + + case Types.DECIMAL: + final int precision = columnDefinition.precision(); + log.debug("DECIMAL with precision: '{}' and scale: '{}'", precision, columnDefinition.scale()); + final int scale = decimalScale(columnDefinition); + return rs.getBigDecimal(columnNumber, scale); + + case Types.CHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + return rs.getString(columnNumber); + + case Types.NCHAR: + case Types.NVARCHAR: + case Types.LONGNVARCHAR: + return rs.getNString(columnNumber); + + // Binary == fixed, VARBINARY and LONGVARBINARY == bytes + case Types.BINARY: + case Types.VARBINARY: + case Types.LONGVARBINARY: + return rs.getBytes(columnNumber); + + // Date is day + month + year + case Types.DATE: + return rs.getDate(columnNumber, + DateTimeUtils.getTimeZoneCalendar(TimeZone.getTimeZone(ZoneOffset.UTC))); + + // Time is a time of day -- hour, minute, seconds, nanoseconds + case Types.TIME: + return rs.getTime(columnNumber, DateTimeUtils.getTimeZoneCalendar(timeZone)); + + // Timestamp is a date + time + case Types.TIMESTAMP: + return rs.getTimestamp(columnNumber, DateTimeUtils.getTimeZoneCalendar(timeZone)); + + // Datalink is basically a URL -> string + case Types.DATALINK: + URL url = rs.getURL(columnNumber); + return (url != null) ? url.toString() : null; + + // BLOB == fixed + case Types.BLOB: + Blob blob = rs.getBlob(columnNumber); + if (blob == null) { + return null; + } else { + try { + if (blob.length() > Integer.MAX_VALUE) { + throw new IOException("Can't process BLOBs longer than " + Integer.MAX_VALUE); + } + return blob.getBytes(1, (int) blob.length()); + } finally { + if (isJdbc4) { + free(blob); + } + } + } + + case Types.CLOB: + Clob clob = rs.getClob(columnNumber); + if (clob == null) { + return null; + } else { + try { + if (clob.length() > Integer.MAX_VALUE) { + throw new IOException("Can't process CLOBs longer than " + Integer.MAX_VALUE); + } + return clob.getSubString(1, (int) clob.length()); + } finally { + if (isJdbc4) { + free(clob); + } + } + } + case Types.NCLOB: + Clob nClob = rs.getNClob(columnNumber); + if (nClob == null) { + return null; + } else { + try { + if (nClob.length() > Integer.MAX_VALUE) { + throw new IOException("Can't process NCLOBs longer than " + Integer.MAX_VALUE); + } + return nClob.getSubString(1, (int) nClob.length()); + } finally { + if (isJdbc4) { + free(nClob); + } + } + } + + // XML -> string + case Types.SQLXML: + SQLXML xml = rs.getSQLXML(columnNumber); + return xml != null ? xml.getString() : null; + + case Types.NULL: + case Types.ARRAY: + case Types.JAVA_OBJECT: + case Types.OTHER: + case Types.DISTINCT: + case Types.STRUCT: + case Types.REF: + case Types.ROWID: + default: + // These are not currently supported + log.warn("JDBC type {} ({}) not supported", columnDefinition.type(), columnDefinition.typeName()); + break; + } + return null; + } + + protected void free(Blob blob) throws SQLException { + blob.free(); + } + + protected void free(Clob clob) throws SQLException { + clob.free(); + } + + protected int decimalScale(ColumnDefinition defn) { + return defn.scale() == NUMERIC_TYPE_SCALE_UNSET ? NUMERIC_TYPE_SCALE_HIGH : defn.scale(); + } + + @Override + public String convertToConnectFieldSchema(ColumnDefinition columnDefinition, SchemaBuilder builder) { + int precision = columnDefinition.precision(); + int scale = columnDefinition.scale(); + int sqlType = columnDefinition.type(); + boolean optional = columnDefinition.isOptional(); + String fieldName = fieldNameFor(columnDefinition); + switch (sqlType) { + case Types.NULL: + log.debug("JDBC type 'NULL' not currently supported for column '{}'", fieldName); + return null; + + case Types.BOOLEAN: + builder.field(fieldName, SchemaBuilder.bool().build()); + break; + + case Types.BIT: + builder.field(fieldName, SchemaBuilder.int8().build()); + break; + + case Types.TINYINT: + if (columnDefinition.isSignedNumber()) { + builder.field(fieldName, SchemaBuilder.int8().build()); + } else { + builder.field(fieldName, SchemaBuilder.int32().build()); + } + break; + + case Types.SMALLINT: + builder.field(fieldName, SchemaBuilder.int32().build()); + break; + + case Types.INTEGER: + if (columnDefinition.isSignedNumber()) { + builder.field(fieldName, SchemaBuilder.int32().build()); + } else { + builder.field(fieldName, SchemaBuilder.int64().build()); + } + break; + + case Types.BIGINT: + builder.field(fieldName, SchemaBuilder.int64().build()); + break; + + case Types.REAL: + builder.field(fieldName, SchemaBuilder.float32().build()); + break; + + case Types.FLOAT: + case Types.DOUBLE: + builder.field(fieldName, SchemaBuilder.float64().build()); + break; + case Types.DECIMAL: + scale = decimalScale(columnDefinition); + SchemaBuilder fieldBuilder = Decimal.builder(scale); + if (optional) { + fieldBuilder.optional(); + } + builder.field(fieldName, fieldBuilder.build()); + break; + + case Types.NUMERIC: + if (numericMapping == NumericMapping.PRECISION_ONLY) { + log.debug("NUMERIC with precision: '{}' and scale: '{}'", precision, scale); + if (scale == 0 && precision <= MAX_INTEGER_TYPE_PRECISION) { // integer + builder.field(fieldName, integerSchema(optional, precision)); + break; + } + } else if (numericMapping == NumericMapping.BEST_FIT) { + log.debug("NUMERIC with precision: '{}' and scale: '{}'", precision, scale); + if (precision <= MAX_INTEGER_TYPE_PRECISION) { + if (scale < 1 && scale >= NUMERIC_TYPE_SCALE_LOW) { + builder.field(fieldName, integerSchema(optional, precision)); + break; + } else if (scale > 0) { + builder.field(fieldName, SchemaBuilder.float64().build()); + break; + } + } + } else if (numericMapping == NumericMapping.BEST_FIT_EAGER_DOUBLE) { + log.debug("NUMERIC with precision: '{}' and scale: '{}'", precision, scale); + if (scale < 1 && scale >= NUMERIC_TYPE_SCALE_LOW) { // integer + if (precision <= MAX_INTEGER_TYPE_PRECISION) { // fits in primitive data types. + builder.field(fieldName, integerSchema(optional, precision)); + break; + } + } else if (scale > 0) { + builder.field(fieldName, SchemaBuilder.float64().build()); + break; + } + } + + case Types.CHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + case Types.NCHAR: + case Types.NVARCHAR: + case Types.LONGNVARCHAR: + case Types.CLOB: + case Types.NCLOB: + case Types.DATALINK: + case Types.SQLXML: + builder.field(fieldName, SchemaBuilder.string().build()); + break; + + case Types.BINARY: + case Types.BLOB: + case Types.VARBINARY: + case Types.LONGVARBINARY: + builder.field(fieldName, SchemaBuilder.bytes().build()); + break; + + case Types.DATE: + SchemaBuilder dateSchemaBuilder = Date.builder(); + builder.field(fieldName, dateSchemaBuilder.build()); + break; + + case Types.TIME: + SchemaBuilder timeSchemaBuilder = Time.builder(); + builder.field(fieldName, timeSchemaBuilder.build()); + break; + + case Types.TIMESTAMP: + SchemaBuilder tsSchemaBuilder = io.openmessaging.connector.api.data.logical.Timestamp.builder(); + builder.field(fieldName, tsSchemaBuilder.build()); + break; + + case Types.ARRAY: + case Types.JAVA_OBJECT: + case Types.OTHER: + case Types.DISTINCT: + case Types.STRUCT: + case Types.REF: + case Types.ROWID: + default: + log.warn("JDBC type {} ({}) not supported", sqlType, columnDefinition.typeName()); + return null; + } + return fieldName; + } + + /** + * Determine the name of the field. By default this is the column alias or name. + */ + protected String fieldNameFor(ColumnDefinition columnDefinition) { + return columnDefinition.id().aliasOrName(); + } + + private Schema integerSchema(boolean optional, int precision) { + Schema schema; + if (precision > 9) { + schema = SchemaBuilder.int64().build(); + } else if (precision > 2) { + schema = SchemaBuilder.int32().build(); + } else { + schema = SchemaBuilder.int8().build(); + } + if (optional) { + schema.setOptional(true); + } + return schema; + } + +} \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/converter/JdbcColumnConverter.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/converter/JdbcColumnConverter.java new file mode 100644 index 000000000..eeb81ad41 --- /dev/null +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/converter/JdbcColumnConverter.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.connect.jdbc.converter; + +import io.openmessaging.connector.api.data.SchemaBuilder; +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; + +public interface JdbcColumnConverter { + + /** + * Convert column to connect schema + * + * @param columnDefinition + * @param schemaBuilder + * @return + */ + String convertToConnectFieldSchema(ColumnDefinition columnDefinition, SchemaBuilder schemaBuilder); + + /** + * Convert column data to connect value + * @param rs + * @param columnDefinition + * @param columnNumber + * @return + * @throws SQLException + * @throws IOException + */ + Object convertToConnectFieldValue(ResultSet rs, ColumnDefinition columnDefinition, int columnNumber) throws SQLException, IOException; +} \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DatabaseDialect.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DatabaseDialect.java new file mode 100644 index 000000000..ecd791b21 --- /dev/null +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DatabaseDialect.java @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.connect.jdbc.dialect; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.util.Calendar; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.rocketmq.connect.jdbc.binder.JdbcRecordBinder; +import org.apache.rocketmq.connect.jdbc.converter.JdbcColumnConverter; +import org.apache.rocketmq.connect.jdbc.connection.ConnectionProvider; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; +import org.apache.rocketmq.connect.jdbc.schema.table.TableDefinition; +import org.apache.rocketmq.connect.jdbc.schema.table.TableId; +import org.apache.rocketmq.connect.jdbc.sink.JdbcSinkConfig; +import org.apache.rocketmq.connect.jdbc.sink.metadata.FieldsMetadata; +import org.apache.rocketmq.connect.jdbc.sink.metadata.SchemaPair; +import org.apache.rocketmq.connect.jdbc.sink.metadata.SinkRecordField; +import org.apache.rocketmq.connect.jdbc.source.TimestampIncrementingCriteria; +import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; +import org.apache.rocketmq.connect.jdbc.util.IdentifierRules; + +import static java.util.Objects.nonNull; + +/** + * database dialect + */ +public interface DatabaseDialect extends ConnectionProvider { + /** + * dialect name + * + * @return + */ + String name(); + + JdbcColumnConverter createJdbcColumnConverter(); + + JdbcRecordBinder getJdbcRecordBinder(PreparedStatement statement, JdbcSinkConfig.PrimaryKeyMode pkMode, + SchemaPair schemaPair, FieldsMetadata fieldsMetadata, TableDefinition tableDefinition, + JdbcSinkConfig.InsertMode insertMode); + + default PreparedStatement createPreparedStatement(Connection db, String query, + int batchMaxRows) throws SQLException { + PreparedStatement stmt = db.prepareStatement(query); + if (batchMaxRows > 0) { + stmt.setFetchSize(batchMaxRows); + } + return stmt; + } + + default PreparedStatement createPreparedStatement(Connection db, String query) throws SQLException { + return createPreparedStatement(db, query, 0); + } + + default Optional executeUpdates(PreparedStatement updatePreparedStatement) throws SQLException { + Optional count = Optional.empty(); + if (nonNull(updatePreparedStatement)) { + try { + for (int updateCount : updatePreparedStatement.executeBatch()) { + if (updateCount != Statement.SUCCESS_NO_INFO) { + count = count.isPresent() + ? count.map(total -> total + updateCount) + : Optional.of((long) updateCount); + } + } + } catch (SQLException e) { + throw new SQLException(e); + } + } + return count; + } + + default Optional executeDeletes(PreparedStatement deletePreparedStatement) throws SQLException { + Optional totalDeleteCount = Optional.empty(); + if (nonNull(deletePreparedStatement)) { + try { + for (int deleteCount : deletePreparedStatement.executeBatch()) { + if (deleteCount != Statement.SUCCESS_NO_INFO) { + totalDeleteCount = totalDeleteCount.isPresent() + ? totalDeleteCount.map(total -> total + deleteCount) + : Optional.of((long) deleteCount); + } + } + } catch (SQLException e) { + throw new SQLException(e); + } + } + return totalDeleteCount; + } + + /** + * parse to Table Id + * + * @param fqn + * @return + */ + default TableId parseTableNameToTableId(String fqn) { + List parts = identifierRules().parseQualifiedIdentifier(fqn); + if (parts.isEmpty()) { + throw new IllegalArgumentException("Invalid fully qualified name: '" + fqn + "'"); + } + if (parts.size() == 1) { + return new TableId(null, null, parts.get(0)); + } + if (parts.size() == 3) { + return new TableId(parts.get(0), parts.get(1), parts.get(2)); + } + if (useCatalog()) { + return new TableId(parts.get(0), null, parts.get(1)); + } + return new TableId(null, parts.get(0), parts.get(1)); + } + + default boolean useCatalog() { + return true; + } + + /** + * Get the identifier rules for this database. + * + * @return the identifier rules + */ + IdentifierRules identifierRules(); + + ExpressionBuilder expressionBuilder(); + + List listTableIds(Connection connection) throws SQLException; + + boolean tableExists(Connection connection, TableId tableId) throws SQLException; + + TableDefinition describeTable(Connection connection, TableId tableId) throws SQLException; + + Map describeColumns(Connection connection, String tablePattern, + String columnPattern) throws SQLException; + + Map describeColumns(Connection connection, String catalogPattern, String schemaPattern, + String tablePattern, String columnPattern) throws SQLException; + + Map describeColumns(Connection conn, TableId tableId, + ResultSetMetaData rsMetadata) throws SQLException; + + Map describeColumnsByQuerying(Connection connection, + TableId tableId) throws SQLException; + + default void executeSchemaChangeStatements(Connection connection, List statements) throws SQLException { + try (Statement statement = connection.createStatement()) { + for (String ddl : statements) { + statement.executeUpdate(ddl); + } + } + } + + // Insert statement + String getInsertSql(JdbcSinkConfig config, FieldsMetadata fieldsMetadata, TableId tableId); + + default String buildInsertStatement(TableId table, Collection keyColumns, + Collection nonKeyColumns) { + ExpressionBuilder builder = expressionBuilder(); + builder.append("INSERT INTO "); + builder.append(table); + builder.append("("); + builder.appendList() + .delimitedBy(",") + .transformedBy(ExpressionBuilder.columnNames()) + .of(keyColumns, nonKeyColumns); + builder.append(") VALUES("); + builder.appendMultiple(",", "?", keyColumns.size() + nonKeyColumns.size()); + builder.append(")"); + return builder.toString(); + } + + default String buildUpdateStatement(TableId table, Collection keyColumns, + Collection nonKeyColumns) { + ExpressionBuilder builder = expressionBuilder(); + builder.append("UPDATE "); + builder.append(table); + builder.append(" SET "); + builder.appendList() + .delimitedBy(", ") + .transformedBy(ExpressionBuilder.columnNamesWith(" = ?")) + .of(nonKeyColumns); + if (!keyColumns.isEmpty()) { + builder.append(" WHERE "); + builder.appendList() + .delimitedBy(" AND ") + .transformedBy(ExpressionBuilder.columnNamesWith(" = ?")) + .of(keyColumns); + } + return builder.toString(); + } + + default String buildUpsertQueryStatement(TableId table, Collection keyColumns, + Collection nonKeyColumns) { + throw new UnsupportedOperationException(); + } + + // build delete statement + String getDeleteSql(JdbcSinkConfig config, FieldsMetadata fieldsMetadata, TableId tableId); + + default String buildDeleteStatement(TableId table, Collection keyColumns) { + ExpressionBuilder builder = expressionBuilder(); + builder.append("DELETE FROM "); + builder.append(table); + if (!keyColumns.isEmpty()) { + builder.append(" WHERE "); + builder.appendList() + .delimitedBy(" AND ") + .transformedBy(ExpressionBuilder.columnNamesWith(" = ?")) + .of(keyColumns); + } + return builder.toString(); + } + + // drop table + default String buildDropTableStatement(TableId table, boolean ifExists, boolean cascade) { + ExpressionBuilder builder = expressionBuilder(); + builder.append("DROP TABLE "); + builder.append(table); + if (ifExists) { + builder.append(" IF EXISTS"); + } + if (cascade) { + builder.append(" CASCADE"); + } + return builder.toString(); + } + + // create table + String buildCreateTableStatement(TableId table, Collection fields); + + // alter table + List buildAlterTable(TableId table, Collection fields); + + default void validateColumnTypes(ResultSetMetaData rsMetadata, + List columns) throws io.openmessaging.connector.api.errors.ConnectException { + // do nothing + } + + default String buildSelectTableMode() { + return "SELECT * FROM "; + } + + default void buildSelectTable(ExpressionBuilder builder, TableId tableId) { + String mode = buildSelectTableMode(); + builder.append(mode).append(tableId); + } + + TimestampIncrementingCriteria criteriaFor(ColumnId incrementingColumn, List timestampColumns); + + default Long getMinTimestampValue(Connection con, String tableOrQuery, + List timestampColumns) throws SQLException { + if (timestampColumns == null || timestampColumns.isEmpty()) { + return null; + } + StringBuilder builder = new StringBuilder(); + builder.append("SELECT "); + boolean appendComma = false; + for (String column : timestampColumns) { + builder.append("MIN("); + builder.append(column); + builder.append(")"); + if (appendComma) { + builder.append(","); + } else { + appendComma = true; + } + } + builder.append(" FROM "); + builder.append(tableOrQuery); + String querySql = builder.toString(); + PreparedStatement st = con.prepareStatement(querySql); + ResultSet resultSet = st.executeQuery(); + ResultSetMetaData metaData = resultSet.getMetaData(); + long minTimestampValue = Long.MAX_VALUE; + for (int i = 1; i <= metaData.getColumnCount(); ++i) { + long t = resultSet.getLong(i); + minTimestampValue = Math.min(minTimestampValue, t); + } + st.close(); + return minTimestampValue; + } + + Timestamp currentTimeOnDB(Connection connection, Calendar cal) throws SQLException; +} \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-openmldb/src/main/java/org/apache/rocketmq/connect/jdbc/openmldb/sink/OpenMLDBJdbcSinkConnector.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DatabaseDialectFactory.java similarity index 66% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-openmldb/src/main/java/org/apache/rocketmq/connect/jdbc/openmldb/sink/OpenMLDBJdbcSinkConnector.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DatabaseDialectFactory.java index 612b5ae8c..9fd0bb3e1 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-openmldb/src/main/java/org/apache/rocketmq/connect/jdbc/openmldb/sink/OpenMLDBJdbcSinkConnector.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DatabaseDialectFactory.java @@ -14,17 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.connect.jdbc.openmldb.sink; +package org.apache.rocketmq.connect.jdbc.dialect; -import io.openmessaging.connector.api.component.task.Task; -import org.apache.rocketmq.connect.jdbc.sink.BaseSinkConnector; +import java.util.Set; +import org.apache.rocketmq.connect.jdbc.config.AbstractConfig; -/** - * OpenMLDB jdbc sink connector - */ -public class OpenMLDBJdbcSinkConnector extends BaseSinkConnector { - @Override - public Class taskClass() { - return OpenMLDBJdbcSinkTask.class; - } -} +public interface DatabaseDialectFactory { + + Set subProtocols(); + + DatabaseDialect create(AbstractConfig config); +} \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DatabaseDialectLoader.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DatabaseDialectLoader.java new file mode 100644 index 000000000..db9362747 --- /dev/null +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DatabaseDialectLoader.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.connect.jdbc.dialect; + +import io.openmessaging.connector.api.errors.ConnectException; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.HashSet; +import java.util.List; +import java.util.ServiceConfigurationError; +import java.util.ServiceLoader; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.apache.rocketmq.connect.jdbc.config.AbstractConfig; +import org.apache.rocketmq.connect.jdbc.util.JdbcUrlInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class DatabaseDialectLoader { + + private static final Logger LOG = LoggerFactory.getLogger(DatabaseDialectLoader.class); + private static final Pattern PROTOCOL_PATTERN = Pattern.compile("jdbc:([^:]+):(.*)"); + private static final Set DATABASE_DIALECT_FACTORY = new HashSet<>(); + + static { + DATABASE_DIALECT_FACTORY.addAll(loadDatabaseDialectFactories()); + } + + /** + * Get database dialect factory + * + * @param config + * @return + */ + public static DatabaseDialect getDatabaseDialect(AbstractConfig config) { + String url = config.getConnectionDbUrl(); + assert url != null; + JdbcUrlInfo jdbcUrlInfo = extractJdbcUrlInfo(url); + final List matchingFactories = + DATABASE_DIALECT_FACTORY.stream().filter(f -> f.subProtocols().contains(jdbcUrlInfo.subprotocol())).collect(Collectors.toList()); + if (matchingFactories.isEmpty()) { + throw new ConnectException(String.format("Cannot get database dialect by url [%s]", url)); + } + return matchingFactories.get(0).create(config); + } + + private static Set loadDatabaseDialectFactories() { + + try { + ClassLoader cl = DatabaseDialectFactory.class.getClassLoader(); + return AccessController.doPrivileged(new PrivilegedAction>() { + public Set run() { + final Set result = new HashSet<>(); + ServiceLoader databaseDialectFactories = ServiceLoader.load(DatabaseDialectFactory.class, cl); + databaseDialectFactories.iterator().forEachRemaining(result::add); + return result; + } + }); + } catch (ServiceConfigurationError e) { + LOG.error("Could not load service provider for jdbc dialects factory.", e); + throw new ConnectException("Could not load service provider for jdbc dialects factory.", e); + } + } + + static JdbcUrlInfo extractJdbcUrlInfo(final String url) { + LOG.info("Validating JDBC URL."); + Matcher matcher = PROTOCOL_PATTERN.matcher(url); + if (matcher.matches()) { + LOG.info("Validated JDBC URL."); + return new JdbcUrlDetails(matcher.group(1), matcher.group(2), url); + } + LOG.error("Not a valid JDBC URL: " + url); + throw new ConnectException("Not a valid JDBC URL: " + url); + } + + /** + * Jdbc url details + */ + static class JdbcUrlDetails implements JdbcUrlInfo { + final String subprotocol; + final String subname; + final String url; + + public JdbcUrlDetails(String subprotocol, String subname, String url) { + this.subprotocol = subprotocol; + this.subname = subname; + this.url = url; + } + + @Override + public String subprotocol() { + return subprotocol; + } + + @Override + public String subname() { + return subname; + } + + @Override + public String url() { + return url; + } + + @Override + public String toString() { + return "JDBC subprotocol '" + subprotocol + "' and source '" + url + "'"; + } + } + +} diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/GenericDatabaseDialect.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/GenericDatabaseDialect.java similarity index 50% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/GenericDatabaseDialect.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/GenericDatabaseDialect.java index 3c84ffc0c..0923a441d 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/GenericDatabaseDialect.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/GenericDatabaseDialect.java @@ -17,42 +17,8 @@ package org.apache.rocketmq.connect.jdbc.dialect; import io.openmessaging.connector.api.data.FieldType; -import io.openmessaging.connector.api.data.Schema; -import io.openmessaging.connector.api.data.SchemaBuilder; -import io.openmessaging.connector.api.data.logical.Date; -import io.openmessaging.connector.api.data.logical.Decimal; -import io.openmessaging.connector.api.data.logical.Time; import io.openmessaging.connector.api.errors.ConnectException; -import org.apache.rocketmq.connect.jdbc.common.DebeziumTimeTypes; -import org.apache.rocketmq.connect.jdbc.config.AbstractConfig; -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefAdjuster; -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; -import org.apache.rocketmq.connect.jdbc.schema.table.TableDefinition; -import org.apache.rocketmq.connect.jdbc.schema.table.TableId; -import org.apache.rocketmq.connect.jdbc.sink.JdbcSinkConfig; -import org.apache.rocketmq.connect.jdbc.sink.metadata.FieldsMetadata; -import org.apache.rocketmq.connect.jdbc.sink.metadata.SchemaPair; -import org.apache.rocketmq.connect.jdbc.sink.metadata.SinkRecordField; -import org.apache.rocketmq.connect.jdbc.source.JdbcSourceConfig; -import org.apache.rocketmq.connect.jdbc.source.TimestampIncrementingCriteria; -import org.apache.rocketmq.connect.jdbc.source.metadata.ColumnMapping; -import org.apache.rocketmq.connect.jdbc.util.DateTimeUtils; -import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; -import org.apache.rocketmq.connect.jdbc.util.IdentifierRules; -import org.apache.rocketmq.connect.jdbc.util.JdbcDriverInfo; -import org.apache.rocketmq.connect.jdbc.util.NumericMapping; -import org.apache.rocketmq.connect.jdbc.util.QuoteMethod; -import org.apache.rocketmq.connect.jdbc.util.TableType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.math.BigDecimal; -import java.net.URL; import java.nio.ByteBuffer; -import java.sql.Blob; -import java.sql.Clob; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.DriverManager; @@ -60,11 +26,8 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.sql.SQLXML; import java.sql.Statement; import java.sql.Timestamp; -import java.sql.Types; -import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; @@ -76,6 +39,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Properties; import java.util.Queue; import java.util.Set; @@ -83,28 +47,37 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import org.apache.rocketmq.connect.jdbc.binder.DefaultJdbcRecordBinder; +import org.apache.rocketmq.connect.jdbc.binder.JdbcRecordBinder; +import org.apache.rocketmq.connect.jdbc.config.AbstractConfig; +import org.apache.rocketmq.connect.jdbc.converter.DefaultColumnConverter; +import org.apache.rocketmq.connect.jdbc.converter.JdbcColumnConverter; +import org.apache.rocketmq.connect.jdbc.util.ColumnDefAdjuster; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; +import org.apache.rocketmq.connect.jdbc.schema.table.TableDefinition; +import org.apache.rocketmq.connect.jdbc.schema.table.TableId; +import org.apache.rocketmq.connect.jdbc.sink.JdbcSinkConfig; +import org.apache.rocketmq.connect.jdbc.sink.metadata.FieldsMetadata; +import org.apache.rocketmq.connect.jdbc.sink.metadata.SchemaPair; +import org.apache.rocketmq.connect.jdbc.sink.metadata.SinkRecordField; +import org.apache.rocketmq.connect.jdbc.source.JdbcSourceConfig; +import org.apache.rocketmq.connect.jdbc.source.TimestampIncrementingCriteria; +import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; +import org.apache.rocketmq.connect.jdbc.util.IdentifierRules; +import org.apache.rocketmq.connect.jdbc.util.JdbcDriverInfo; +import org.apache.rocketmq.connect.jdbc.util.NumericMapping; +import org.apache.rocketmq.connect.jdbc.util.QuoteMethod; +import org.apache.rocketmq.connect.jdbc.util.TableType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * generic database dialect */ -public class GenericDatabaseDialect implements DatabaseDialect { +public abstract class GenericDatabaseDialect implements DatabaseDialect { private static final Logger log = LoggerFactory.getLogger(GenericDatabaseDialect.class); - protected static final int NUMERIC_TYPE_SCALE_LOW = -84; - protected static final int NUMERIC_TYPE_SCALE_HIGH = 127; - protected static final int NUMERIC_TYPE_SCALE_UNSET = -127; - - private static final int MAX_INTEGER_TYPE_PRECISION = 18; - - /** - * The provider for GenericDatabaseDialect - */ - public static class DialectName { - public static String generateDialectName(Class clazz) { - return clazz.getSimpleName().replace("DatabaseDialect", ""); - } - } - protected AbstractConfig config; /** * Whether to map {@code NUMERIC} JDBC types by precision. @@ -114,12 +87,12 @@ public static String generateDialectName(Class clazz) { protected String schemaPattern; protected Set tableTypes; protected String jdbcUrl; - private QuoteMethod quoteSqlIdentifiers = QuoteMethod.ALWAYS; + private final QuoteMethod quoteSqlIdentifiers; private IdentifierRules defaultIdentifierRules; private AtomicReference identifierRules = new AtomicReference<>(); private Queue connections = new ConcurrentLinkedQueue<>(); private volatile JdbcDriverInfo jdbcDriverInfo; - private int batchMaxRows; + private TimeZone timeZone; public GenericDatabaseDialect(AbstractConfig config) { @@ -137,7 +110,6 @@ protected GenericDatabaseDialect(AbstractConfig config, IdentifierRules defaultI tableTypes = sinkConfig.tableTypeNames(); quoteSqlIdentifiers = QuoteMethod.get(config.getQuoteSqlIdentifiers()); mapNumerics = NumericMapping.NONE; - batchMaxRows = 0; timeZone = sinkConfig.getTimeZone(); } else { JdbcSourceConfig sourceConfig = (JdbcSourceConfig) config; @@ -146,16 +118,10 @@ protected GenericDatabaseDialect(AbstractConfig config, IdentifierRules defaultI tableTypes = sourceConfig.getTableTypes().stream().map(TableType::toString).collect(Collectors.toSet()); quoteSqlIdentifiers = QuoteMethod.get(config.getQuoteSqlIdentifiers()); mapNumerics = sourceConfig.numericMapping(); - batchMaxRows = sourceConfig.getBatchMaxRows(); timeZone = sourceConfig.getTimeZone(); } } - @Override - public String name() { - return DialectName.generateDialectName(getClass()); - } - /** * init jdbc connection * @@ -176,8 +142,9 @@ public Connection getConnection() throws SQLException { properties = addConnectionProperties(properties); DriverManager.setLoginTimeout(40); Connection connection = DriverManager.getConnection(jdbcUrl, properties); - if (jdbcDriverInfo == null) { - jdbcDriverInfo = createJdbcDriverInfo(connection); + // init jdbc driver info + if (Objects.isNull(jdbcDriverInfo)){ + jdbcDriverInfo = createdJdbcDriverInfo(connection); } connections.add(connection); return connection; @@ -207,7 +174,7 @@ public void close() { @Override public boolean isConnectionValid(Connection connection, int timeout) throws SQLException { - if (jdbcDriverInfo().jdbcMajorVersion() >= 4) { + if (jdbcDriverInfo.jdbcMajorVersion() >= 4) { return connection.isValid(timeout); } String query = checkConnectionQuery(); @@ -239,76 +206,18 @@ protected String checkConnectionQuery() { /** * Get jdbc driver info - * * @return */ - protected JdbcDriverInfo jdbcDriverInfo() { - if (jdbcDriverInfo == null) { - try (Connection connection = getConnection()) { - jdbcDriverInfo = createJdbcDriverInfo(connection); - } catch (SQLException e) { - throw new io.openmessaging.connector.api.errors.ConnectException("Unable to get JDBC driver information", e); - } - } - return jdbcDriverInfo; - } - - protected JdbcDriverInfo createJdbcDriverInfo(Connection connection) throws SQLException { + protected JdbcDriverInfo createdJdbcDriverInfo(Connection connection) throws SQLException { DatabaseMetaData metadata = connection.getMetaData(); - return new JdbcDriverInfo( - metadata.getJDBCMajorVersion(), - metadata.getJDBCMinorVersion(), - metadata.getDriverName(), - metadata.getDatabaseProductName(), - metadata.getDatabaseProductVersion() + this.jdbcDriverInfo = new JdbcDriverInfo( + metadata.getJDBCMajorVersion(), + metadata.getJDBCMinorVersion(), + metadata.getDriverName(), + metadata.getDatabaseProductName(), + metadata.getDatabaseProductVersion() ); - } - - @Override - public PreparedStatement createPreparedStatement(Connection db, String query) throws SQLException { - log.trace("Creating a PreparedStatement '{}'", query); - PreparedStatement stmt = db.prepareStatement(query); - initializePreparedStatement(stmt); - return stmt; - } - - /** - * init PreparedStatement - * - * @param stmt - * @throws SQLException - */ - protected void initializePreparedStatement(PreparedStatement stmt) throws SQLException { - if (batchMaxRows > 0) { - stmt.setFetchSize(batchMaxRows); - } - } - - @Override - public TableId parseToTableId(String fqn) { - List parts = identifierRules().parseQualifiedIdentifier(fqn); - if (parts.isEmpty()) { - throw new IllegalArgumentException("Invalid fully qualified name: '" + fqn + "'"); - } - if (parts.size() == 1) { - return new TableId(null, null, parts.get(0)); - } - if (parts.size() == 3) { - return new TableId(parts.get(0), parts.get(1), parts.get(2)); - } - assert parts.size() >= 2; - if (useCatalog()) { - return new TableId(parts.get(0), null, parts.get(1)); - } - return new TableId(null, parts.get(0), parts.get(1)); - } - - /** - * Return whether the database uses JDBC catalogs. - * @return true if catalogs are used, or false otherwise - */ - protected boolean useCatalog() { - return false; + return jdbcDriverInfo; } /** @@ -318,7 +227,6 @@ protected boolean useCatalog() { protected String catalogPattern() { return catalogPattern; } - /** * schema config * @return @@ -328,7 +236,7 @@ protected String schemaPattern() { } @Override - public List tableIds(Connection conn) throws SQLException { + public List listTableIds(Connection conn) throws SQLException { DatabaseMetaData metadata = conn.getMetaData(); String[] tableTypes = tableTypes(metadata, this.tableTypes); String tableTypeDisplay = displayableTableTypes(tableTypes, ", "); @@ -422,7 +330,7 @@ public IdentifierRules identifierRules() { @Override public ExpressionBuilder expressionBuilder() { return identifierRules().expressionBuilder() - .setQuoteIdentifiers(quoteSqlIdentifiers); + .setQuoteIdentifiers(quoteSqlIdentifiers); } /** @@ -504,7 +412,7 @@ public Map describeColumns( String columnPattern ) throws SQLException { //if the table pattern is fqn, then just use the actual table name - TableId tableId = parseToTableId(tablePattern); + TableId tableId = parseTableNameToTableId(tablePattern); String catalog = tableId.catalogName() != null ? tableId.catalogName() : catalogPattern; String schema = tableId.schemaName() != null ? tableId.schemaName() : schemaPattern; return describeColumns(connection, catalog, schema, tableId.tableName(), columnPattern); @@ -833,608 +741,18 @@ protected ColumnDefinition columnDefinition( ); } - /** - * Determine the name of the field. By default this is the column alias or name. - * - * @param columnDefinition the column definition; never null - * @return the field name; never null - */ - protected String fieldNameFor(ColumnDefinition columnDefinition) { - return columnDefinition.id().aliasOrName(); - } - - @Override - public String addFieldToSchema( - ColumnDefinition columnDefn, - SchemaBuilder schemaBuilder - ) { - return addFieldToSchema(columnDefn, schemaBuilder, fieldNameFor(columnDefn), columnDefn.type(), - columnDefn.isOptional() - ); + public JdbcColumnConverter createJdbcColumnConverter() { + return new DefaultColumnConverter(mapNumerics, jdbcDriverInfo.jdbcVersionAtLeast(4, 0), timeZone); } - /** - * add field to schema - * - * @param columnDefn - * @param builder - * @param fieldName - * @param sqlType - * @param optional - * @return - */ - @SuppressWarnings("fallthrough") - protected String addFieldToSchema( - final ColumnDefinition columnDefn, - final SchemaBuilder builder, - final String fieldName, - final int sqlType, - final boolean optional - ) { - int precision = columnDefn.precision(); - int scale = columnDefn.scale(); - switch (sqlType) { - case Types.NULL: { - log.debug("JDBC type 'NULL' not currently supported for column '{}'", fieldName); - return null; - } - case Types.BOOLEAN: { - builder.field(fieldName, SchemaBuilder.bool().build()); - break; - } - - // ints <= 8 bits - case Types.BIT: { - builder.field(fieldName, SchemaBuilder.int8().build()); - break; - } - - case Types.TINYINT: { - if (columnDefn.isSignedNumber()) { - builder.field(fieldName, SchemaBuilder.int8().build()); - } else { - builder.field(fieldName, SchemaBuilder.int32().build()); - } - break; - } - - // 16 bit ints - case Types.SMALLINT: { - builder.field(fieldName, SchemaBuilder.int32().build()); - break; - } - - // 32 bit ints - case Types.INTEGER: { - if (columnDefn.isSignedNumber()) { - builder.field(fieldName, SchemaBuilder.int32().build()); - } else { - builder.field(fieldName, SchemaBuilder.int64().build()); - } - break; - } - - // 64 bit ints - case Types.BIGINT: { - builder.field(fieldName, SchemaBuilder.int64().build()); - break; - } - - // REAL is a single precision floating point value, i.e. a Java float - case Types.REAL: { - builder.field(fieldName, SchemaBuilder.float32().build()); - break; - } - - // FLOAT is, confusingly, double precision and effectively the same as DOUBLE. See REAL - // for single precision - case Types.FLOAT: - case Types.DOUBLE: - builder.field(fieldName, SchemaBuilder.float64().build()); - break; - case Types.DECIMAL: - scale = decimalScale(columnDefn); - SchemaBuilder fieldBuilder = Decimal.builder(scale); - if (optional) { - fieldBuilder.optional(); - } - builder.field(fieldName, fieldBuilder.build()); - break; - - /** - * numeric - */ - case Types.NUMERIC: - if (mapNumerics == NumericMapping.PRECISION_ONLY) { - log.debug("NUMERIC with precision: '{}' and scale: '{}'", precision, scale); - if (scale == 0 && precision <= MAX_INTEGER_TYPE_PRECISION) { // integer - builder.field(fieldName, integerSchema(optional, precision)); - break; - } - } else if (mapNumerics == NumericMapping.BEST_FIT) { - log.debug("NUMERIC with precision: '{}' and scale: '{}'", precision, scale); - if (precision <= MAX_INTEGER_TYPE_PRECISION) { // fits in primitive data types. - if (scale < 1 && scale >= NUMERIC_TYPE_SCALE_LOW) { // integer - builder.field(fieldName, integerSchema(optional, precision)); - break; - } else if (scale > 0) { // floating point - use double in all cases - builder.field(fieldName, SchemaBuilder.float64().build()); - break; - } - } - } else if (mapNumerics == NumericMapping.BEST_FIT_EAGER_DOUBLE) { - log.debug("NUMERIC with precision: '{}' and scale: '{}'", precision, scale); - if (scale < 1 && scale >= NUMERIC_TYPE_SCALE_LOW) { // integer - if (precision <= MAX_INTEGER_TYPE_PRECISION) { // fits in primitive data types. - builder.field(fieldName, integerSchema(optional, precision)); - break; - } - } else if (scale > 0) { // floating point - use double in all cases - builder.field(fieldName, SchemaBuilder.float64().build()); - break; - } - } - - case Types.CHAR: - case Types.VARCHAR: - case Types.LONGVARCHAR: - case Types.NCHAR: - case Types.NVARCHAR: - case Types.LONGNVARCHAR: - case Types.CLOB: - case Types.NCLOB: - case Types.DATALINK: - case Types.SQLXML: { - // Some of these types will have fixed size, but we drop this from the schema conversion - // since only fixed byte arrays can have a fixed size - builder.field(fieldName, SchemaBuilder.string().build()); - break; - } - - // Binary == fixed bytes - // BLOB, VARBINARY, LONGVARBINARY == bytes - case Types.BINARY: - case Types.BLOB: - case Types.VARBINARY: - case Types.LONGVARBINARY: { - builder.field(fieldName, SchemaBuilder.bytes().build()); - break; - } - - // Date is day + moth + year - case Types.DATE: { - SchemaBuilder dateSchemaBuilder = Date.builder(); - builder.field(fieldName, dateSchemaBuilder.build()); - break; - } - - // Time is a time of day -- hour, minute, seconds, nanoseconds - case Types.TIME: { - SchemaBuilder timeSchemaBuilder = Time.builder(); - builder.field(fieldName, timeSchemaBuilder.build()); - break; - } - - // Timestamp is a date + time - case Types.TIMESTAMP: { - SchemaBuilder tsSchemaBuilder = io.openmessaging.connector.api.data.logical.Timestamp.builder(); - builder.field(fieldName, tsSchemaBuilder.build()); - break; - } - - case Types.ARRAY: - case Types.JAVA_OBJECT: - case Types.OTHER: - case Types.DISTINCT: - case Types.STRUCT: - case Types.REF: - case Types.ROWID: - default: { - log.warn("JDBC type {} ({}) not currently supported", sqlType, columnDefn.typeName()); - return null; - } - } - return fieldName; - } - - private Schema integerSchema(boolean optional, int precision) { - Schema schema; - if (precision > 9) { - schema = SchemaBuilder.int64().build(); - } else if (precision > 2) { - schema = SchemaBuilder.int32().build(); - } else { - schema = SchemaBuilder.int8().build(); - } - return schema; - } - - /** - * execute ddl - * - * @param connection the connection to use - * @param statements the list of DDL statements to execute - * @throws SQLException - */ @Override - public void applyDdlStatements( - Connection connection, - List statements - ) throws SQLException { - try (Statement statement = connection.createStatement()) { - for (String ddlStatement : statements) { - statement.executeUpdate(ddlStatement); - } - } - } - - @Override - public ColumnConverter createColumnConverter(ColumnMapping mapping) { - return columnConverterFor( - mapping, - mapping.columnDefn(), - mapping.columnNumber(), - jdbcDriverInfo().jdbcVersionAtLeast(4, 0) - ); + public JdbcRecordBinder getJdbcRecordBinder(PreparedStatement statement, JdbcSinkConfig.PrimaryKeyMode pkMode, + SchemaPair schemaPair, FieldsMetadata fieldsMetadata, TableDefinition tableDefinition, + JdbcSinkConfig.InsertMode insertMode) { + return new DefaultJdbcRecordBinder(statement, tableDefinition, fieldsMetadata, schemaPair, pkMode, insertMode, timeZone); } - /** - * column converter - * - * @param mapping - * @param defn - * @param col - * @param isJdbc4 - * @return - */ - protected ColumnConverter columnConverterFor( - final ColumnMapping mapping, - final ColumnDefinition defn, - final int col, - final boolean isJdbc4 - ) { - switch (mapping.columnDefn().type()) { - case Types.BOOLEAN: { - return rs -> rs.getBoolean(col); - } - case Types.BIT: { - /** - * BIT should be either 0 or 1. - * TODO: Postgres handles this differently, returning a string "t" or "f". See the - * elasticsearch-jdbc plugin for an example of how this is handled - */ - return rs -> rs.getByte(col); - } - - // 8 bits int - case Types.TINYINT: { - if (defn.isSignedNumber()) { - return rs -> rs.getByte(col); - } else { - return rs -> rs.getShort(col); - } - } - - // 16 bits int - case Types.SMALLINT: { - if (defn.isSignedNumber()) { - return rs -> rs.getShort(col); - } else { - return rs -> rs.getInt(col); - } - } - - // 32 bits int - case Types.INTEGER: { - if (defn.isSignedNumber()) { - return rs -> rs.getInt(col); - } else { - return rs -> rs.getLong(col); - } - } - - // 64 bits int - case Types.BIGINT: { - return rs -> rs.getLong(col); - } - - // REAL is a single precision floating point value, i.e. a Java float - case Types.REAL: { - return rs -> rs.getFloat(col); - } - - // FLOAT is, confusingly, double precision and effectively the same as DOUBLE. See REAL - // for single precision - case Types.FLOAT: - case Types.DOUBLE: { - return rs -> rs.getDouble(col); - } - - case Types.NUMERIC: - if (mapNumerics == NumericMapping.PRECISION_ONLY) { - int precision = defn.precision(); - int scale = defn.scale(); - log.trace("NUMERIC with precision: '{}' and scale: '{}'", precision, scale); - if (scale == 0 && precision <= MAX_INTEGER_TYPE_PRECISION) { // integer - if (precision > 9) { - return rs -> rs.getLong(col); - } else if (precision > 4) { - return rs -> rs.getInt(col); - } else if (precision > 2) { - return rs -> rs.getShort(col); - } else { - return rs -> rs.getByte(col); - } - } - } else if (mapNumerics == NumericMapping.BEST_FIT) { - int precision = defn.precision(); - int scale = defn.scale(); - log.trace("NUMERIC with precision: '{}' and scale: '{}'", precision, scale); - if (precision <= MAX_INTEGER_TYPE_PRECISION) { // fits in primitive data types. - if (scale < 1 && scale >= NUMERIC_TYPE_SCALE_LOW) { // integer - if (precision > 9) { - return rs -> rs.getLong(col); - } else if (precision > 4) { - return rs -> rs.getInt(col); - } else if (precision > 2) { - return rs -> rs.getShort(col); - } else { - return rs -> rs.getByte(col); - } - } else if (scale > 0) { // floating point - use double in all cases - return rs -> rs.getDouble(col); - } - } - } else if (mapNumerics == NumericMapping.BEST_FIT_EAGER_DOUBLE) { - int precision = defn.precision(); - int scale = defn.scale(); - log.trace("NUMERIC with precision: '{}' and scale: '{}'", precision, scale); - if (scale < 1 && scale >= NUMERIC_TYPE_SCALE_LOW) { // integer - if (precision <= MAX_INTEGER_TYPE_PRECISION) { // fits in primitive data types. - if (precision > 9) { - return rs -> rs.getLong(col); - } else if (precision > 4) { - return rs -> rs.getInt(col); - } else if (precision > 2) { - return rs -> rs.getShort(col); - } else { - return rs -> rs.getByte(col); - } - } - } else if (scale > 0) { // floating point - use double in all cases - return rs -> rs.getDouble(col); - } - } - // fallthrough - - case Types.DECIMAL: { - final int precision = defn.precision(); - log.debug("DECIMAL with precision: '{}' and scale: '{}'", precision, defn.scale()); - final int scale = decimalScale(defn); - return rs -> rs.getBigDecimal(col, scale); - } - - case Types.CHAR: - case Types.VARCHAR: - case Types.LONGVARCHAR: { - return rs -> rs.getString(col); - } - - case Types.NCHAR: - case Types.NVARCHAR: - case Types.LONGNVARCHAR: { - return rs -> rs.getNString(col); - } - - // Binary == fixed, VARBINARY and LONGVARBINARY == bytes - case Types.BINARY: - case Types.VARBINARY: - case Types.LONGVARBINARY: { - return rs -> rs.getBytes(col); - } - - // Date is day + month + year - case Types.DATE: { - return rs -> rs.getDate(col, - DateTimeUtils.getTimeZoneCalendar(TimeZone.getTimeZone(ZoneOffset.UTC))); - } - - // Time is a time of day -- hour, minute, seconds, nanoseconds - case Types.TIME: { - return rs -> rs.getTime(col, DateTimeUtils.getTimeZoneCalendar(timeZone)); - } - - // Timestamp is a date + time - case Types.TIMESTAMP: { - return rs -> rs.getTimestamp(col, DateTimeUtils.getTimeZoneCalendar(timeZone)); - } - - // Datalink is basically a URL -> string - case Types.DATALINK: { - return rs -> { - URL url = rs.getURL(col); - return url != null ? url.toString() : null; - }; - } - - // BLOB == fixed - case Types.BLOB: { - return rs -> { - Blob blob = rs.getBlob(col); - if (blob == null) { - return null; - } else { - try { - if (blob.length() > Integer.MAX_VALUE) { - throw new IOException("Can't process BLOBs longer than " + Integer.MAX_VALUE); - } - return blob.getBytes(1, (int) blob.length()); - } finally { - if (isJdbc4) { - free(blob); - } - } - } - }; - } - case Types.CLOB: - return rs -> { - Clob clob = rs.getClob(col); - if (clob == null) { - return null; - } else { - try { - if (clob.length() > Integer.MAX_VALUE) { - throw new IOException("Can't process CLOBs longer than " + Integer.MAX_VALUE); - } - return clob.getSubString(1, (int) clob.length()); - } finally { - if (isJdbc4) { - free(clob); - } - } - } - }; - case Types.NCLOB: { - return rs -> { - Clob clob = rs.getNClob(col); - if (clob == null) { - return null; - } else { - try { - if (clob.length() > Integer.MAX_VALUE) { - throw new IOException("Can't process NCLOBs longer than " + Integer.MAX_VALUE); - } - return clob.getSubString(1, (int) clob.length()); - } finally { - if (isJdbc4) { - free(clob); - } - } - } - }; - } - - // XML -> string - case Types.SQLXML: { - return rs -> { - SQLXML xml = rs.getSQLXML(col); - return xml != null ? xml.getString() : null; - }; - } - - case Types.NULL: - case Types.ARRAY: - case Types.JAVA_OBJECT: - case Types.OTHER: - case Types.DISTINCT: - case Types.STRUCT: - case Types.REF: - case Types.ROWID: - default: { - // These are not currently supported, but we don't want to log something for every single - // record we translate. There will already be errors logged for the schema translation - break; - } - } - return null; - } - - protected int decimalScale(ColumnDefinition defn) { - return defn.scale() == NUMERIC_TYPE_SCALE_UNSET ? NUMERIC_TYPE_SCALE_HIGH : defn.scale(); - } - - /** - * Called when the object has been fully read and {@link Blob#free()} should be called. - * - * @param blob the Blob; never null - * @throws SQLException if there is a problem calling free() - */ - protected void free(Blob blob) throws SQLException { - blob.free(); - } - - /** - * Called when the object has been fully read and {@link Clob#free()} should be called. - * - * @param clob the Clob; never null - * @throws SQLException if there is a problem calling free() - */ - protected void free(Clob clob) throws SQLException { - clob.free(); - } - - @Override - public String buildInsertStatement( - TableId table, - Collection keyColumns, - Collection nonKeyColumns - ) { - ExpressionBuilder builder = expressionBuilder(); - builder.append("INSERT INTO "); - builder.append(table); - builder.append("("); - builder.appendList() - .delimitedBy(",") - .transformedBy(ExpressionBuilder.columnNames()) - .of(keyColumns, nonKeyColumns); - builder.append(") VALUES("); - builder.appendMultiple(",", "?", keyColumns.size() + nonKeyColumns.size()); - builder.append(")"); - return builder.toString(); - } - - @Override - public String buildUpdateStatement( - TableId table, - Collection keyColumns, - Collection nonKeyColumns - ) { - ExpressionBuilder builder = expressionBuilder(); - builder.append("UPDATE "); - builder.append(table); - builder.append(" SET "); - builder.appendList() - .delimitedBy(", ") - .transformedBy(ExpressionBuilder.columnNamesWith(" = ?")) - .of(nonKeyColumns); - if (!keyColumns.isEmpty()) { - builder.append(" WHERE "); - builder.appendList() - .delimitedBy(" AND ") - .transformedBy(ExpressionBuilder.columnNamesWith(" = ?")) - .of(keyColumns); - } - return builder.toString(); - } - - @Override - public String buildUpsertQueryStatement( - TableId table, - Collection keyColumns, - Collection nonKeyColumns - ) { - throw new UnsupportedOperationException(); - } - - @Override - public String buildDeleteStatement( - TableId table, - Collection keyColumns - ) { - ExpressionBuilder builder = expressionBuilder(); - builder.append("DELETE FROM "); - builder.append(table); - if (!keyColumns.isEmpty()) { - builder.append(" WHERE "); - builder.appendList() - .delimitedBy(" AND ") - .transformedBy(ExpressionBuilder.columnNamesWith(" = ?")) - .of(keyColumns); - } - return builder.toString(); - } @Override public String getInsertSql(JdbcSinkConfig config, FieldsMetadata fieldsMetadata, TableId tableId) { @@ -1463,7 +781,7 @@ public String getInsertSql(JdbcSinkConfig config, FieldsMetadata fieldsMetadata, throw new ConnectException(String.format( "Write to table '%s' in UPSERT mode is not supported with the %s dialect.", tableId, - name() + name() )); } case UPDATE: @@ -1495,7 +813,7 @@ public String getDeleteSql(JdbcSinkConfig config, FieldsMetadata fieldsMetadata, throw new ConnectException(String.format( "Deletes to table '%s' are not supported with the %s dialect.", tableId, - name() + name() )); } break; @@ -1506,242 +824,6 @@ public String getDeleteSql(JdbcSinkConfig config, FieldsMetadata fieldsMetadata, return sql; } - /** - * table mode - * - * @return - */ - @Override - public String buildSelectTableMode() { - return "SELECT * FROM "; - } - - @Override - public void buildSelectTable(ExpressionBuilder builder, TableId tableId) { - String mode = buildSelectTableMode(); - builder.append(mode).append(tableId); - } - - @Override - public StatementBinder statementBinder( - PreparedStatement statement, - JdbcSinkConfig.PrimaryKeyMode pkMode, - SchemaPair schemaPair, - FieldsMetadata fieldsMetadata, - TableDefinition tableDefinition, - JdbcSinkConfig.InsertMode insertMode) { - return new PreparedStatementBinder( - this, - statement, - pkMode, - schemaPair, - fieldsMetadata, - tableDefinition, - insertMode - ); - } - - @Override - public void bindField( - PreparedStatement statement, - int index, Schema schema, - Object value, - ColumnDefinition colDef) throws SQLException { - if (value == null) { - Integer type = getSqlTypeForSchema(schema); - if (type != null) { - statement.setNull(index, type); - } else { - statement.setObject(index, null); - } - } else { - boolean bound = maybeBindLogical(statement, index, schema, value); - if (!bound) { - bound = maybeBindDebeziumLogical(statement, index, schema, value); - } - if (!bound) { - bound = maybeBindPrimitive(statement, index, schema, value); - } - if (!bound) { - throw new io.openmessaging.connector.api.errors.ConnectException("Unsupported source data type: " + schema.getFieldType()); - } - } - } - - protected boolean maybeBindLogical( - PreparedStatement statement, - int index, - Schema schema, - Object value - ) throws SQLException { - if (schema.getName() != null) { - switch (schema.getName()) { - case Decimal.LOGICAL_NAME: - statement.setBigDecimal(index, (BigDecimal) value); - return true; - case Date.LOGICAL_NAME: - java.sql.Date date; - if (value instanceof java.util.Date) { - date = new java.sql.Date(((java.util.Date) value).getTime()); - } else { - date = new java.sql.Date((int) value); - } - statement.setDate( - index, date, - DateTimeUtils.getTimeZoneCalendar(timeZone) - ); - return true; - case Time.LOGICAL_NAME: - java.sql.Time time; - if (value instanceof java.util.Date) { - time = new java.sql.Time(((java.util.Date) value).getTime()); - } else { - time = new java.sql.Time((int) value); - } - statement.setTime( - index, time, - DateTimeUtils.getTimeZoneCalendar(timeZone) - ); - return true; - case io.openmessaging.connector.api.data.logical.Timestamp.LOGICAL_NAME: - Timestamp timestamp; - if (value instanceof java.util.Date) { - timestamp = new Timestamp(((java.util.Date) value).getTime()); - } else { - timestamp = new Timestamp((long) value); - } - statement.setTimestamp( - index, timestamp, - DateTimeUtils.getTimeZoneCalendar(timeZone) - ); - return true; - default: - return false; - } - } - return false; - } - - - @Override - public TimestampIncrementingCriteria criteriaFor(ColumnId incrementingColumn, List timestampColumns) { - return new TimestampIncrementingCriteria( - incrementingColumn, timestampColumns, timeZone); - } - - @Override - public Long getMinTimestampValue(Connection con, String tableOrQuery, List timestampColumns) throws SQLException { - if (timestampColumns == null || timestampColumns.isEmpty()) { - return null; - } - StringBuilder builder = new StringBuilder(); - builder.append("SELECT "); - boolean appendComma = false; - for (String column : timestampColumns) { - builder.append("MIN("); - builder.append(column); - builder.append(")"); - if (appendComma) { - builder.append(","); - } else { - appendComma = true; - } - } - builder.append(" FROM "); - builder.append(tableOrQuery); - String querySql = builder.toString(); - PreparedStatement st = con.prepareStatement(querySql); - ResultSet resultSet = st.executeQuery(); - ResultSetMetaData metaData = resultSet.getMetaData(); - long minTimestampValue = Long.MAX_VALUE; - for (int i = 1; i <= metaData.getColumnCount(); ++i) { - long t = resultSet.getLong(i); - minTimestampValue = Math.min(minTimestampValue, t); - } - st.close(); - return minTimestampValue; - } - - /** - * Dialects not supporting `setObject(index, null)` can override this method - * to provide a specific sqlType, as per the JDBC documentation - * https://docs.oracle.com/javase/7/docs/api/java/sql/PreparedStatement.html - * - * @param schema the schema - * @return the SQL type - */ - protected Integer getSqlTypeForSchema(Schema schema) { - return null; - } - - protected boolean maybeBindDebeziumLogical( - PreparedStatement statement, - int index, - Schema schema, - Object value - ) throws SQLException { - return DebeziumTimeTypes.maybeBindDebeziumLogical(statement, index, schema, value, timeZone); - } - - protected boolean maybeBindPrimitive( - PreparedStatement statement, - int index, - Schema schema, - Object value - ) throws SQLException { - switch (schema.getFieldType()) { - case INT8: - statement.setByte(index, Byte.parseByte(value.toString())); - break; - case INT32: - statement.setInt(index, Integer.parseInt(value.toString())); - break; - case INT64: - statement.setLong(index, Long.parseLong(value.toString())); - break; - case FLOAT32: - statement.setFloat(index, Float.parseFloat(value.toString())); - break; - case FLOAT64: - statement.setDouble(index, Double.parseDouble(value.toString())); - break; - case BOOLEAN: - statement.setBoolean(index, Boolean.parseBoolean(value.toString())); - break; - case STRING: - statement.setString(index, (String) value); - break; - case BYTES: - final byte[] bytes; - if (value instanceof ByteBuffer) { - final ByteBuffer buffer = ((ByteBuffer) value).slice(); - bytes = new byte[buffer.remaining()]; - buffer.get(bytes); - } else { - bytes = (byte[]) value; - } - statement.setBytes(index, bytes); - break; - case DATETIME: - java.sql.Date date; - if (value instanceof java.util.Date) { - date = new java.sql.Date(((java.util.Date) value).getTime()); - } else { - date = new java.sql.Date((int) value); - } - statement.setDate( - index, date, - DateTimeUtils.getTimeZoneCalendar(timeZone) - ); - - - break; - default: - return false; - } - return true; - } - /** * create table statement * @@ -1773,33 +855,8 @@ public String buildCreateTableStatement( return builder.toString(); } - /** - * Drop table statement - * - * @param table - * @param options - * @return - */ - @Override - public String buildDropTableStatement( - TableId table, - DropOptions options - ) { - ExpressionBuilder builder = expressionBuilder(); - builder.append("DROP TABLE "); - builder.append(table); - if (options.ifExists()) { - builder.append(" IF EXISTS"); - } - if (options.cascade()) { - builder.append(" CASCADE"); - } - return builder.toString(); - } - /** * alter table statement - * * @param table * @param fields * @return @@ -1829,11 +886,9 @@ public List buildAlterTable( } @Override - public void validateColumnTypes( - ResultSetMetaData rsMetadata, - List columns - ) throws io.openmessaging.connector.api.errors.ConnectException { - // No-op + public TimestampIncrementingCriteria criteriaFor(ColumnId incrementingColumn, List timestampColumns) { + return new TimestampIncrementingCriteria( + incrementingColumn, timestampColumns, timeZone); } protected List extractPrimaryKeyFieldNames(Collection fields) { @@ -1922,34 +977,19 @@ protected void formatColumnValue( protected String getSqlType(SinkRecordField f) { throw new io.openmessaging.connector.api.errors.ConnectException(String.format( - "%s (%s) type doesn't have a mapping to the SQL database column type", f.schemaName(), - f.schemaType() + "%s (%s) type doesn't have a mapping to the SQL database column type", f.schemaName(), + f.schemaType() )); } - - /** - * @param url - * @return - */ - protected String sanitizedUrl(String url) { - return url.replaceAll("(?i)([?&]([^=&]*)password([^=&]*)=)[^&]*", "$1****"); - } - - @Override - public String identifier() { - return name() + " database " + sanitizedUrl(jdbcUrl); + private Collection asColumns(Collection names, TableId tableId) { + return names.stream() + .map(name -> new ColumnId(tableId, name)) + .collect(Collectors.toList()); } @Override public String toString() { return name(); } - - - private Collection asColumns(Collection names, TableId tableId) { - return names.stream() - .map(name -> new ColumnId(tableId, name)) - .collect(Collectors.toList()); - } } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/java/org/apache/rocketmq/connect/jdbc/mysql/dialect/MySqlDatabaseDialect.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/mysql/MySqlDatabaseDialect.java similarity index 77% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/java/org/apache/rocketmq/connect/jdbc/mysql/dialect/MySqlDatabaseDialect.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/mysql/MySqlDatabaseDialect.java index da03bb4be..7ebac1276 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/java/org/apache/rocketmq/connect/jdbc/mysql/dialect/MySqlDatabaseDialect.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/mysql/MySqlDatabaseDialect.java @@ -14,8 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.connect.jdbc.mysql.dialect; +package org.apache.rocketmq.connect.jdbc.dialect.mysql; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collection; import org.apache.rocketmq.connect.jdbc.config.AbstractConfig; import org.apache.rocketmq.connect.jdbc.dialect.GenericDatabaseDialect; import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; @@ -23,41 +28,33 @@ import org.apache.rocketmq.connect.jdbc.sink.metadata.SinkRecordField; import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; import org.apache.rocketmq.connect.jdbc.util.IdentifierRules; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collection; /** * mysql database dialect */ public class MySqlDatabaseDialect extends GenericDatabaseDialect { - - private final static Logger log = LoggerFactory.getLogger(MySqlDatabaseDialect.class); - - /** - * Create a new dialect instance with the given connector configuration. - * - * @param config the connector configuration; may not be null - */ public MySqlDatabaseDialect(AbstractConfig config) { super(config, new IdentifierRules(".", "`", "`")); } - /** - * initialize prepared statement - * - * @param stmt - * @throws SQLException - */ @Override - protected void initializePreparedStatement(PreparedStatement stmt) throws SQLException { - stmt.setFetchSize(Integer.MIN_VALUE); - log.trace("Initializing PreparedStatement fetch direction to FETCH_FORWARD for '{}'", stmt); + public String name() { + return "mysql"; + } + + @Override public PreparedStatement createPreparedStatement(Connection db, String query) throws SQLException { + return createPreparedStatement(db, query, Integer.MIN_VALUE); + } + + @Override + public PreparedStatement createPreparedStatement(Connection db, String query, + int batchMaxRows) throws SQLException { + PreparedStatement stmt = db.prepareStatement(query); + if (batchMaxRows > 0 || batchMaxRows == Integer.MIN_VALUE) { + stmt.setFetchSize(batchMaxRows); + } stmt.setFetchDirection(ResultSet.FETCH_FORWARD); + return stmt; } /** @@ -121,13 +118,4 @@ public String buildUpsertQueryStatement( .of(nonKeyColumns.isEmpty() ? keyColumns : nonKeyColumns); return builder.toString(); } - - @Override - protected String sanitizedUrl(String url) { - // MySQL can also have "username:password@" at the beginning of the host list and - // in parenthetical properties - return super.sanitizedUrl(url) - .replaceAll("(?i)([(,]password=)[^,)]*", "$1****") - .replaceAll("(://[^:]*:)([^@]*)@", "$1****@"); - } } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/java/org/apache/rocketmq/connect/jdbc/mysql/source/MysqlJdbcSourceTask.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/mysql/MysqlDatabaseDialectFactory.java similarity index 65% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/java/org/apache/rocketmq/connect/jdbc/mysql/source/MysqlJdbcSourceTask.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/mysql/MysqlDatabaseDialectFactory.java index 402936851..5c93254f2 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/java/org/apache/rocketmq/connect/jdbc/mysql/source/MysqlJdbcSourceTask.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/mysql/MysqlDatabaseDialectFactory.java @@ -14,19 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.connect.jdbc.mysql.source; +package org.apache.rocketmq.connect.jdbc.dialect.mysql; +import com.google.common.collect.Sets; +import java.util.Set; +import org.apache.rocketmq.connect.jdbc.config.AbstractConfig; import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; -import org.apache.rocketmq.connect.jdbc.mysql.dialect.MySqlDatabaseDialect; -import org.apache.rocketmq.connect.jdbc.source.BaseSourceTask; -import org.apache.rocketmq.connect.jdbc.source.JdbcSourceTaskConfig; +import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialectFactory; -/** - * mysql jdbc source task - */ -public class MysqlJdbcSourceTask extends BaseSourceTask{ +public class MysqlDatabaseDialectFactory implements DatabaseDialectFactory { @Override - protected DatabaseDialect newDialect(JdbcSourceTaskConfig config) { + public Set subProtocols() { + return Sets.newHashSet("mariadb", "mysql"); + } + + @Override public DatabaseDialect create(AbstractConfig config) { return new MySqlDatabaseDialect(config); } -} +} \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-openmldb/src/main/java/org/apache/rocketmq/connect/jdbc/openmldb/dialect/OpenMLDBDatabaseDialect.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/openmldb/OpenMLDBDatabaseDialect.java similarity index 84% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-openmldb/src/main/java/org/apache/rocketmq/connect/jdbc/openmldb/dialect/OpenMLDBDatabaseDialect.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/openmldb/OpenMLDBDatabaseDialect.java index 00d115fc6..9b810bc59 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-openmldb/src/main/java/org/apache/rocketmq/connect/jdbc/openmldb/dialect/OpenMLDBDatabaseDialect.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/openmldb/OpenMLDBDatabaseDialect.java @@ -14,25 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.connect.jdbc.openmldb.dialect; +package org.apache.rocketmq.connect.jdbc.dialect.openmldb; -import io.openmessaging.connector.api.data.Schema; import io.openmessaging.connector.api.data.logical.Date; import io.openmessaging.connector.api.data.logical.Time; import io.openmessaging.connector.api.data.logical.Timestamp; +import java.util.Collection; +import java.util.List; import org.apache.rocketmq.connect.jdbc.config.AbstractConfig; -import org.apache.rocketmq.connect.jdbc.dialect.DropOptions; import org.apache.rocketmq.connect.jdbc.dialect.GenericDatabaseDialect; import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; import org.apache.rocketmq.connect.jdbc.schema.table.TableId; import org.apache.rocketmq.connect.jdbc.sink.metadata.SinkRecordField; import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; import org.apache.rocketmq.connect.jdbc.util.IdentifierRules; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collection; -import java.util.List; /** @@ -40,8 +35,6 @@ */ public class OpenMLDBDatabaseDialect extends GenericDatabaseDialect { - private final Logger log = LoggerFactory.getLogger(OpenMLDBDatabaseDialect.class); - /** * create openMLDB database dialect * @@ -51,6 +44,10 @@ public OpenMLDBDatabaseDialect(AbstractConfig config) { super(config, new IdentifierRules(".", "`", "`")); } + @Override + public String name() { + return "OpenMLDB"; + } @Override protected String currentTimestampDatabaseQuery() { @@ -111,11 +108,10 @@ protected void writeColumnSpec(ExpressionBuilder builder, SinkRecordField f) { } else if (!this.isColumnOptional(f)) { builder.append(" NOT NULL"); } - } @Override - public String buildDropTableStatement(TableId table, DropOptions options) { + public String buildDropTableStatement(TableId table, boolean ifExists, boolean cascade) { ExpressionBuilder builder = this.expressionBuilder(); builder.append("DROP TABLE "); builder.append(table); @@ -137,18 +133,4 @@ public String buildDeleteStatement(TableId table, Collection keyColumn throw new UnsupportedOperationException("delete is unsupported"); } - @Override - protected Integer getSqlTypeForSchema(Schema schema) { - return 0; - } - - - @Override - protected String sanitizedUrl(String url) { - // MySQL can also have "username:password@" at the beginning of the host list and - // in parenthetical properties - return super.sanitizedUrl(url) - .replaceAll("(?i)([(,]password=)[^,)]*", "$1****") - .replaceAll("(://[^:]*:)([^@]*)@", "$1****@"); - } } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-openmldb/src/main/java/org/apache/rocketmq/connect/jdbc/openmldb/sink/OpenMLDBJdbcSinkTask.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/openmldb/OpenMLDBDatabaseDialectFactory.java similarity index 65% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-openmldb/src/main/java/org/apache/rocketmq/connect/jdbc/openmldb/sink/OpenMLDBJdbcSinkTask.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/openmldb/OpenMLDBDatabaseDialectFactory.java index d237e8da0..54c2fba98 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-openmldb/src/main/java/org/apache/rocketmq/connect/jdbc/openmldb/sink/OpenMLDBJdbcSinkTask.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/openmldb/OpenMLDBDatabaseDialectFactory.java @@ -14,19 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.connect.jdbc.openmldb.sink; +package org.apache.rocketmq.connect.jdbc.dialect.openmldb; +import com.google.common.collect.Sets; +import java.util.Set; +import org.apache.rocketmq.connect.jdbc.config.AbstractConfig; import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; -import org.apache.rocketmq.connect.jdbc.openmldb.dialect.OpenMLDBDatabaseDialect; -import org.apache.rocketmq.connect.jdbc.source.BaseSourceTask; -import org.apache.rocketmq.connect.jdbc.source.JdbcSourceTaskConfig; +import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialectFactory; -/** - * OpenMLDB jdbc source task - */ -public class OpenMLDBJdbcSinkTask extends BaseSourceTask { +public class OpenMLDBDatabaseDialectFactory implements DatabaseDialectFactory { @Override - protected DatabaseDialect newDialect(JdbcSourceTaskConfig config) { + public Set subProtocols() { + return Sets.newHashSet("openmldb"); + } + + @Override public DatabaseDialect create(AbstractConfig config) { return new OpenMLDBDatabaseDialect(config); } -} +} \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/exception/ConfigException.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/exception/ConfigException.java similarity index 100% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/exception/ConfigException.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/exception/ConfigException.java diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/exception/TableAlterOrCreateException.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/exception/TableAlterOrCreateException.java similarity index 100% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/exception/TableAlterOrCreateException.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/exception/TableAlterOrCreateException.java diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnDefinition.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnDefinition.java similarity index 99% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnDefinition.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnDefinition.java index fceadefb8..364d52706 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnDefinition.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnDefinition.java @@ -16,10 +16,9 @@ */ package org.apache.rocketmq.connect.jdbc.schema.column; -import org.apache.rocketmq.connect.jdbc.schema.table.TableId; - import java.sql.Types; import java.util.Objects; +import org.apache.rocketmq.connect.jdbc.schema.table.TableId; /** * column definition diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnId.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnId.java similarity index 99% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnId.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnId.java index a6b280322..e02c48083 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnId.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnId.java @@ -16,12 +16,11 @@ */ package org.apache.rocketmq.connect.jdbc.schema.column; +import java.util.Objects; import org.apache.rocketmq.connect.jdbc.schema.table.TableId; import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; import org.apache.rocketmq.connect.jdbc.util.QuoteMethod; -import java.util.Objects; - /** * column id */ diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableDefinition.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableDefinition.java similarity index 99% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableDefinition.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableDefinition.java index be6472450..925cf7a0f 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableDefinition.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableDefinition.java @@ -16,14 +16,13 @@ */ package org.apache.rocketmq.connect.jdbc.schema.table; -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; -import org.apache.rocketmq.connect.jdbc.util.TableType; - import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import java.util.Set; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; +import org.apache.rocketmq.connect.jdbc.util.TableType; /** diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableDefinitions.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableDefinitions.java similarity index 99% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableDefinitions.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableDefinitions.java index 719ee2e43..6bd980709 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableDefinitions.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableDefinitions.java @@ -16,14 +16,13 @@ */ package org.apache.rocketmq.connect.jdbc.schema.table; -import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.sql.Connection; import java.sql.SQLException; import java.util.HashMap; import java.util.Map; +import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A simple cache of {@link TableDefinition} keyed. diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableId.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableId.java similarity index 99% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableId.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableId.java index 776f78629..79fbd9a5c 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableId.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/table/TableId.java @@ -16,12 +16,11 @@ */ package org.apache.rocketmq.connect.jdbc.schema.table; +import java.util.Objects; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; import org.apache.rocketmq.connect.jdbc.util.QuoteMethod; -import java.util.Objects; - public class TableId implements Comparable, ExpressionBuilder.Expressable { private final String catalogName; diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/BufferedRecords.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/BufferedRecords.java similarity index 57% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/BufferedRecords.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/BufferedRecords.java index b0418184e..2ce03e6eb 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/BufferedRecords.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/BufferedRecords.java @@ -18,16 +18,6 @@ import io.openmessaging.connector.api.data.ConnectRecord; import io.openmessaging.connector.api.data.Schema; -import io.openmessaging.connector.api.errors.ConnectException; -import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; -import org.apache.rocketmq.connect.jdbc.dialect.GenericDatabaseDialect; -import org.apache.rocketmq.connect.jdbc.schema.db.DbStructure; -import org.apache.rocketmq.connect.jdbc.schema.table.TableId; -import org.apache.rocketmq.connect.jdbc.sink.metadata.FieldsMetadata; -import org.apache.rocketmq.connect.jdbc.sink.metadata.SchemaPair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -35,6 +25,13 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import org.apache.rocketmq.connect.jdbc.binder.JdbcRecordBinder; +import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; +import org.apache.rocketmq.connect.jdbc.schema.table.TableId; +import org.apache.rocketmq.connect.jdbc.sink.metadata.FieldsMetadata; +import org.apache.rocketmq.connect.jdbc.sink.metadata.SchemaPair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static java.util.Objects.isNull; import static java.util.Objects.nonNull; @@ -57,11 +54,12 @@ public class BufferedRecords { private RecordValidator recordValidator; private PreparedStatement updatePreparedStatement; private PreparedStatement deletePreparedStatement; - private DatabaseDialect.StatementBinder updateStatementBinder; - private DatabaseDialect.StatementBinder deleteStatementBinder; + private JdbcRecordBinder updateRecordBinder; + private JdbcRecordBinder deleteRecordBinder; private boolean deletesInBatch = false; - public BufferedRecords(JdbcSinkConfig config, TableId tableId, DatabaseDialect dbDialect, DbStructure dbStructure, Connection connection) { + public BufferedRecords(JdbcSinkConfig config, TableId tableId, DatabaseDialect dbDialect, DbStructure dbStructure, + Connection connection) { this.tableId = tableId; this.config = config; this.dbDialect = dbDialect; @@ -80,32 +78,11 @@ public BufferedRecords(JdbcSinkConfig config, TableId tableId, DatabaseDialect d public List add(ConnectRecord record) throws SQLException { recordValidator.validate(record); final List flushed = new ArrayList<>(); - // check schema changed - boolean schemaChanged = isSchemaChanged(record); - //Judge and flush delete data - flushDeletedRecord(record, flushed); + boolean schemaChanged = schemaChangedOrFlushedDeleteRecordIfNeed(record, flushed); - if (schemaChanged || updateStatementBinder == null) { - // Each batch needs to have the same schemas, so get the buffered records out - flushed.addAll(flush()); - // re-initialize everything that depends on the record schema - final SchemaPair schemaPair = new SchemaPair(record.getKeySchema(), record.getSchema(), record.getExtensions()); - // extract field - fieldsMetadata = FieldsMetadata.extract(tableId.tableName(), config.pkMode, config.getPkFields(), config.getFieldsWhitelist(), schemaPair); - // create or alter table - dbStructure.createOrAmendIfNecessary(config, connection, tableId, fieldsMetadata); - // build insert sql and delete sql - final String insertSql = dbDialect.getInsertSql(config, fieldsMetadata, tableId); - final String deleteSql = dbDialect.getDeleteSql(config, fieldsMetadata, tableId); - - log.debug("{} sql: {} deleteSql: {} meta: {}", config.getInsertMode(), insertSql, deleteSql, fieldsMetadata); - close(); - updatePreparedStatement = dbDialect.createPreparedStatement(connection, insertSql); - updateStatementBinder = dbDialect.statementBinder(updatePreparedStatement, config.pkMode, schemaPair, fieldsMetadata, dbStructure.tableDefinition(connection, tableId), config.getInsertMode()); - if (config.isDeleteEnabled() && nonNull(deleteSql)) { - deletePreparedStatement = dbDialect.createPreparedStatement(connection, deleteSql); - deleteStatementBinder = dbDialect.statementBinder(deletePreparedStatement, config.pkMode, schemaPair, fieldsMetadata, dbStructure.tableDefinition(connection, tableId), config.getInsertMode()); - } + // First record add or schema changed + if (schemaChanged || updateRecordBinder == null) { + flushedAndReinitializeStatement(record, flushed); } // set deletesInBatch if schema value is not null if (isNull(record.getData()) && config.isDeleteEnabled()) { @@ -118,6 +95,52 @@ public List add(ConnectRecord record) throws SQLException { return flushed; } + private void flushedAndReinitializeStatement(ConnectRecord record, + List flushed) throws SQLException { + // Each batch needs to have the same schemas, so get the buffered records out + flushed.addAll(flush()); + // First close statement + close(); + // re-initialize everything that depends on the record schema + final SchemaPair schemaPair = new SchemaPair(record.getKeySchema(), record.getSchema(), record.getExtensions()); + // extract field + fieldsMetadata = FieldsMetadata.extract(tableId.tableName(), config.pkMode, config.getPkFields(), config.getFieldsWhitelist(), schemaPair); + // create or alter table + dbStructure.createOrAmendIfNecessary(config, connection, tableId, fieldsMetadata); + // build insert sql and delete sql + final String insertSql = dbDialect.getInsertSql(config, fieldsMetadata, tableId); + final String deleteSql = dbDialect.getDeleteSql(config, fieldsMetadata, tableId); + log.info("{} sql: {} deleteSql: {} meta: {}", config.getInsertMode(), insertSql, deleteSql, fieldsMetadata); + updatePreparedStatement = dbDialect.createPreparedStatement(connection, insertSql); + updateRecordBinder = dbDialect.getJdbcRecordBinder(updatePreparedStatement, config.pkMode, schemaPair, fieldsMetadata, dbStructure.tableDefinition(connection, tableId), config.getInsertMode()); + if (config.isDeleteEnabled() && nonNull(deleteSql)) { + deletePreparedStatement = dbDialect.createPreparedStatement(connection, deleteSql); + deleteRecordBinder = dbDialect.getJdbcRecordBinder(deletePreparedStatement, config.pkMode, schemaPair, fieldsMetadata, dbStructure.tableDefinition(connection, tableId), config.getInsertMode()); + } + } + + private boolean schemaChangedOrFlushedDeleteRecordIfNeed(ConnectRecord record, + List flushed) throws SQLException { + boolean schemaChanged = false; + if (!Objects.equals(keySchema, record.getKeySchema())) { + keySchema = record.getKeySchema(); + schemaChanged = true; + } + if (isNull(record.getSchema())) { + if (config.isDeleteEnabled()) { + deletesInBatch = true; + } + } else if (Objects.equals(valueSchema, record.getSchema())) { + if (config.isDeleteEnabled() && deletesInBatch) { + flushed.addAll(flush()); + } + } else { + valueSchema = record.getSchema(); + schemaChanged = true; + } + return schemaChanged; + } + public List flush() throws SQLException { if (records.isEmpty()) { log.debug("Records is empty"); @@ -125,41 +148,26 @@ public List flush() throws SQLException { } log.debug("Flushing {} buffered records", records.size()); for (ConnectRecord record : records) { - if (isNull(record.getData()) && nonNull(deleteStatementBinder)) { - deleteStatementBinder.bindRecord(record); + if (isNull(record.getData()) && nonNull(deleteRecordBinder)) { + deleteRecordBinder.bindRecord(record); } else { - updateStatementBinder.bindRecord(record); + updateRecordBinder.bindRecord(record); } } - Optional totalUpdateCount = - updateStatementBinder != null ? updateStatementBinder.executeUpdates(updatePreparedStatement) : - Optional.empty(); - long totalDeleteCount = deleteStatementBinder != null ? - deleteStatementBinder.executeDeletes(deletePreparedStatement) : 0; + Optional totalUpdateCount = dbDialect.executeUpdates(updatePreparedStatement); + Optional totalDeleteCount = dbDialect.executeDeletes(deletePreparedStatement); final long expectedCount = updateRecordCount(); - log.trace("{} records:{} resulting in totalUpdateCount:{} totalDeleteCount:{}", - config.getInsertMode(), - records.size(), - totalUpdateCount, - totalDeleteCount - ); + log.trace("{} records:{} resulting in totalUpdateCount:{} totalDeleteCount:{}", config.getInsertMode(), records.size(), totalUpdateCount, totalDeleteCount); + if (totalUpdateCount.filter(total -> total != expectedCount).isPresent() && config.getInsertMode() == JdbcSinkConfig.InsertMode.INSERT) { - if (dbDialect.name().equals(GenericDatabaseDialect.DialectName.generateDialectName(dbDialect.getDialectClass())) && totalUpdateCount.get() == 0) { - // openMLDB execute success result 0; do nothing - } else { - throw new ConnectException( - String.format( - "Update count (%d) did not sum up to total number of records inserted (%d)", - totalUpdateCount.get(), - expectedCount - ) - ); - } + log.warn(String.format("Update count (%d) did not sum up to total number of records inserted (%d)", totalUpdateCount.get(), expectedCount)); } + if (!totalUpdateCount.isPresent()) { log.info("{} records:{} , but no count of the number of rows it affected is available", config.getInsertMode(), records.size()); } + final List flushedRecords = records; records = new ArrayList<>(); return flushedRecords; @@ -173,33 +181,6 @@ private long updateRecordCount() { .count(); } - private boolean isSchemaChanged(ConnectRecord record) { - boolean schemaChanged = false; - if (!Objects.equals(keySchema, record.getKeySchema())) { - keySchema = record.getKeySchema(); - schemaChanged = true; - } - if (!isNull(record.getSchema()) && !Objects.equals(valueSchema, record.getSchema())) { - // value schema is not null and has changed. This is a real schema change. - valueSchema = record.getSchema(); - schemaChanged = true; - } - return schemaChanged; - } - - private void flushDeletedRecord(ConnectRecord record, List flushed) throws SQLException { - if (isNull(record.getSchema())) { - if (config.isDeleteEnabled()) { - deletesInBatch = true; - } - } else if (Objects.equals(valueSchema, record.getSchema())) { - if (config.isDeleteEnabled() && deletesInBatch) { - // flush so an insert after a delete of same record isn't lost - flushed.addAll(flush()); - } - } - } - public void close() throws SQLException { log.debug( "Closing BufferedRecords with updatePreparedStatement: {} deletePreparedStatement: {}", diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/db/DbStructure.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/DbStructure.java similarity index 80% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/db/DbStructure.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/DbStructure.java index f1e7cfe71..7f1f3f510 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/db/DbStructure.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/DbStructure.java @@ -14,68 +14,56 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.connect.jdbc.schema.db; +package org.apache.rocketmq.connect.jdbc.sink; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; import org.apache.rocketmq.connect.jdbc.exception.TableAlterOrCreateException; import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; import org.apache.rocketmq.connect.jdbc.schema.table.TableDefinition; import org.apache.rocketmq.connect.jdbc.schema.table.TableDefinitions; import org.apache.rocketmq.connect.jdbc.schema.table.TableId; -import org.apache.rocketmq.connect.jdbc.sink.JdbcSinkConfig; import org.apache.rocketmq.connect.jdbc.sink.metadata.FieldsMetadata; import org.apache.rocketmq.connect.jdbc.sink.metadata.SinkRecordField; import org.apache.rocketmq.connect.jdbc.util.TableType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - /** * */ public class DbStructure { private static final Logger log = LoggerFactory.getLogger(DbStructure.class); private final DatabaseDialect dbDialect; - private final TableDefinitions tableDefns; + private final TableDefinitions tableDefinitions; public DbStructure(DatabaseDialect dbDialect) { this.dbDialect = dbDialect; - this.tableDefns = new TableDefinitions(dbDialect); + this.tableDefinitions = new TableDefinitions(dbDialect); } - /** - * Create or amend table. - * - * @param config the connector configuration - * @param connection the database connection handle - * @param tableId the table ID - * @param fieldsMetadata the fields metadata - * @return whether a DDL operation was performed - * @throws SQLException if a DDL operation was deemed necessary but failed - */ + public boolean createOrAmendIfNecessary( final JdbcSinkConfig config, final Connection connection, final TableId tableId, final FieldsMetadata fieldsMetadata ) throws SQLException { - if (tableDefns.get(connection, tableId) == null) { - // Table does not yet exist, so attempt to create it ... + if (tableDefinitions.get(connection, tableId) == null) { try { create(config, connection, tableId, fieldsMetadata); } catch (SQLException sqle) { log.warn("Create failed, will attempt amend if table already exists", sqle); try { - TableDefinition newDefn = tableDefns.refresh(connection, tableId); + TableDefinition newDefn = tableDefinitions.refresh(connection, tableId); if (newDefn == null) { throw sqle; } @@ -87,30 +75,18 @@ public boolean createOrAmendIfNecessary( return amendIfNecessary(config, connection, tableId, fieldsMetadata, config.getMaxRetries()); } - /** - * Get the definition for the table with the given ID. This returns a cached definition if - * there is one; otherwise, it reads the definition from the database - * - * @param connection the connection that may be used to fetch the table definition if not - * already known; may not be null - * @param tableId the ID of the table; may not be null - * @return the table definition; or null if the table does not exist - * @throws SQLException if there is an error getting the definition from the database - */ + public TableDefinition tableDefinition( Connection connection, TableId tableId ) throws SQLException { - TableDefinition defn = tableDefns.get(connection, tableId); - if (defn != null) { - return defn; + TableDefinition tableDefinition = tableDefinitions.get(connection, tableId); + if (tableDefinition != null) { + return tableDefinition; } - return tableDefns.refresh(connection, tableId); + return tableDefinitions.refresh(connection, tableId); } - /** - * @throws SQLException if CREATE failed - */ void create( final JdbcSinkConfig config, final Connection connection, @@ -124,13 +100,9 @@ void create( } String sql = dbDialect.buildCreateTableStatement(tableId, fieldsMetadata.allFields.values()); log.info("Creating table with sql: {}", sql); - dbDialect.applyDdlStatements(connection, Collections.singletonList(sql)); + dbDialect.executeSchemaChangeStatements(connection, Collections.singletonList(sql)); } - /** - * @return whether an ALTER was successfully performed - * @throws SQLException if ALTER was deemed necessary but failed - */ boolean amendIfNecessary( final JdbcSinkConfig config, final Connection connection, @@ -138,16 +110,15 @@ boolean amendIfNecessary( final FieldsMetadata fieldsMetadata, final int maxRetries ) throws SQLException, TableAlterOrCreateException { - final TableDefinition tableDefn = tableDefns.get(connection, tableId); + final TableDefinition tableDefn = tableDefinitions.get(connection, tableId); final Set missingFields = missingFields( - fieldsMetadata.allFields.values(), - tableDefn.columnNames() + fieldsMetadata.allFields.values(), + tableDefn.columnNames() ); if (missingFields.isEmpty()) { return false; } - // At this point there are missing fields TableType type = tableDefn.type(); switch (type) { case TABLE: @@ -197,7 +168,7 @@ boolean amendIfNecessary( amendTableQueries ); try { - dbDialect.applyDdlStatements(connection, amendTableQueries); + dbDialect.executeSchemaChangeStatements(connection, amendTableQueries); } catch (SQLException sqle) { if (maxRetries <= 0) { throw new TableAlterOrCreateException( @@ -211,7 +182,7 @@ boolean amendIfNecessary( ); } log.warn("Amend failed, re-attempting", sqle); - tableDefns.refresh(connection, tableId); + tableDefinitions.refresh(connection, tableId); // Perhaps there was a race with other tasks to add the columns return amendIfNecessary( config, @@ -222,7 +193,7 @@ boolean amendIfNecessary( ); } - tableDefns.refresh(connection, tableId); + tableDefinitions.refresh(connection, tableId); return true; } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcSinkConfig.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcSinkConfig.java similarity index 70% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcSinkConfig.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcSinkConfig.java index 398537a42..b8655e888 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcSinkConfig.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcSinkConfig.java @@ -17,12 +17,6 @@ package org.apache.rocketmq.connect.jdbc.sink; import io.openmessaging.KeyValue; -import org.apache.rocketmq.connect.jdbc.config.AbstractConfig; -import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; -import org.apache.rocketmq.connect.jdbc.exception.ConfigException; -import org.apache.rocketmq.connect.jdbc.schema.table.TableId; -import org.apache.rocketmq.connect.jdbc.util.TableType; - import java.time.ZoneId; import java.util.EnumSet; import java.util.HashSet; @@ -31,6 +25,11 @@ import java.util.Set; import java.util.TimeZone; import java.util.stream.Collectors; +import org.apache.rocketmq.connect.jdbc.config.AbstractConfig; +import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; +import org.apache.rocketmq.connect.jdbc.exception.ConfigException; +import org.apache.rocketmq.connect.jdbc.schema.table.TableId; +import org.apache.rocketmq.connect.jdbc.util.TableType; /** * jdbc sink config @@ -52,94 +51,31 @@ public enum PrimaryKeyMode { public static final String TABLE_NAME_FORMAT = "table.name.format"; public static final String TABLE_NAME_FORMAT_DEFAULT = "${topic}"; - private static final String TABLE_NAME_FORMAT_DISPLAY = "Table Name Format"; - /** * table name from header */ public static final String TABLE_NAME_FROM_HEADER = "table.name.from.header"; - private static final boolean TABLE_NAME_FROM_HEADER_DEFAULT = Boolean.FALSE; - private static final String TABLE_NAME_FROM_HEADER_DISPLAY = "Table from header"; - /** * max retries */ public static final String MAX_RETRIES = "max.retries"; private static final int MAX_RETRIES_DEFAULT = 10; - private static final String MAX_RETRIES_DOC = - "The maximum number of times to retry on errors before failing the task."; - public static final String RETRY_BACKOFF_MS = "retry.backoff.ms"; private static final int RETRY_BACKOFF_MS_DEFAULT = 3000; - public static final String BATCH_SIZE = "batch.size"; private static final int BATCH_SIZE_DEFAULT = 100; - - public static final String DELETE_ENABLED = "delete.enabled"; private static final boolean DELETE_ENABLED_DEFAULT = false; - - public static final String AUTO_CREATE = "auto.create"; private static final boolean AUTO_CREATE_DEFAULT = false; - public static final String AUTO_EVOLVE = "auto.evolve"; private static final boolean AUTO_EVOLVE_DEFAULT = false; - private static final String AUTO_EVOLVE_DOC = - "Whether to automatically add columns in the table schema when found to be missing relative " - + "to the record schema by issuing ``ALTER``."; - public static final String INSERT_MODE = "insert.mode"; private static final String INSERT_MODE_DEFAULT = "insert"; - private static final String INSERT_MODE_DOC = - "The insertion mode to use. Supported modes are:\n" - + "``insert``\n" - + " Use standard SQL ``INSERT`` statements.\n" - + "``upsert``\n" - + " Use the appropriate upsert semantics for the target database if it is supported by " - + "the connector, e.g. ``INSERT OR IGNORE``.\n" - + "``update``\n" - + " Use the appropriate update semantics for the target database if it is supported by " - + "the connector, e.g. ``UPDATE``."; - private static final String INSERT_MODE_DISPLAY = "Insert Mode"; - - public static final String PK_FIELDS = "pk.fields"; - private static final String PK_FIELDS_DEFAULT = ""; - private static final String PK_FIELDS_DOC = - "List of comma-separated primary key field names. The runtime interpretation of this config" - + " depends on the ``pk.mode``:\n" - + "``none``\n" - + " Ignored as no fields are used as primary key in this mode.\n" - + "``record_key``\n" - + " If empty, all fields from the key struct will be used, otherwise used to extract the" - + " desired fields - for primitive key only a single field name must be configured.\n" - + "``record_value``\n" - + " If empty, all fields from the value struct will be used, otherwise used to extract " - + "the desired fields."; - private static final String PK_FIELDS_DISPLAY = "Primary Key Fields"; - public static final String PK_MODE = "pk.mode"; private static final String PK_MODE_DEFAULT = "none"; - private static final String PK_MODE_DOC = - "The primary key mode, also refer to ``" + PK_FIELDS + "`` documentation for interplay. " - + "Supported modes are:\n" - + "``none``\n" - + " No keys utilized.\n" - + "``record_value``\n" - + " Field(s) from the record value are used, which must be a struct."; - private static final String PK_MODE_DISPLAY = "Primary Key Mode"; - public static final String FIELDS_WHITELIST = "fields.whitelist"; - private static final String FIELDS_WHITELIST_DEFAULT = ""; - private static final String FIELDS_WHITELIST_DOC = - "List of comma-separated record value field names. If empty, all fields from the record " - + "value are utilized, otherwise used to filter to the desired fields.\n" - + "Note that ``" + PK_FIELDS + "`` is applied independently in the context of which field" - + "(s) form the primary key columns in the destination database," - + " while this configuration is applicable for the other columns."; - private static final String FIELDS_WHITELIST_DISPLAY = "Fields Whitelist"; - public static final String DB_TIMEZONE_CONFIG = "db.timezone"; public static final String DB_TIMEZONE_DEFAULT = "UTC"; @@ -281,7 +217,7 @@ public boolean filterWhiteTable(DatabaseDialect dbDialect, TableId tableId) { return true; } for (String tableName : tableWhitelist) { - TableId table = dbDialect.parseToTableId(tableName); + TableId table = dbDialect.parseTableNameToTableId(tableName); if (table.catalogName() != null && table.catalogName().equals(tableId.catalogName())) { return true; } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/BaseSinkConnector.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcSinkConnector.java similarity index 87% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/BaseSinkConnector.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcSinkConnector.java index a7f4f75a9..7da550212 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/BaseSinkConnector.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcSinkConnector.java @@ -18,19 +18,19 @@ package org.apache.rocketmq.connect.jdbc.sink; import io.openmessaging.KeyValue; +import io.openmessaging.connector.api.component.task.Task; import io.openmessaging.connector.api.component.task.sink.SinkConnector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.ArrayList; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * jdbc sink connector */ -public abstract class BaseSinkConnector extends SinkConnector { - private static final Logger log = LoggerFactory.getLogger(BaseSinkConnector.class); +public class JdbcSinkConnector extends SinkConnector { + private static final Logger log = LoggerFactory.getLogger(JdbcSinkConnector.class); private KeyValue connectConfig; @Override @@ -70,4 +70,9 @@ public List taskConfigs(int maxTasks) { } return configs; } + + @Override + public Class taskClass() { + return JdbcSinkTask.class; + } } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/BaseSinkTask.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcSinkTask.java similarity index 88% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/BaseSinkTask.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcSinkTask.java index 2c2a5f550..4d88597c3 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/BaseSinkTask.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcSinkTask.java @@ -17,27 +17,24 @@ package org.apache.rocketmq.connect.jdbc.sink; - import io.openmessaging.KeyValue; import io.openmessaging.connector.api.component.task.sink.SinkTask; import io.openmessaging.connector.api.data.ConnectRecord; import io.openmessaging.connector.api.errors.ConnectException; import io.openmessaging.connector.api.errors.RetriableException; +import java.sql.SQLException; +import java.util.List; import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; +import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialectLoader; import org.apache.rocketmq.connect.jdbc.exception.TableAlterOrCreateException; -import org.apache.rocketmq.connect.jdbc.schema.db.DbStructure; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.SQLException; -import java.util.List; - /** * jdbc sink task */ -public abstract class BaseSinkTask extends SinkTask { - - private static final Logger log = LoggerFactory.getLogger(BaseSinkTask.class); +public class JdbcSinkTask extends SinkTask { + private static final Logger log = LoggerFactory.getLogger(JdbcSinkTask.class); private KeyValue originalConfig; private JdbcSinkConfig config; private DatabaseDialect dialect; @@ -46,7 +43,6 @@ public abstract class BaseSinkTask extends SinkTask { /** * Start the component - * * @param keyValue */ @Override @@ -54,10 +50,9 @@ public void start(KeyValue keyValue) { originalConfig = keyValue; config = new JdbcSinkConfig(keyValue); remainingRetries = config.getMaxRetries(); - this.dialect = newDialect(config); - final DbStructure dbStructure = new DbStructure(dialect); + this.dialect = DatabaseDialectLoader.getDatabaseDialect(config); log.info("Initializing writer using SQL dialect: {}", dialect.getClass().getSimpleName()); - this.jdbcWriter = new JdbcWriter(config, dialect, dbStructure); + this.jdbcWriter = new JdbcWriter(config, dialect); } /** @@ -117,6 +112,4 @@ public void stop() { } } - protected abstract DatabaseDialect newDialect(JdbcSinkConfig config); - } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcWriter.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcWriter.java similarity index 85% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcWriter.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcWriter.java index d82e44127..5fc130076 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcWriter.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/JdbcWriter.java @@ -17,21 +17,19 @@ package org.apache.rocketmq.connect.jdbc.sink; import io.openmessaging.connector.api.data.ConnectRecord; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; import org.apache.rocketmq.connect.jdbc.common.HeaderField; import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; -import org.apache.rocketmq.connect.jdbc.dialect.provider.CachedConnectionProvider; +import org.apache.rocketmq.connect.jdbc.connection.CachedConnectionProvider; import org.apache.rocketmq.connect.jdbc.exception.TableAlterOrCreateException; -import org.apache.rocketmq.connect.jdbc.schema.db.DbStructure; import org.apache.rocketmq.connect.jdbc.schema.table.TableId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - /** * jdbc db updater */ @@ -43,10 +41,10 @@ public class JdbcWriter { private final DbStructure dbStructure; final CachedConnectionProvider cachedConnectionProvider; - public JdbcWriter(final JdbcSinkConfig config, DatabaseDialect dbDialect, DbStructure dbStructure) { + public JdbcWriter(final JdbcSinkConfig config, DatabaseDialect dbDialect) { this.config = config; this.dbDialect = dbDialect; - this.dbStructure = dbStructure; + this.dbStructure = new DbStructure(dbDialect); this.cachedConnectionProvider = connectionProvider( config.getAttempts(), @@ -63,8 +61,7 @@ protected void onConnect(final Connection connection) throws SQLException { }; } - public void write(final Collection records) - throws SQLException, TableAlterOrCreateException { + public void write(final Collection records) throws SQLException, TableAlterOrCreateException { final Connection connection = cachedConnectionProvider.getConnection(); try { final Map bufferByTable = new HashMap<>(); @@ -91,7 +88,12 @@ public void write(final Collection records) connection.commit(); } catch (SQLException | TableAlterOrCreateException e) { log.error("Jdbc writer error {}", e); - connection.rollback(); + try { + connection.rollback(); + } catch (Exception ex) { + // ignore exception + log.warn(ex.getMessage()); + } } } @@ -102,8 +104,8 @@ public void closeQuietly() { TableId destinationTable(ConnectRecord record) { // todo table from header if (config.isTableFromHeader()) { - return dbDialect.parseToTableId(record.getExtensions().getString(HeaderField.SOURCE_TABLE_KEY)); + return dbDialect.parseTableNameToTableId(record.getExtensions().getString(HeaderField.SOURCE_TABLE_KEY)); } - return dbDialect.parseToTableId(record.getSchema().getName()); + return dbDialect.parseTableNameToTableId(record.getSchema().getName()); } } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/RecordValidator.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/RecordValidator.java similarity index 98% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/RecordValidator.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/RecordValidator.java index 805d92fbc..4c2fd5061 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/RecordValidator.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/RecordValidator.java @@ -24,8 +24,7 @@ @FunctionalInterface public interface RecordValidator { - RecordValidator NO_OP = (record) -> { - }; + RecordValidator NO_OP = (record) -> { }; static RecordValidator create(JdbcSinkConfig config) { RecordValidator requiresKey = requiresKey(config); diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/FieldsMetadata.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/FieldsMetadata.java similarity index 98% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/FieldsMetadata.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/FieldsMetadata.java index c12280d2b..4148c2e67 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/FieldsMetadata.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/FieldsMetadata.java @@ -21,8 +21,6 @@ import io.openmessaging.connector.api.data.FieldType; import io.openmessaging.connector.api.data.Schema; import io.openmessaging.connector.api.errors.ConnectException; -import org.apache.rocketmq.connect.jdbc.sink.JdbcSinkConfig; - import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -31,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.rocketmq.connect.jdbc.sink.JdbcSinkConfig; /** @@ -70,13 +69,13 @@ public static FieldsMetadata extract( final SchemaPair schemaPair ) { return extract( - tableName, - pkMode, - configuredPkFields, - fieldsWhitelist, - schemaPair.keySchema, - schemaPair.schema, - schemaPair.extensions + tableName, + pkMode, + configuredPkFields, + fieldsWhitelist, + schemaPair.keySchema, + schemaPair.valueSchema, + schemaPair.extensions ); } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/SchemaPair.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/SchemaPair.java similarity index 86% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/SchemaPair.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/SchemaPair.java index f2196f413..b87072ed2 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/SchemaPair.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/SchemaPair.java @@ -18,7 +18,6 @@ import io.openmessaging.KeyValue; import io.openmessaging.connector.api.data.Schema; - import java.util.Objects; /** @@ -26,18 +25,18 @@ */ public class SchemaPair { public final Schema keySchema; - public final Schema schema; + public final Schema valueSchema; public final KeyValue extensions; public SchemaPair(Schema keySchema, Schema schema) { this.keySchema = keySchema; - this.schema = schema; + this.valueSchema = schema; this.extensions = null; } public SchemaPair(Schema keySchema, Schema schema, KeyValue extensions) { this.keySchema = keySchema; - this.schema = schema; + this.valueSchema = schema; this.extensions = extensions; } @@ -50,16 +49,16 @@ public boolean equals(Object o) { return false; } SchemaPair that = (SchemaPair) o; - return Objects.equals(schema, that.schema); + return Objects.equals(valueSchema, that.valueSchema); } @Override public int hashCode() { - return Objects.hash(schema); + return Objects.hash(valueSchema); } @Override public String toString() { - return String.format("", schema); + return String.format("", valueSchema); } } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/SinkRecordField.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/SinkRecordField.java similarity index 99% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/SinkRecordField.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/SinkRecordField.java index 633ef366e..d11233fdf 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/SinkRecordField.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/metadata/SinkRecordField.java @@ -21,7 +21,6 @@ public class SinkRecordField { - private final Schema schema; private final String name; private final boolean isPrimaryKey; diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceConfig.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceConfig.java similarity index 70% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceConfig.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceConfig.java index 4becb900d..9682cf388 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceConfig.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceConfig.java @@ -17,59 +17,27 @@ package org.apache.rocketmq.connect.jdbc.source; import io.openmessaging.KeyValue; -import org.apache.rocketmq.connect.jdbc.config.AbstractConfig; -import org.apache.rocketmq.connect.jdbc.util.NumericMapping; -import org.apache.rocketmq.connect.jdbc.util.TableType; - import java.time.ZoneId; import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.TimeZone; +import org.apache.rocketmq.connect.jdbc.config.AbstractConfig; +import org.apache.rocketmq.connect.jdbc.util.NumericMapping; +import org.apache.rocketmq.connect.jdbc.util.TableType; /** * jdbc source config */ public class JdbcSourceConfig extends AbstractConfig { - /** - * table load mode - */ - public enum TableLoadMode { - MODE_BULK("bulk"), - MODE_TIMESTAMP("timestamp"), - MODE_INCREMENTING("incrementing"), - MODE_TIMESTAMP_INCREMENTING("timestamp+incrementing"); - private String name; - - TableLoadMode(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - public static TableLoadMode findTableLoadModeByName(String name) { - for (TableLoadMode mode : TableLoadMode.values()) { - if (mode.getName().equals(name)) { - return mode; - } - } - throw new IllegalArgumentException("Unsupports mode " + name); - } - } //source poll interval ms public static final String POLL_INTERVAL_MS_CONFIG = "poll.interval.ms"; - private static final String POLL_INTERVAL_MS_DOC = "Frequency in ms to poll for new data in each table."; public static final int POLL_INTERVAL_MS_DEFAULT = 5000; - // batch max rows + // batch max rows public static final String BATCH_MAX_ROWS_CONFIG = "batch.max.rows"; public static final int BATCH_MAX_ROWS_DEFAULT = 100; @@ -82,7 +50,6 @@ public static TableLoadMode findTableLoadModeByName(String name) { public static final String NUMERIC_MAPPING_DEFAULT = null; // dialect name - public static final String DIALECT_NAME_CONFIG = "dialect.name"; public static final String DIALECT_NAME_DEFAULT = ""; @@ -91,85 +58,44 @@ public static TableLoadMode findTableLoadModeByName(String name) { // incrementing column name public static final String INCREMENTING_COLUMN_NAME_CONFIG = "incrementing.column.name"; - public static final String INCREMENTING_COLUMN_NAME_DEFAULT = ""; // timestamp column name public static final String TIMESTAMP_COLUMN_NAME_CONFIG = "timestamp.column.name"; - public static final String TIMESTAMP_COLUMN_NAME_DEFAULT = ""; - // timestamp initial public static final String TIMESTAMP_INITIAL_CONFIG = "timestamp.initial"; public static final Long TIMESTAMP_INITIAL_DEFAULT = null; public static final long TIMESTAMP_INITIAL_CURRENT = Long.valueOf(-1); - // Metadata Change Monitoring Interval (ms) - public static final String TABLE_POLL_INTERVAL_MS_CONFIG = "table.poll.interval.ms"; - public static final long TABLE_POLL_INTERVAL_MS_DEFAULT = 60 * 1000; - // table white list public static final String TABLE_WHITELIST_CONFIG = "table.whitelist"; - public static final String TABLE_WHITELIST_DEFAULT = ""; // table black list public static final String TABLE_BLACKLIST_CONFIG = "table.blacklist"; public static final String TABLE_BLACKLIST_DEFAULT = ""; public static final String SCHEMA_PATTERN_CONFIG = "schema.pattern"; - private static final String SCHEMA_PATTERN_DOC = - "Schema pattern to fetch table metadata from the database.\n" - + " * ``\"\"`` retrieves those without a schema.\n" - + " * null (default) indicates that the schema name is not used to narrow the search and " - + "that all table metadata is fetched, regardless of the schema."; - private static final String SCHEMA_PATTERN_DISPLAY = "Schema pattern"; + public static final String SCHEMA_PATTERN_DEFAULT = null; public static final String CATALOG_PATTERN_CONFIG = "catalog.pattern"; - private static final String CATALOG_PATTERN_DOC = - "Catalog pattern to fetch table metadata from the database.\n" - + " * ``\"\"`` retrieves those without a catalog \n" - + " * null (default) indicates that the schema name is not used to narrow the search and " - + "that all table metadata is fetched, regardless of the catalog."; - private static final String CATALOG_PATTERN_DISPLAY = "Schema pattern"; + public static final String CATALOG_PATTERN_DEFAULT = null; public static final String QUERY_CONFIG = "query"; - public static final String QUERY_DEFAULT = ""; public static final String TOPIC_PREFIX_CONFIG = "topic.prefix"; - private static final String TOPIC_PREFIX_DOC = - "Prefix to prepend to table names to generate the name of the Kafka topic to publish data " - + "to, or in the case of a custom query, the full name of the topic to publish to."; - private static final String TOPIC_PREFIX_DISPLAY = "Topic Prefix"; - /** * validate non null */ public static final String VALIDATE_NON_NULL_CONFIG = "validate.non.null"; - private static final String VALIDATE_NON_NULL_DOC = - "By default, the JDBC connector will validate that all incrementing and timestamp tables " - + "have NOT NULL set for the columns being used as their ID/timestamp. If the tables don't," - + " JDBC connector will fail to start. Setting this to false will disable these checks."; + public static final boolean VALIDATE_NON_NULL_DEFAULT = true; - private static final String VALIDATE_NON_NULL_DISPLAY = "Validate Non Null"; public static final String TIMESTAMP_DELAY_INTERVAL_MS_CONFIG = "timestamp.delay.interval.ms"; - private static final String TIMESTAMP_DELAY_INTERVAL_MS_DOC = - "How long to wait after a row with certain timestamp appears before we include it in the " - + "result. You may choose to add some delay to allow transactions with earlier timestamp to" - + " complete. The first execution will fetch all available records (i.e. starting at " - + "timestamp 0) until current time minus the delay. Every following execution will get data" - + " from the last time we fetched until current time minus the delay."; - public static final long TIMESTAMP_DELAY_INTERVAL_MS_DEFAULT = 0; - private static final String TIMESTAMP_DELAY_INTERVAL_MS_DISPLAY = "Delay Interval (ms)"; - public static final String DB_TIMEZONE_CONFIG = "db.timezone"; public static final String DB_TIMEZONE_DEFAULT = "UTC"; - private static final String DB_TIMEZONE_CONFIG_DOC = - "Name of the JDBC timezone used in the connector when " - + "querying with time-based criteria. Defaults to UTC."; - private static final String DB_TIMEZONE_CONFIG_DISPLAY = "DB time zone"; public static final String TABLE_TYPE_DEFAULT = "TABLE"; public static final String TABLE_TYPE_CONFIG = "table.types"; diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/BaseSourceConnector.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceConnector.java similarity index 91% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/BaseSourceConnector.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceConnector.java index ac3eb98ad..a98b9826e 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/BaseSourceConnector.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceConnector.java @@ -19,24 +19,23 @@ import com.google.common.collect.Lists; import io.openmessaging.KeyValue; +import io.openmessaging.connector.api.component.task.Task; import io.openmessaging.connector.api.component.task.source.SourceConnector; import io.openmessaging.internal.DefaultKeyValue; +import java.util.List; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.connect.jdbc.util.ConnectorGroupUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; - /** * jdbc source connector */ -public abstract class BaseSourceConnector extends SourceConnector { - private static final Logger log = LoggerFactory.getLogger(BaseSourceConnector.class); +public class JdbcSourceConnector extends SourceConnector { + private static final Logger log = LoggerFactory.getLogger(JdbcSourceConnector.class); private JdbcSourceConfig jdbcSourceConfig; private KeyValue originalConfig; - /** * Should invoke before start the connector. * @@ -95,4 +94,9 @@ public List taskConfigs(int maxTasks) { } return keyValues; } + + @Override + public Class taskClass() { + return JdbcSourceTask.class; + } } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/BaseSourceTask.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceTask.java similarity index 63% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/BaseSourceTask.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceTask.java index 437fda095..6c79e15cb 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/BaseSourceTask.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceTask.java @@ -20,16 +20,6 @@ import io.openmessaging.KeyValue; import io.openmessaging.connector.api.component.task.source.SourceTask; import io.openmessaging.connector.api.data.ConnectRecord; -import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; -import org.apache.rocketmq.connect.jdbc.dialect.provider.CachedConnectionProvider; -import org.apache.rocketmq.connect.jdbc.source.offset.SourceOffsetCompute; -import org.apache.rocketmq.connect.jdbc.source.querier.BulkQuerier; -import org.apache.rocketmq.connect.jdbc.source.querier.Querier; -import org.apache.rocketmq.connect.jdbc.source.querier.TimestampIncrementingQuerier; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.sql.Connection; import java.sql.SQLException; import java.util.ArrayList; @@ -42,18 +32,30 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.connect.jdbc.connection.CachedConnectionProvider; +import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; +import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialectLoader; +import org.apache.rocketmq.connect.jdbc.source.common.IncrementContext; +import org.apache.rocketmq.connect.jdbc.source.common.QueryContext; +import org.apache.rocketmq.connect.jdbc.source.common.QueryMode; +import org.apache.rocketmq.connect.jdbc.source.common.TableLoadMode; +import org.apache.rocketmq.connect.jdbc.source.offset.SourceOffsetCompute; +import org.apache.rocketmq.connect.jdbc.source.querier.BulkQuerier; +import org.apache.rocketmq.connect.jdbc.source.querier.Querier; +import org.apache.rocketmq.connect.jdbc.source.querier.TimestampIncrementingQuerier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * jdbc source task */ -public abstract class BaseSourceTask extends SourceTask { - - private static final Logger log = LoggerFactory.getLogger(BaseSourceTask.class); +public class JdbcSourceTask extends SourceTask { + private static final Logger log = LoggerFactory.getLogger(JdbcSourceTask.class); private static final int CONSECUTIVE_EMPTY_RESULTS_BEFORE_RETURN = 3; private JdbcSourceTaskConfig config; private DatabaseDialect dialect; private CachedConnectionProvider cachedConnectionProvider; - BlockingQueue tableQueue = new LinkedBlockingQueue(); private final AtomicBoolean running = new AtomicBoolean(false); @@ -63,37 +65,21 @@ public List poll() { Map consecutiveEmptyResults = tableQueue.stream().collect(Collectors.toMap(Function.identity(), (q) -> 0)); while (running.get()) { final Querier querier = tableQueue.peek(); - if (!querier.querying()) { - // If not in the middle of an update, wait for next update time - final long nextUpdate = querier.getLastUpdate() + config.getPollIntervalMs(); - final long now = System.currentTimeMillis(); - final long sleepMs = Math.min(nextUpdate - now, 100); - if (sleepMs > 0) { - log.trace("Waiting {} ms to poll {} next", nextUpdate - now, querier.toString()); - try { - Thread.sleep(sleepMs); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - continue; - } - } + if (sleepIfNeed(querier)) + continue; // poll data final List results = new ArrayList<>(); try { log.debug("Checking for next block of results from {}", querier); querier.maybeStartQuery(cachedConnectionProvider); - int batchMaxRows = config.getBatchMaxRows(); boolean hasNext = true; - while (results.size() < batchMaxRows && (hasNext = querier.hasNext())) { + while (results.size() < config.getBatchMaxRows() && (hasNext = querier.hasNext())) { results.add(querier.extractRecord()); } if (!hasNext) { - // the querier to the tail of the queue resetAndRequeueHead(querier); } - if (results.isEmpty()) { consecutiveEmptyResults.compute(querier, (k, v) -> v + 1); log.trace("No updates for {}", querier); @@ -106,7 +92,6 @@ public List poll() { } else { consecutiveEmptyResults.put(querier, 0); } - log.debug("Returning {} records for {}", results.size(), querier.toString()); return results; } catch (SQLException sqle) { @@ -120,7 +105,7 @@ public List poll() { throw t; } } - // Only in case of shutdown + final Querier querier = tableQueue.peek(); if (querier != null) { resetAndRequeueHead(querier); @@ -129,6 +114,30 @@ public List poll() { return null; } + /** + * Sleep if need + * + * @param querier + * @return + */ + private boolean sleepIfNeed(Querier querier) { + if (!querier.querying()) { + final long nextUpdate = querier.getLastUpdate() + config.getPollIntervalMs(); + final long now = System.currentTimeMillis(); + final long sleepMs = Math.min(nextUpdate - now, 100); + if (sleepMs > 0) { + log.trace("Waiting {} ms to poll {} next", nextUpdate - now, querier); + try { + Thread.sleep(sleepMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return true; + } + } + return false; + } + private void resetAndRequeueHead(Querier querier) { log.debug("Resetting querier {}", querier.toString()); tableQueue.poll(); @@ -158,23 +167,20 @@ public void validate(KeyValue config) { public void start(KeyValue props) { // init config config = new JdbcSourceTaskConfig(props); - this.dialect = newDialect(config); - final int maxConnAttempts = config.getAttempts(); - final long retryBackoff = config.getBackoffMs(); - cachedConnectionProvider = connectionProvider(maxConnAttempts, retryBackoff); + this.dialect = DatabaseDialectLoader.getDatabaseDialect(config); + cachedConnectionProvider = connectionProvider(config.getAttempts(), config.getBackoffMs()); log.info("Using JDBC dialect {}", dialect.name()); - // compute table offset Map> offsetValues = SourceOffsetCompute.initOffset(config, sourceTaskContext, dialect, cachedConnectionProvider); for (String tableOrQuery : offsetValues.keySet()) { this.buildAndAddQuerier( - JdbcSourceConfig.TableLoadMode.findTableLoadModeByName(this.config.getMode()), - this.config.getQuerySuffix(), - this.config.getIncrementingColumnName(), - this.config.getTimestampColumnNames(), - this.config.getTimestampDelayIntervalMs(), - this.config.getTimeZone(), tableOrQuery, - offsetValues.get(tableOrQuery) + TableLoadMode.findTableLoadModeByName(this.config.getMode()), + this.config.getQuerySuffix(), + this.config.getIncrementingColumnName(), + this.config.getTimestampColumnNames(), + this.config.getTimestampDelayIntervalMs(), + this.config.getTimeZone(), tableOrQuery, + offsetValues.get(tableOrQuery) ); } running.set(true); @@ -193,73 +199,87 @@ public void start(KeyValue props) { * @param tableOrQuery * @param offset */ - private void buildAndAddQuerier(JdbcSourceConfig.TableLoadMode loadMode, String querySuffix, String incrementingColumn, List timestampColumns, Long timestampDelayInterval, TimeZone timeZone, String tableOrQuery, Map offset) { + private void buildAndAddQuerier(TableLoadMode loadMode, String querySuffix, String incrementingColumn, + List timestampColumns, Long timestampDelayInterval, TimeZone timeZone, String tableOrQuery, + Map offset) { String topicPrefix = config.getTopicPrefix(); - Querier.QueryMode queryMode = !StringUtils.isEmpty(config.getQuery()) ? Querier.QueryMode.QUERY : Querier.QueryMode.TABLE; - Querier querier = null; + QueryMode queryMode = !StringUtils.isEmpty(config.getQuery()) ? QueryMode.QUERY : QueryMode.TABLE; + Querier querier; switch (loadMode) { case MODE_BULK: querier = new BulkQuerier( - dialect, - queryMode, - tableOrQuery, - topicPrefix, - querySuffix, - this.config.getOffsetSuffix() + dialect, + getContext(querySuffix, tableOrQuery, topicPrefix, queryMode) ); tableQueue.add(querier); break; case MODE_INCREMENTING: querier = new TimestampIncrementingQuerier( - dialect, - queryMode, - tableOrQuery, - topicPrefix, - null, - incrementingColumn, - offset, - timestampDelayInterval, - timeZone, - querySuffix, - this.config.getOffsetSuffix() + dialect, + this.getIncrementContext(querySuffix, tableOrQuery, topicPrefix, queryMode, null, incrementingColumn, offset, timestampDelayInterval, timeZone) ); tableQueue.add(querier); break; case MODE_TIMESTAMP: querier = new TimestampIncrementingQuerier( - dialect, - queryMode, - tableOrQuery, - topicPrefix, - timestampColumns, - null, - offset, - timestampDelayInterval, - timeZone, - querySuffix, - this.config.getOffsetSuffix() + dialect, + this.getIncrementContext(querySuffix, tableOrQuery, topicPrefix, queryMode, timestampColumns, null, offset, timestampDelayInterval, timeZone) ); tableQueue.add(querier); break; case MODE_TIMESTAMP_INCREMENTING: querier = new TimestampIncrementingQuerier( - dialect, - queryMode, - tableOrQuery, - topicPrefix, - timestampColumns, - incrementingColumn, - offset, - timestampDelayInterval, - timeZone, - querySuffix, - this.config.getOffsetSuffix() + dialect, + this.getIncrementContext(querySuffix, tableOrQuery, topicPrefix, queryMode, timestampColumns, incrementingColumn, offset, timestampDelayInterval, timeZone) ); tableQueue.add(querier); break; } } + // Common context + private QueryContext getContext(String querySuffix, String tableOrQuery, String topicPrefix, QueryMode queryMode) { + QueryContext context = new QueryContext( + queryMode, + queryMode == QueryMode.TABLE ? dialect.parseTableNameToTableId(tableOrQuery) : null, + queryMode == QueryMode.QUERY ? tableOrQuery : null, + topicPrefix, + this.config.getOffsetSuffix(), + querySuffix, + config.getBatchMaxRows() + ); + return context; + } + + // Increment context + private IncrementContext getIncrementContext( + String querySuffix, + String tableOrQuery, + String topicPrefix, + QueryMode queryMode, + List timestampColumnNames, + String incrementingColumnName, + Map offsetMap, + Long timestampDelay, + TimeZone timeZone + + ) { + IncrementContext context = new IncrementContext( + queryMode, + queryMode == QueryMode.TABLE ? dialect.parseTableNameToTableId(tableOrQuery) : null, + queryMode == QueryMode.QUERY ? tableOrQuery : null, + topicPrefix, + this.config.getOffsetSuffix(), + querySuffix, + config.getBatchMaxRows(), + timestampColumnNames != null ? timestampColumnNames : Collections.emptyList(), + incrementingColumnName, + offsetMap, + timestampDelay, + timeZone + ); + return context; + } protected CachedConnectionProvider connectionProvider(int maxConnAttempts, long retryBackoff) { return new CachedConnectionProvider(dialect, maxConnAttempts, retryBackoff) { @@ -299,6 +319,4 @@ protected void closeResources() { } } } - - protected abstract DatabaseDialect newDialect(JdbcSourceTaskConfig config); } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceTaskConfig.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceTaskConfig.java similarity index 99% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceTaskConfig.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceTaskConfig.java index ea779dfc5..5e6c49bf1 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceTaskConfig.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcSourceTaskConfig.java @@ -17,7 +17,6 @@ package org.apache.rocketmq.connect.jdbc.source; import io.openmessaging.KeyValue; - import java.util.List; diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingCriteria.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingCriteria.java similarity index 89% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingCriteria.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingCriteria.java index 22aeb4cbf..210b03acb 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingCriteria.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingCriteria.java @@ -20,13 +20,6 @@ import io.openmessaging.connector.api.data.Schema; import io.openmessaging.connector.api.data.Struct; import io.openmessaging.connector.api.errors.ConnectException; -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; -import org.apache.rocketmq.connect.jdbc.source.offset.TimestampIncrementingOffset; -import org.apache.rocketmq.connect.jdbc.util.DateTimeUtils; -import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.math.BigDecimal; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -35,6 +28,12 @@ import java.util.List; import java.util.TimeZone; import java.util.stream.Collectors; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; +import org.apache.rocketmq.connect.jdbc.source.offset.TimestampIncrementingOffset; +import org.apache.rocketmq.connect.jdbc.util.DateTimeUtils; +import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TimestampIncrementingCriteria { @@ -42,29 +41,8 @@ public class TimestampIncrementingCriteria { * The values that can be used in a statement's WHERE clause. */ public interface CriteriaValues { - - /** - * Get the beginning of the time period. - * - * @return the beginning timestamp; may be null - * @throws SQLException if there is a problem accessing the value - */ Timestamp beginTimestampValue() throws SQLException; - - /** - * Get the end of the time period. - * - * @return the ending timestamp; never null - * @throws SQLException if there is a problem accessing the value - */ Timestamp endTimestampValue(Timestamp beginTime) throws SQLException; - - /** - * Get the last incremented value seen. - * - * @return the last incremented value from one of the rows - * @throws SQLException if there is a problem accessing the value - */ Long lastIncrementedValue() throws SQLException; } @@ -94,11 +72,6 @@ protected boolean hasIncrementedColumn() { return incrementingColumn != null; } - /** - * Build the WHERE clause for the columns used in this criteria. - * - * @param builder the string builder to which the WHERE clause should be appended; never null - */ public void whereClause(ExpressionBuilder builder) { if (hasTimestampColumns() && hasIncrementedColumn()) { timestampIncrementingWhereClause(builder); @@ -109,14 +82,6 @@ public void whereClause(ExpressionBuilder builder) { } } - /** - * Set the query parameters on the prepared statement whose WHERE clause was generated with the - * previous call to {@link #whereClause(ExpressionBuilder)}. - * - * @param stmt the prepared statement; never null - * @param values the values that can be used in the criteria parameters; never null - * @throws SQLException if there is a problem using the prepared statement - */ public void setQueryParameters( PreparedStatement stmt, CriteriaValues values @@ -306,8 +271,6 @@ protected void timestampIncrementingWhereClause(ExpressionBuilder builder) { builder.append(" ASC"); } - // where id > ? order by id asc - // where id > ? and id < ? order by id asc protected void incrementingWhereClause(ExpressionBuilder builder) { builder.append(" WHERE "); builder.append(incrementingColumn); diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/metadata/ColumnMapping.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/ColumnMapping.java similarity index 66% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/metadata/ColumnMapping.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/ColumnMapping.java index 6c1804065..0a528a4b4 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/metadata/ColumnMapping.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/ColumnMapping.java @@ -14,63 +14,44 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.connect.jdbc.source.metadata; +package org.apache.rocketmq.connect.jdbc.source.common; import io.openmessaging.connector.api.data.Field; -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; - -import java.sql.ResultSet; import java.util.Objects; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; /** * column mapping */ public class ColumnMapping { - private final Field field; - private final ColumnDefinition columnDefn; + private final ColumnDefinition columnDefinition; private final int columnNumber; private final int hash; public ColumnMapping( - ColumnDefinition columnDefn, - int columnNumber, - Field field + ColumnDefinition columnDefinition, + int columnNumber, + Field field ) { - assert columnDefn != null; + assert columnDefinition != null; assert field != null; assert columnNumber > 0; - this.columnDefn = columnDefn; + this.columnDefinition = columnDefinition; this.field = field; this.columnNumber = columnNumber; - this.hash = Objects.hash(this.columnNumber, this.columnDefn, this.field); + this.hash = Objects.hash(this.columnNumber, this.columnDefinition, this.field); } - /** - * Get this mapping's {@link Field}. - * - * @return the field; never null - */ public Field field() { return field; } - /** - * Get this mapping's {@link ColumnDefinition result set column definition}. - * - * @return the column definition; never null - */ - public ColumnDefinition columnDefn() { - return columnDefn; + public ColumnDefinition columnDefinition() { + return columnDefinition; } - /** - * Get the 1-based number of the column within the result set. This can be used to access the - * corresponding value from the {@link ResultSet}. - * - * @return the column number within the {@link ResultSet}; always positive - */ public int columnNumber() { return columnNumber; } @@ -88,13 +69,13 @@ public boolean equals(Object obj) { if (obj instanceof ColumnMapping) { ColumnMapping that = (ColumnMapping) obj; return this.columnNumber == that.columnNumber && Objects.equals( - this.columnDefn, that.columnDefn) && Objects.equals(this.field, that.field); + this.columnDefinition, that.columnDefinition) && Objects.equals(this.field, that.field); } return false; } @Override public String toString() { - return field.getName() + " (col=" + columnNumber + ", " + columnDefn + ")"; + return field.getName() + " (col=" + columnNumber + ", " + columnDefinition + ")"; } } diff --git a/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/IncrementContext.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/IncrementContext.java new file mode 100644 index 000000000..89cedc82d --- /dev/null +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/IncrementContext.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.connect.jdbc.source.common; + +import java.util.List; +import java.util.Map; +import java.util.TimeZone; +import org.apache.rocketmq.connect.jdbc.schema.table.TableId; + +public class IncrementContext extends QueryContext { + public IncrementContext( + QueryMode mode, + TableId tableId, + String querySql, + String topicPrefix, + String offsetSuffix, + String querySuffix, + int batchMaxSize, + List timestampColumnNames, + String incrementingColumnName, + Map offsetMap, + Long timestampDelay, + TimeZone timeZone + ) { + super(mode, tableId, querySql, topicPrefix, offsetSuffix, querySuffix, batchMaxSize); + this.timestampColumnNames = timestampColumnNames; + this.incrementingColumnName = incrementingColumnName; + this.offsetMap = offsetMap; + this.timestampDelay = timestampDelay; + this.timeZone = timeZone; + } + + private List timestampColumnNames; + private String incrementingColumnName; + private Map offsetMap; + private Long timestampDelay; + private TimeZone timeZone; + + public List getTimestampColumnNames() { + return timestampColumnNames; + } + + public void setTimestampColumnNames(List timestampColumnNames) { + this.timestampColumnNames = timestampColumnNames; + } + + public String getIncrementingColumnName() { + return incrementingColumnName; + } + + public void setIncrementingColumnName(String incrementingColumnName) { + this.incrementingColumnName = incrementingColumnName; + } + + public Map getOffsetMap() { + return offsetMap; + } + + public void setOffsetMap(Map offsetMap) { + this.offsetMap = offsetMap; + } + + public Long getTimestampDelay() { + return timestampDelay; + } + + public void setTimestampDelay(Long timestampDelay) { + this.timestampDelay = timestampDelay; + } + + public TimeZone getTimeZone() { + return timeZone; + } + + public void setTimeZone(TimeZone timeZone) { + this.timeZone = timeZone; + } +} diff --git a/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/QueryContext.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/QueryContext.java new file mode 100644 index 000000000..203e9b6df --- /dev/null +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/QueryContext.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.connect.jdbc.source.common; + +import org.apache.rocketmq.connect.jdbc.schema.table.TableId; + +public class QueryContext { + + public QueryContext() { + } + + public QueryContext(QueryMode mode, TableId tableId, String querySql, String topicPrefix, String offsetSuffix, + String querySuffix, int batchMaxSize) { + this.mode = mode; + this.tableId = tableId; + this.querySql = querySql; + this.topicPrefix = topicPrefix; + this.offsetSuffix = offsetSuffix; + this.querySuffix = querySuffix; + this.batchMaxSize = batchMaxSize; + this.lastUpdate = 0; + } + + private QueryMode mode; + private TableId tableId; + private String querySql; + private String topicPrefix; + private String offsetSuffix; + private String querySuffix; + private int batchMaxSize; + private long lastUpdate = 0; + + public QueryMode getMode() { + return mode; + } + + public void setMode(QueryMode mode) { + this.mode = mode; + } + + public TableId getTableId() { + return tableId; + } + + public void setTableId(TableId tableId) { + this.tableId = tableId; + } + + public String getQuerySql() { + return querySql; + } + + public void setQuerySql(String querySql) { + this.querySql = querySql; + } + + public String getTopicPrefix() { + return topicPrefix; + } + + public void setTopicPrefix(String topicPrefix) { + this.topicPrefix = topicPrefix; + } + + public String getOffsetSuffix() { + return offsetSuffix; + } + + public void setOffsetSuffix(String offsetSuffix) { + this.offsetSuffix = offsetSuffix; + } + + public String getQuerySuffix() { + return querySuffix; + } + + public void setQuerySuffix(String querySuffix) { + this.querySuffix = querySuffix; + } + + public int getBatchMaxSize() { + return batchMaxSize; + } + + public void setBatchMaxSize(int batchMaxSize) { + this.batchMaxSize = batchMaxSize; + } + + public long getLastUpdate() { + return lastUpdate; + } + + public void setLastUpdate(long lastUpdate) { + this.lastUpdate = lastUpdate; + } +} diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/java/org/apache/rocketmq/connect/jdbc/mysql/source/MysqlJdbcSourceConnector.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/QueryMode.java similarity index 66% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/java/org/apache/rocketmq/connect/jdbc/mysql/source/MysqlJdbcSourceConnector.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/QueryMode.java index 0732509b6..3f76387d7 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/java/org/apache/rocketmq/connect/jdbc/mysql/source/MysqlJdbcSourceConnector.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/QueryMode.java @@ -14,17 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.connect.jdbc.mysql.source; -import io.openmessaging.connector.api.component.task.Task; -import org.apache.rocketmq.connect.jdbc.source.BaseSourceConnector; +package org.apache.rocketmq.connect.jdbc.source.common; -/** - * mysql jdbc source connector - */ -public class MysqlJdbcSourceConnector extends BaseSourceConnector { - @Override - public Class taskClass() { - return MysqlJdbcSourceTask.class; - } -} +public enum QueryMode { + TABLE, + QUERY +} \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/SchemaMapping.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/SchemaMapping.java new file mode 100644 index 000000000..983208559 --- /dev/null +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/SchemaMapping.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.connect.jdbc.source.common; + +import io.openmessaging.connector.api.data.Field; +import io.openmessaging.connector.api.data.Schema; +import io.openmessaging.connector.api.data.SchemaBuilder; +import java.sql.Connection; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.rocketmq.connect.jdbc.converter.JdbcColumnConverter; +import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; +import org.apache.rocketmq.connect.jdbc.schema.table.TableId; + +/** + * schema mapping + */ +public final class SchemaMapping { + + private final Schema schema; + private final Map columnMappings; + + private SchemaMapping( + Schema schema, + Map columnMappings + ) { + assert schema != null; + assert columnMappings != null; + assert !columnMappings.isEmpty(); + this.schema = schema; + this.columnMappings = columnMappings; + } + + public static SchemaMapping create( + Connection conn, + TableId tableId, + ResultSetMetaData metadata, + DatabaseDialect dialect + ) throws SQLException { + String schemaName = tableId != null ? tableId.tableName() : null; + // Get columns + Map columnDefinitionMap = dialect.describeColumns(conn, tableId, metadata); + Map columnMappingMap = new LinkedHashMap<>(); + + // Build schema + SchemaBuilder builder = SchemaBuilder.struct().name(schemaName); + JdbcColumnConverter jdbcColumnConverter = dialect.createJdbcColumnConverter(); + AtomicInteger columnNumber = new AtomicInteger(0); + for (ColumnDefinition columnDefinition : columnDefinitionMap.values()) { + String fieldName = jdbcColumnConverter.convertToConnectFieldSchema(columnDefinition, builder); + if (fieldName == null) { + continue; + } + Field field = builder.field(fieldName); + ColumnMapping columnMapping = new ColumnMapping(columnDefinition, columnNumber.incrementAndGet(), field); + columnMappingMap.put(fieldName, columnMapping); + } + return new SchemaMapping(builder.build(), columnMappingMap); + } + + /** + * schema + * + * @return + */ + public Schema schema() { + return schema; + } + + public Collection columnMappings() { + return columnMappings.values(); + } + + @Override + public String toString() { + return "Mapping for " + schema.getName(); + } +} diff --git a/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/TableLoadMode.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/TableLoadMode.java new file mode 100644 index 000000000..7bac84690 --- /dev/null +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/TableLoadMode.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.connect.jdbc.source.common; + +/** + * table load mode + */ +public enum TableLoadMode { + MODE_BULK("bulk"), + MODE_TIMESTAMP("timestamp"), + MODE_INCREMENTING("incrementing"), + MODE_TIMESTAMP_INCREMENTING("timestamp+incrementing"); + private String name; + + TableLoadMode(String name) { + this.name = name; + } + + public static TableLoadMode findTableLoadModeByName(String name) { + for (TableLoadMode mode : TableLoadMode.values()) { + if (mode.getName().equals(name)) { + return mode; + } + } + throw new IllegalArgumentException("Unsupports mode " + name); + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/offset/SourceOffsetCompute.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/offset/SourceOffsetCompute.java similarity index 81% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/offset/SourceOffsetCompute.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/offset/SourceOffsetCompute.java index 0598542ec..d7f99f79f 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/offset/SourceOffsetCompute.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/offset/SourceOffsetCompute.java @@ -16,27 +16,11 @@ */ package org.apache.rocketmq.connect.jdbc.source.offset; - import com.google.common.collect.Maps; import io.openmessaging.connector.api.component.task.source.SourceTaskContext; import io.openmessaging.connector.api.data.RecordOffset; import io.openmessaging.connector.api.data.RecordPartition; import io.openmessaging.connector.api.errors.ConnectException; -import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.connect.jdbc.common.JdbcSourceConfigConstants; -import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; -import org.apache.rocketmq.connect.jdbc.dialect.provider.CachedConnectionProvider; -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; -import org.apache.rocketmq.connect.jdbc.schema.table.TableId; -import org.apache.rocketmq.connect.jdbc.source.JdbcSourceConfig; -import org.apache.rocketmq.connect.jdbc.source.JdbcSourceTaskConfig; -import org.apache.rocketmq.connect.jdbc.source.querier.Querier; -import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; -import org.apache.rocketmq.connect.jdbc.util.QuoteMethod; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.sql.Connection; import java.sql.SQLException; import java.util.Calendar; @@ -49,6 +33,22 @@ import java.util.Objects; import java.util.Set; import java.util.TimeZone; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.connect.jdbc.common.JdbcSourceConfigConstants; +import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; +import org.apache.rocketmq.connect.jdbc.connection.CachedConnectionProvider; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; +import org.apache.rocketmq.connect.jdbc.schema.table.TableId; +import org.apache.rocketmq.connect.jdbc.source.JdbcSourceTaskConfig; +import org.apache.rocketmq.connect.jdbc.source.common.QueryMode; +import org.apache.rocketmq.connect.jdbc.source.common.TableLoadMode; +import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; +import org.apache.rocketmq.connect.jdbc.util.QuoteMethod; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.rocketmq.connect.jdbc.source.JdbcSourceConfig.TIMESTAMP_INITIAL_CURRENT; /** * offset compute utils @@ -105,8 +105,8 @@ public static Map> initOffset( List tables = config.getTables(); String query = config.getQuery(); - JdbcSourceConfig.TableLoadMode mode = JdbcSourceConfig.TableLoadMode.findTableLoadModeByName(config.getMode()); - Querier.QueryMode queryMode = !StringUtils.isEmpty(query) ? Querier.QueryMode.QUERY : Querier.QueryMode.TABLE; + TableLoadMode mode = TableLoadMode.findTableLoadModeByName(config.getMode()); + QueryMode queryMode = !StringUtils.isEmpty(query) ? QueryMode.QUERY : QueryMode.TABLE; // step 1 -——-- compute partitions Map partitionsByTableFqn = buildTablePartitions(mode, queryMode, tables, dialect, config.getOffsetSuffix(), config.getTopicPrefix()); @@ -116,7 +116,7 @@ public static Map> initOffset( offsets = context.offsetStorageReader().readOffsets(partitionsByTableFqn.values()); } // step 3 ----- compute offset init value - List tablesOrQuery = queryMode == Querier.QueryMode.QUERY ? Collections.singletonList(query) : tables; + List tablesOrQuery = queryMode == QueryMode.QUERY ? Collections.singletonList(query) : tables; return initOffsetValues( cachedConnectionProvider, dialect, queryMode, @@ -128,13 +128,13 @@ public static Map> initOffset( } private static Map> initOffsetValues( - CachedConnectionProvider cachedConnectionProvider, - DatabaseDialect dialect, - Querier.QueryMode queryMode, - Map partitionsByTableFqn, - Map offsets, - JdbcSourceTaskConfig config, - List tablesOrQuery) { + CachedConnectionProvider cachedConnectionProvider, + DatabaseDialect dialect, + QueryMode queryMode, + Map partitionsByTableFqn, + Map offsets, + JdbcSourceTaskConfig config, + List tablesOrQuery) { Map> offsetsValues = Maps.newHashMap(); @@ -219,20 +219,20 @@ private static void validateNonNullable( } finally { conn.setAutoCommit(autoCommit); } - if ((incrementalMode.equals(JdbcSourceConfig.TableLoadMode.MODE_INCREMENTING.getName()) - || incrementalMode.equals(JdbcSourceConfig.TableLoadMode.MODE_TIMESTAMP_INCREMENTING.getName())) - && incrementingOptional) { + if ((incrementalMode.equals(TableLoadMode.MODE_INCREMENTING.getName()) + || incrementalMode.equals(TableLoadMode.MODE_TIMESTAMP_INCREMENTING.getName())) + && incrementingOptional) { throw new ConnectException("Cannot make incremental queries using incrementing column " - + incrementingColumn + " on " + table + " because this column " - + "is nullable."); + + incrementingColumn + " on " + table + " because this column " + + "is nullable."); } - if ((incrementalMode.equals(JdbcSourceConfig.TableLoadMode.MODE_TIMESTAMP.getName()) - || incrementalMode.equals(JdbcSourceConfig.TableLoadMode.MODE_TIMESTAMP_INCREMENTING.getName())) - && !atLeastOneTimestampNotOptional) { + if ((incrementalMode.equals(TableLoadMode.MODE_TIMESTAMP.getName()) + || incrementalMode.equals(TableLoadMode.MODE_TIMESTAMP_INCREMENTING.getName())) + && !atLeastOneTimestampNotOptional) { throw new ConnectException("Cannot make incremental queries using timestamp columns " - + timestampColumns + " on " + table + " because all of these " - + "columns " - + "nullable."); + + timestampColumns + " on " + table + " because all of these " + + "columns " + + "nullable."); } } catch (SQLException e) { throw new ConnectException("Failed trying to validate that columns used for offsets are NOT" @@ -242,14 +242,14 @@ private static void validateNonNullable( private static Map computeInitialOffset( - CachedConnectionProvider cachedConnectionProvider, - DatabaseDialect dialect, - Querier.QueryMode queryMode, - String tableOrQuery, - Map partitionOffset, - TimeZone timezone, - Long timestampInitial, - List timestampColumns + CachedConnectionProvider cachedConnectionProvider, + DatabaseDialect dialect, + QueryMode queryMode, + String tableOrQuery, + Map partitionOffset, + TimeZone timezone, + Long timestampInitial, + List timestampColumns ) { if (Objects.nonNull(partitionOffset)) { return partitionOffset; @@ -258,7 +258,7 @@ private static Map computeInitialOffset( // no offsets found if (timestampInitial != null) { // start at the specified timestamp - if (timestampInitial == JdbcSourceConfig.TIMESTAMP_INITIAL_CURRENT) { + if (timestampInitial == TIMESTAMP_INITIAL_CURRENT) { // use the current time try { final Connection con = cachedConnectionProvider.getConnection(); @@ -273,7 +273,7 @@ private static Map computeInitialOffset( log.info("No offsets found for '{}', so using configured timestamp {}", tableOrQuery, timestampInitial); } else { - if (queryMode != Querier.QueryMode.TABLE || timestampColumns == null || timestampColumns.isEmpty()) { + if (queryMode != QueryMode.TABLE || timestampColumns == null || timestampColumns.isEmpty()) { return initialPartitionOffset; } try { @@ -301,22 +301,22 @@ private static Map computeInitialOffset( * @return */ private static Map buildTablePartitions( - JdbcSourceConfig.TableLoadMode tableLoadMode, - Querier.QueryMode queryMode, - List tables, - DatabaseDialect dialect, - String offsetSuffix, String topicPrefix) { + TableLoadMode tableLoadMode, + QueryMode queryMode, + List tables, + DatabaseDialect dialect, + String offsetSuffix, String topicPrefix) { Map partitionsByTableFqn = new HashMap<>(); - if (tableLoadMode == JdbcSourceConfig.TableLoadMode.MODE_INCREMENTING - || tableLoadMode == JdbcSourceConfig.TableLoadMode.MODE_TIMESTAMP - || tableLoadMode == JdbcSourceConfig.TableLoadMode.MODE_TIMESTAMP_INCREMENTING) { + if (tableLoadMode == TableLoadMode.MODE_INCREMENTING + || tableLoadMode == TableLoadMode.MODE_TIMESTAMP + || tableLoadMode == TableLoadMode.MODE_TIMESTAMP_INCREMENTING) { switch (queryMode) { case TABLE: for (String table : tables) { // Find possible partition maps for different offset protocols // We need to search by all offset protocol partition keys to support compatibility - TableId tableId = dialect.parseToTableId(table); + TableId tableId = dialect.parseTableNameToTableId(table); RecordPartition tablePartition = new RecordPartition(SourceOffsetCompute.sourcePartitions(topicPrefix, tableId, offsetSuffix)); partitionsByTableFqn.put(table, tablePartition); } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/offset/TimestampIncrementingOffset.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/offset/TimestampIncrementingOffset.java similarity index 99% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/offset/TimestampIncrementingOffset.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/offset/TimestampIncrementingOffset.java index 0dab7afde..82a8aafa2 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/offset/TimestampIncrementingOffset.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/offset/TimestampIncrementingOffset.java @@ -16,16 +16,15 @@ */ package org.apache.rocketmq.connect.jdbc.source.offset; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.sql.Timestamp; import java.util.HashMap; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * time stamp incrementing offset + * Timestamp incrementing offset */ public class TimestampIncrementingOffset { private static final Logger log = LoggerFactory.getLogger(TimestampIncrementingOffset.class); diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/BulkQuerier.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/BulkQuerier.java similarity index 73% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/BulkQuerier.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/BulkQuerier.java index 9afc6ede4..fdbef6d98 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/BulkQuerier.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/BulkQuerier.java @@ -22,20 +22,19 @@ import io.openmessaging.connector.api.data.Schema; import io.openmessaging.connector.api.data.Struct; import io.openmessaging.connector.api.errors.ConnectException; -import org.apache.rocketmq.connect.jdbc.common.JdbcSourceConfigConstants; -import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; -import org.apache.rocketmq.connect.jdbc.source.metadata.SchemaMapping; -import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.util.HashMap; import java.util.Map; - +import org.apache.rocketmq.connect.jdbc.common.JdbcSourceConfigConstants; +import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; +import org.apache.rocketmq.connect.jdbc.source.common.ColumnMapping; +import org.apache.rocketmq.connect.jdbc.source.common.QueryContext; +import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * bulk mode @@ -43,35 +42,28 @@ public class BulkQuerier extends Querier { private static final Logger log = LoggerFactory.getLogger(BulkQuerier.class); - public BulkQuerier( - DatabaseDialect dialect, - QueryMode mode, - String name, - String topicPrefix, - String suffix, - String offsetSuffix - ) { - super(dialect, mode, name, topicPrefix, suffix, offsetSuffix); + public BulkQuerier(DatabaseDialect dialect, QueryContext context) { + super(dialect, context); } @Override protected void createPreparedStatement(Connection db) throws SQLException { ExpressionBuilder builder = dialect.expressionBuilder(); - switch (mode) { + switch (context.getMode()) { case TABLE: - dialect.buildSelectTable(builder, tableId); + dialect.buildSelectTable(builder, context.getTableId()); break; case QUERY: - builder.append(query); + builder.append(context.getQuerySql()); break; default: - throw new ConnectException("Unknown mode: " + mode); + throw new ConnectException("Unknown mode: " + context.getMode()); } String queryStr = builder.toString(); recordQuery(queryStr); log.debug("{} prepared SQL query: {}", this, queryStr); - stmt = dialect.createPreparedStatement(db, queryStr); + stmt = dialect.createPreparedStatement(db, queryStr, context.getBatchMaxSize()); } @Override @@ -87,9 +79,10 @@ protected ResultSet executeQuery() throws SQLException { public ConnectRecord extractRecord() throws SQLException { Schema schema = schemaMapping.schema(); Struct payload = new Struct(schema); - for (SchemaMapping.FieldSetter setter : schemaMapping.fieldSetters()) { + for (ColumnMapping columnMapping : schemaMapping.columnMappings()) { try { - setter.setField(payload, resultSet); + Object value = jdbcColumnConverter.convertToConnectFieldValue(resultSet, columnMapping.columnDefinition(), columnMapping.columnNumber()); + payload.put(columnMapping.field(), value); } catch (IOException e) { log.warn("Error mapping fields into Connect record", e); throw new ConnectException(e); @@ -98,30 +91,27 @@ public ConnectRecord extractRecord() throws SQLException { throw new SQLException(e); } } - // TODO: key from primary key? partition? + final String topic; final Map partition = new HashMap<>(); - switch (mode) { + switch (context.getMode()) { case TABLE: - // backwards compatible - String name = tableId.tableName(); - topic = topicPrefix + name; - partition.put(JdbcSourceConfigConstants.TABLE_NAME_KEY(this.offsetSuffix), name); + String name = context.getTableId().tableName(); + topic = context.getTopicPrefix() + name; + partition.put(JdbcSourceConfigConstants.TABLE_NAME_KEY(context.getOffsetSuffix()), name); partition.put("topic", topic); break; case QUERY: - partition.put(JdbcSourceConfigConstants.QUERY_NAME_KEY(this.offsetSuffix), - JdbcSourceConfigConstants.QUERY_NAME_VALUE); - topic = topicPrefix; + partition.put(JdbcSourceConfigConstants.QUERY_NAME_KEY(context.getOffsetSuffix()), + JdbcSourceConfigConstants.QUERY_NAME_VALUE); + topic = context.getTopicPrefix(); partition.put("topic", topic); break; default: - throw new ConnectException("Unexpected query mode: " + mode); + throw new ConnectException("Unexpected query mode: " + context.getMode()); } // build record ConnectRecord record = new ConnectRecord( - // offset partition - // offset partition new RecordPartition(partition), new RecordOffset(new HashMap<>()), System.currentTimeMillis(), @@ -133,8 +123,8 @@ public ConnectRecord extractRecord() throws SQLException { @Override public String toString() { - return "BulkTableQuerier{" + "table='" + tableId + '\'' + ", query='" + query + '\'' - + ", topicPrefix='" + topicPrefix + '\'' + '}'; + return "BulkTableQuerier{" + "table='" + context.getMode() + '\'' + ", query='" + context.getQuerySql() + '\'' + + ", topicPrefix='" + context.getTopicPrefix() + '\'' + '}'; } } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/Querier.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/Querier.java similarity index 71% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/Querier.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/Querier.java index 8052ec2d7..13fa3c64a 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/Querier.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/Querier.java @@ -17,62 +17,37 @@ package org.apache.rocketmq.connect.jdbc.source.querier; import io.openmessaging.connector.api.data.ConnectRecord; -import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; -import org.apache.rocketmq.connect.jdbc.dialect.provider.CachedConnectionProvider; -import org.apache.rocketmq.connect.jdbc.schema.table.TableId; -import org.apache.rocketmq.connect.jdbc.source.metadata.SchemaMapping; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import org.apache.rocketmq.connect.jdbc.converter.JdbcColumnConverter; +import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; +import org.apache.rocketmq.connect.jdbc.connection.CachedConnectionProvider; +import org.apache.rocketmq.connect.jdbc.source.common.QueryContext; +import org.apache.rocketmq.connect.jdbc.source.common.SchemaMapping; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class Querier { - public enum QueryMode { - TABLE, - QUERY - } - private final Logger log = LoggerFactory.getLogger(Querier.class); - protected final DatabaseDialect dialect; - protected final QueryMode mode; - protected final String query; - protected final String topicPrefix; - protected final TableId tableId; - protected final String suffix; - protected String offsetSuffix; - // Mutable state - - protected long lastUpdate; + protected final QueryContext context; protected Connection db; protected PreparedStatement stmt; protected ResultSet resultSet; protected SchemaMapping schemaMapping; + protected JdbcColumnConverter jdbcColumnConverter; private String loggedQueryString; - public Querier( - DatabaseDialect dialect, - QueryMode mode, - String nameOrQuery, - String topicPrefix, - String suffix, - String offsetSuffix - ) { + public Querier(DatabaseDialect dialect, QueryContext context) { this.dialect = dialect; - this.mode = mode; - this.tableId = mode.equals(QueryMode.TABLE) ? dialect.parseToTableId(nameOrQuery) : null; - this.query = mode.equals(QueryMode.QUERY) ? nameOrQuery : null; - this.topicPrefix = topicPrefix; - this.lastUpdate = 0; - this.suffix = suffix; - this.offsetSuffix = offsetSuffix; + this.context = context; + this.jdbcColumnConverter = dialect.createJdbcColumnConverter(); } public long getLastUpdate() { - return lastUpdate; + return context.getLastUpdate(); } public PreparedStatement getOrCreatePreparedStatement(Connection db) throws SQLException { @@ -94,7 +69,7 @@ public void maybeStartQuery(CachedConnectionProvider provider) throws SQLExcepti this.db = provider.getConnection(); stmt = getOrCreatePreparedStatement(db); resultSet = executeQuery(); - schemaMapping = SchemaMapping.create(this.db, tableId, resultSet.getMetaData(), dialect); + schemaMapping = SchemaMapping.create(this.db, context.getTableId(), resultSet.getMetaData(), dialect); } } @@ -110,10 +85,8 @@ public void reset(long now) { closeResultSetQuietly(); closeStatementQuietly(); releaseLocksQuietly(); - // TODO: Can we cache this and quickly check that it's identical for the next query - // instead of constructing from scratch since it's almost always the same schemaMapping = null; - lastUpdate = now; + context.setLastUpdate(now); } private void releaseLocksQuietly() { @@ -157,8 +130,4 @@ protected void recordQuery(String query) { loggedQueryString = query; } } - - public void setOffsetSuffix(String offsetSuffix) { - this.offsetSuffix = offsetSuffix; - } } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/TimestampIncrementingQuerier.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/TimestampIncrementingQuerier.java similarity index 68% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/TimestampIncrementingQuerier.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/TimestampIncrementingQuerier.java index dfb602066..ddee30351 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/TimestampIncrementingQuerier.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/querier/TimestampIncrementingQuerier.java @@ -22,19 +22,6 @@ import io.openmessaging.connector.api.data.Schema; import io.openmessaging.connector.api.data.Struct; import io.openmessaging.connector.api.errors.ConnectException; -import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; -import org.apache.rocketmq.connect.jdbc.dialect.provider.CachedConnectionProvider; -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; -import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; -import org.apache.rocketmq.connect.jdbc.source.TimestampIncrementingCriteria; -import org.apache.rocketmq.connect.jdbc.source.metadata.SchemaMapping; -import org.apache.rocketmq.connect.jdbc.source.offset.SourceOffsetCompute; -import org.apache.rocketmq.connect.jdbc.source.offset.TimestampIncrementingOffset; -import org.apache.rocketmq.connect.jdbc.util.DateTimeUtils; -import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.sql.Connection; import java.sql.ResultSet; @@ -42,10 +29,23 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.TimeZone; +import org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialect; +import org.apache.rocketmq.connect.jdbc.connection.CachedConnectionProvider; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; +import org.apache.rocketmq.connect.jdbc.source.TimestampIncrementingCriteria; +import org.apache.rocketmq.connect.jdbc.source.common.ColumnMapping; +import org.apache.rocketmq.connect.jdbc.source.common.IncrementContext; +import org.apache.rocketmq.connect.jdbc.source.common.SchemaMapping; +import org.apache.rocketmq.connect.jdbc.source.offset.SourceOffsetCompute; +import org.apache.rocketmq.connect.jdbc.source.offset.TimestampIncrementingOffset; +import org.apache.rocketmq.connect.jdbc.util.DateTimeUtils; +import org.apache.rocketmq.connect.jdbc.util.ExpressionBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TimestampIncrementingQuerier extends Querier implements TimestampIncrementingCriteria.CriteriaValues { private static final Logger log = LoggerFactory.getLogger( @@ -59,66 +59,51 @@ public class TimestampIncrementingQuerier extends Querier implements TimestampIn private TimestampIncrementingOffset offset; private TimestampIncrementingCriteria criteria; private Map partition; - private final TimeZone timeZone; + private final TimeZone timeZone; - public TimestampIncrementingQuerier(DatabaseDialect dialect, - QueryMode mode, - String name, - String topicPrefix, - List timestampColumnNames, - String incrementingColumnName, - Map offsetMap, - Long timestampDelay, - TimeZone timeZone, - String suffix, - String offsetSuffix) { - super(dialect, mode, name, topicPrefix, suffix, offsetSuffix); - this.incrementingColumnName = incrementingColumnName; - this.timestampColumnNames = timestampColumnNames != null - ? timestampColumnNames : Collections.emptyList(); - this.timestampDelay = timestampDelay; - this.offset = TimestampIncrementingOffset.fromMap(offsetMap); - + public TimestampIncrementingQuerier(DatabaseDialect dialect, IncrementContext context) { + super(dialect, context); + this.incrementingColumnName = context.getIncrementingColumnName(); + this.timestampColumnNames = context.getTimestampColumnNames(); + this.timestampDelay = context.getTimestampDelay(); + this.offset = TimestampIncrementingOffset.fromMap(context.getOffsetMap()); this.timestampColumns = new ArrayList<>(); for (String timestampColumn : this.timestampColumnNames) { if (timestampColumn != null && !timestampColumn.isEmpty()) { - timestampColumns.add(new ColumnId(tableId, timestampColumn)); + timestampColumns.add(new ColumnId(context.getTableId(), timestampColumn)); } } - - switch (mode) { + switch (context.getMode()) { case TABLE: - partition = SourceOffsetCompute.sourcePartitions(topicPrefix, tableId, this.offsetSuffix); + partition = SourceOffsetCompute.sourcePartitions(context.getTopicPrefix(), context.getTableId(), context.getOffsetSuffix()); break; case QUERY: - partition = SourceOffsetCompute.sourceQueryPartitions(topicPrefix, this.offsetSuffix); + partition = SourceOffsetCompute.sourceQueryPartitions(context.getTopicPrefix(), context.getOffsetSuffix()); break; default: - throw new ConnectException("Unexpected query mode: " + mode); + throw new ConnectException("Unexpected query mode: " + context.getMode()); } - this.timeZone = timeZone; + this.timeZone = context.getTimeZone(); } @Override protected void createPreparedStatement(Connection db) throws SQLException { findDefaultAutoIncrementingColumn(db); - ColumnId incrementingColumn = null; if (incrementingColumnName != null && !incrementingColumnName.isEmpty()) { - incrementingColumn = new ColumnId(tableId, incrementingColumnName); + incrementingColumn = new ColumnId(context.getTableId(), incrementingColumnName); } - ExpressionBuilder builder = dialect.expressionBuilder(); - switch (mode) { + switch (context.getMode()) { case TABLE: - dialect.buildSelectTable(builder, tableId); + dialect.buildSelectTable(builder, context.getTableId()); break; case QUERY: - builder.append(query); + builder.append(context.getQuerySql()); break; default: - throw new ConnectException("Unknown mode encountered when preparing query: " + mode); + throw new ConnectException("Unknown mode encountered when preparing query: " + context.getMode()); } // Append the criteria using the columns ... @@ -128,7 +113,7 @@ protected void createPreparedStatement(Connection db) throws SQLException { String queryString = builder.toString(); recordQuery(queryString); log.debug("{} prepared SQL query: {}", this, queryString); - stmt = dialect.createPreparedStatement(db, queryString); + stmt = dialect.createPreparedStatement(db, queryString, context.getBatchMaxSize()); } @Override @@ -139,7 +124,7 @@ public void maybeStartQuery(CachedConnectionProvider provider) throws SQLExcepti resultSet = executeQuery(); ResultSetMetaData metadata = resultSet.getMetaData(); dialect.validateColumnTypes(metadata, timestampColumns); - schemaMapping = SchemaMapping.create(this.db, tableId, metadata, dialect); + schemaMapping = SchemaMapping.create(this.db, context.getTableId(), metadata, dialect); } } @@ -147,23 +132,22 @@ private void findDefaultAutoIncrementingColumn(Connection db) throws SQLExceptio // Default when unspecified uses an autoincrementing column if (incrementingColumnName != null && incrementingColumnName.isEmpty()) { // Find the first auto-incremented column ... - for (ColumnDefinition defn : dialect.describeColumns( - db, - tableId.catalogName(), - tableId.schemaName(), - tableId.tableName(), - null).values()) { - if (defn.isAutoIncrement()) { - incrementingColumnName = defn.id().name(); + for (ColumnDefinition columnDefinition : dialect.describeColumns( + db, + context.getTableId().catalogName(), + context.getTableId().schemaName(), + context.getTableId().tableName(), + null).values()) { + if (columnDefinition.isAutoIncrement()) { + incrementingColumnName = columnDefinition.id().name(); break; } } } - // If still not found, query the table and use the result set metadata. - // This doesn't work if the table is empty. + if (incrementingColumnName != null && incrementingColumnName.isEmpty()) { - log.debug("Falling back to describe '{}' table by querying {}", tableId, db); - for (ColumnDefinition defn : dialect.describeColumnsByQuerying(db, tableId).values()) { + log.debug("Falling back to describe '{}' table by querying {}", context.getTableId(), db); + for (ColumnDefinition defn : dialect.describeColumnsByQuerying(db, context.getTableId()).values()) { if (defn.isAutoIncrement()) { incrementingColumnName = defn.id().name(); break; @@ -186,9 +170,10 @@ protected ResultSet executeQuery() throws SQLException { public ConnectRecord extractRecord() throws SQLException { Schema schema = schemaMapping.schema(); Struct payload = new Struct(schema); - for (SchemaMapping.FieldSetter setter : schemaMapping.fieldSetters()) { + for (ColumnMapping columnMapping : schemaMapping.columnMappings()) { try { - setter.setField(payload, resultSet); + Object value = jdbcColumnConverter.convertToConnectFieldValue(resultSet, columnMapping.columnDefinition(), columnMapping.columnNumber()); + payload.put(columnMapping.field(), value); } catch (IOException e) { log.warn("Error mapping fields into Connect record", e); throw new ConnectException(e); @@ -197,30 +182,36 @@ public ConnectRecord extractRecord() throws SQLException { throw new SQLException(e); } } - offset = criteria.extractValues(schemaMapping.schema(), payload, offset); + + offset = criteria.extractValues(schema, payload, offset); // build record return new ConnectRecord( - // offset partition - new RecordPartition(partition), - new RecordOffset(offset.toMap()), - System.currentTimeMillis(), - schema, - payload + // offset partition + new RecordPartition(partition), + new RecordOffset(offset.toMap()), + System.currentTimeMillis(), + schema, + payload ); } - + /** + * get begin timestamp from offset topic + * + * @return + */ @Override public Timestamp beginTimestampValue() { return offset.getTimestampOffset(); } + //Get end timestamp from db @Override public Timestamp endTimestampValue(Timestamp beginTime) throws SQLException { long endTimestamp; final long currentDbTime = dialect.currentTimeOnDB( - stmt.getConnection(), - DateTimeUtils.getTimeZoneCalendar(timeZone) + stmt.getConnection(), + DateTimeUtils.getTimeZoneCalendar(timeZone) ).getTime(); endTimestamp = currentDbTime - timestampDelay; return new Timestamp(endTimestamp); @@ -234,14 +225,14 @@ public Long lastIncrementedValue() { @Override public String toString() { return "TimestampIncrementingQuerier{" - + "table=" + tableId - + ", query='" + query + '\'' - + ", topicPrefix='" + topicPrefix + '\'' - + ", incrementingColumn='" + (incrementingColumnName != null - ? incrementingColumnName - : "") + '\'' - + ", timestampColumns=" + timestampColumnNames - + '}'; + + "table=" + context.getTableId() + + ", query='" + context.getQuerySql() + '\'' + + ", topicPrefix='" + context.getTopicPrefix() + '\'' + + ", incrementingColumn='" + (incrementingColumnName != null + ? incrementingColumnName + : "") + '\'' + + ", timestampColumns=" + timestampColumnNames + + '}'; } } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/BytesUtil.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/BytesUtil.java similarity index 99% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/BytesUtil.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/BytesUtil.java index 3e08bdcba..da4241ba3 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/BytesUtil.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/BytesUtil.java @@ -17,9 +17,7 @@ package org.apache.rocketmq.connect.jdbc.util; public class BytesUtil { - private static final char[] HEX_CODE = "0123456789ABCDEF".toCharArray(); - public static String toHex(byte[] data) { StringBuilder r = new StringBuilder(data.length * 2); for (byte b : data) { diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnDefAdjuster.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/ColumnDefAdjuster.java similarity index 90% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnDefAdjuster.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/ColumnDefAdjuster.java index 5e8b6c31c..4ca49b1ee 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnDefAdjuster.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/ColumnDefAdjuster.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.connect.jdbc.schema.column; +package org.apache.rocketmq.connect.jdbc.util; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -22,6 +22,7 @@ import java.sql.SQLException; import java.util.HashMap; import java.util.Map; +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnDefinition; public class ColumnDefAdjuster { Map nullable = new HashMap<>(); @@ -35,9 +36,7 @@ public static ColumnDefAdjuster create(Connection conn, String tablePattern, String columnPattern) { ColumnDefAdjuster adjuster = new ColumnDefAdjuster(); - try (ResultSet rs = conn.getMetaData().getColumns( - catalogPattern, schemaPattern, tablePattern, columnPattern)) { - final int rsColumnCount = rs.getMetaData().getColumnCount(); + try (ResultSet rs = conn.getMetaData().getColumns(catalogPattern, schemaPattern, tablePattern, columnPattern)) { while (rs.next()) { final String columnName = rs.getString(4); ColumnDefinition.Nullability nullability; diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/ConnectorGroupUtils.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/ConnectorGroupUtils.java similarity index 100% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/ConnectorGroupUtils.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/ConnectorGroupUtils.java diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/DateTimeUtils.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/DateTimeUtils.java similarity index 100% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/DateTimeUtils.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/DateTimeUtils.java diff --git a/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/ExpressionBuilder.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/ExpressionBuilder.java new file mode 100644 index 000000000..ae788f35c --- /dev/null +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/ExpressionBuilder.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.connect.jdbc.util; + +import org.apache.rocketmq.connect.jdbc.schema.column.ColumnId; + +public class ExpressionBuilder { + + @FunctionalInterface + public interface Expressable { + + void appendTo( + ExpressionBuilder builder, + boolean useQuotes + ); + default void appendTo( + ExpressionBuilder builder, + QuoteMethod useQuotes + ) { + switch (useQuotes) { + case ALWAYS: + appendTo(builder, true); + break; + case NEVER: + default: + // do nothing + break; + } + } + } + + @FunctionalInterface + public interface Transform { + void apply( + ExpressionBuilder builder, + T input + ); + } + public interface ListBuilder { + ListBuilder delimitedBy(String delimiter); + ListBuilder transformedBy(Transform transform); + + ExpressionBuilder of(Iterable objects); + default ExpressionBuilder of(Iterable objects1, Iterable objects2) { + of(objects1); + return of(objects2); + } + + default ExpressionBuilder of( + Iterable objects1, + Iterable objects2, + Iterable objects3 + ) { + of(objects1); + of(objects2); + return of(objects3); + } + } + + public static Transform quote() { + return (builder, input) -> builder.appendColumnName(input); + } + + public static Transform columnNames() { + return (builder, input) -> builder.appendColumnName(input.name()); + } + + public static Transform columnNamesWith(final String appended) { + return (builder, input) -> { + builder.appendColumnName(input.name()); + builder.append(appended); + }; + } + + public static Transform placeholderInsteadOfColumnNames(final String str) { + return (builder, input) -> builder.append(str); + } + + public static Transform columnNamesWithPrefix(final String prefix) { + return (builder, input) -> { + builder.append(prefix); + builder.appendColumnName(input.name()); + }; + } + + public static ExpressionBuilder create() { + return new ExpressionBuilder(); + } + + protected static final QuoteMethod DEFAULT_QUOTE_METHOD = QuoteMethod.ALWAYS; + + private final IdentifierRules rules; + private final StringBuilder sb = new StringBuilder(); + private QuoteMethod quoteSqlIdentifiers = DEFAULT_QUOTE_METHOD; + + public ExpressionBuilder() { + this(null); + } + + public ExpressionBuilder(IdentifierRules rules) { + this.rules = rules != null ? rules : IdentifierRules.DEFAULT; + } + + public ExpressionBuilder setQuoteIdentifiers(QuoteMethod method) { + this.quoteSqlIdentifiers = method != null ? method : DEFAULT_QUOTE_METHOD; + return this; + } + + public ExpressionBuilder escapeQuotesWith(String prefix) { + if (prefix == null || prefix.isEmpty()) { + return this; + } + return new ExpressionBuilder(this.rules.escapeQuotesWith(prefix)); + } + public ExpressionBuilder appendIdentifierDelimiter() { + sb.append(rules.identifierDelimiter()); + return this; + } + + public ExpressionBuilder appendLeadingQuote() { + return appendLeadingQuote(QuoteMethod.ALWAYS); + } + + + protected ExpressionBuilder appendLeadingQuote(QuoteMethod method) { + switch (method) { + case ALWAYS: + sb.append(rules.leadingQuoteString()); + break; + case NEVER: + default: + break; + } + return this; + } + + public ExpressionBuilder appendTrailingQuote() { + return appendTrailingQuote(QuoteMethod.ALWAYS); + } + + protected ExpressionBuilder appendTrailingQuote(QuoteMethod method) { + switch (method) { + case ALWAYS: + sb.append(rules.trailingQuoteString()); + break; + case NEVER: + default: + break; + } + return this; + } + + public ExpressionBuilder appendStringQuote() { + sb.append("'"); + return this; + } + + public ExpressionBuilder appendStringQuoted(Object name) { + appendStringQuote(); + sb.append(name); + appendStringQuote(); + return this; + } + + @Deprecated + public ExpressionBuilder appendIdentifier( + String name, + boolean quoted + ) { + return appendIdentifier(name, quoted ? QuoteMethod.ALWAYS : QuoteMethod.NEVER); + } + + public ExpressionBuilder appendIdentifier( + String name, + QuoteMethod quoted + ) { + appendLeadingQuote(quoted); + sb.append(name); + appendTrailingQuote(quoted); + return this; + } + + public ExpressionBuilder appendTableName(String name) { + return appendTableName(name, quoteSqlIdentifiers); + } + + public ExpressionBuilder appendTableName(String name, QuoteMethod quote) { + appendLeadingQuote(quote); + sb.append(name); + appendTrailingQuote(quote); + return this; + } + + public ExpressionBuilder appendColumnName(String name) { + return appendColumnName(name, quoteSqlIdentifiers); + } + + public ExpressionBuilder appendColumnName(String name, QuoteMethod quote) { + appendLeadingQuote(quote); + sb.append(name); + appendTrailingQuote(quote); + return this; + } + + public ExpressionBuilder appendIdentifierQuoted(String name) { + appendLeadingQuote(); + sb.append(name); + appendTrailingQuote(); + return this; + } + + public ExpressionBuilder appendBinaryLiteral(byte[] value) { + return append("x'").append(BytesUtil.toHex(value)).append("'"); + } + + public ExpressionBuilder appendNewLine() { + sb.append(System.lineSeparator()); + return this; + } + + @Deprecated + public ExpressionBuilder append( + Object obj, + boolean useQuotes + ) { + return append(obj, useQuotes ? QuoteMethod.ALWAYS : QuoteMethod.NEVER); + } + + public ExpressionBuilder append( + Object obj, + QuoteMethod useQuotes + ) { + if (obj instanceof Expressable) { + ((Expressable) obj).appendTo(this, useQuotes); + } else if (obj != null) { + sb.append(obj); + } + return this; + } + + public ExpressionBuilder append(Object obj) { + return append(obj, quoteSqlIdentifiers); + } + + public ExpressionBuilder append( + T obj, + Transform transform + ) { + if (transform != null) { + transform.apply(this, obj); + } else { + append(obj); + } + return this; + } + + protected class BasicListBuilder implements ListBuilder { + private final String delimiter; + private final Transform transform; + private boolean first = true; + + BasicListBuilder() { + this(", ", null); + } + + BasicListBuilder(String delimiter, Transform transform) { + this.delimiter = delimiter; + this.transform = transform != null ? transform : ExpressionBuilder::append; + } + + @Override + public ListBuilder delimitedBy(String delimiter) { + return new BasicListBuilder(delimiter, transform); + } + + @Override + public ListBuilder transformedBy(Transform transform) { + return new BasicListBuilder<>(delimiter, transform); + } + + @Override + public ExpressionBuilder of(Iterable objects) { + for (T obj : objects) { + if (first) { + first = false; + } else { + append(delimiter); + } + append(obj, transform); + } + return ExpressionBuilder.this; + } + } + + public ListBuilder appendList() { + return new BasicListBuilder<>(); + } + + public ExpressionBuilder appendMultiple( + String delimiter, + String expression, + int times + ) { + for (int i = 0; i < times; i++) { + if (i > 0) { + append(delimiter); + } + append(expression); + } + return this; + } + + @Override + public String toString() { + return sb.toString(); + } +} diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/IdentifierRules.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/IdentifierRules.java similarity index 70% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/IdentifierRules.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/IdentifierRules.java index 25533f56c..e87a0dfef 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/IdentifierRules.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/IdentifierRules.java @@ -23,7 +23,6 @@ * The rules for how identifiers are parsed and quoted. */ public class IdentifierRules { - public static final String UNSUPPORTED_QUOTE = " "; private static final String DEFAULT_QUOTE = "\""; private static final String DEFAULT_ID_DELIM = "."; @@ -36,25 +35,6 @@ public class IdentifierRules { private final String trailingQuoteString; private final String identifierDelimiter; - /** - * Create new identifier rules using the supplied quote string for both leading and trailing - * quotes, and the '{@link #DEFAULT_ID_DELIM}' character for identifier delimiters. - * - * @param quoteString the string used for leading and trailing quotes; may be null if {@link - * #DEFAULT_QUOTE} is to be used - */ - public IdentifierRules(String quoteString) { - this(DEFAULT_ID_DELIM, quoteString, quoteString); - } - - /** - * Create new identifier rules using the supplied parameters. - * - * @param delimiter the delimiter used within fully qualified names; may be null if {@link - * #DEFAULT_ID_DELIM} is to be used - * @param quoteString the string used for leading and trailing quotes; may be null if {@link - * #DEFAULT_QUOTE} is to be used - */ public IdentifierRules( String delimiter, String quoteString @@ -62,16 +42,6 @@ public IdentifierRules( this(delimiter, quoteString, quoteString); } - /** - * Create new identifier rules using the supplied parameters. - * - * @param identifierDelimiter the delimiter used within fully qualified names; may be null if - * {@link #DEFAULT_ID_DELIM} is to be used - * @param leadingQuoteString the string used for leading quotes; may be null if {@link - * #DEFAULT_QUOTE} is to be used - * @param trailingQuoteString the string used for leading quotes; may be null if {@link - * #DEFAULT_QUOTE} is to be used - */ public IdentifierRules( String identifierDelimiter, String leadingQuoteString, @@ -109,21 +79,10 @@ public String trailingQuoteString() { return trailingQuoteString; } - /** - * Get an expression builder that uses these identifier rules. - * - * @return the new expression builder; never null - */ public ExpressionBuilder expressionBuilder() { return new ExpressionBuilder(this); } - /** - * Parse the unqualified or fully qualified name into its segments. - * - * @param fqn the unqualified or fully-qualified name; may not be null - * @return the segments in the supplied name; never null, but possibly empty - */ public List parseQualifiedIdentifier(String fqn) { String orig = fqn; String delim = identifierDelimiter(); @@ -167,12 +126,6 @@ public List parseQualifiedIdentifier(String fqn) { return parts; } - /** - * Return a new IdentifierRules that escapes quotes with the specified prefix. - * - * @param prefix the prefix - * @return the new IdentifierRules, or this builder if the prefix is null or empty - */ public IdentifierRules escapeQuotesWith(String prefix) { if (prefix == null || prefix.isEmpty()) { return this; diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/JdbcDriverInfo.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/JdbcDriverInfo.java similarity index 99% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/JdbcDriverInfo.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/JdbcDriverInfo.java index aa77bf289..9c286ca3a 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/JdbcDriverInfo.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/JdbcDriverInfo.java @@ -21,7 +21,6 @@ * A summary of the version information about a JDBC driver and the database. */ public class JdbcDriverInfo { - private final int jdbcMajorVersion; private final int jdbcMinorVersion; private final String jdbcDriverName; diff --git a/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/JdbcUrlInfo.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/JdbcUrlInfo.java new file mode 100644 index 000000000..5b2ab50dd --- /dev/null +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/JdbcUrlInfo.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.connect.jdbc.util; + +public interface JdbcUrlInfo { + + /** + * Get the subprotocol in the JDBC URL. + * + * @return the subprotocol + */ + String subprotocol(); + + /** + * Get the subname in the JDBC URL, which is everything after the ':' character following the + * subprotocol. + * + * @return the subname + */ + String subname(); + + /** + * Get the full JDBC URL. + * + * @return the URL. + */ + String url(); +} \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/NumericMapping.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/NumericMapping.java similarity index 90% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/NumericMapping.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/NumericMapping.java index 8f3033648..e29a8044f 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/NumericMapping.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/NumericMapping.java @@ -16,11 +16,10 @@ */ package org.apache.rocketmq.connect.jdbc.util; -import org.apache.rocketmq.connect.jdbc.source.JdbcSourceConfig; - import java.util.HashMap; import java.util.Locale; import java.util.Map; +import org.apache.rocketmq.connect.jdbc.source.JdbcSourceConfig; public enum NumericMapping { NONE, @@ -37,12 +36,10 @@ public enum NumericMapping { } public static NumericMapping get(String prop) { - // not adding a check for null value because the recommender/validator should catch those. return REVERSE.get(prop.toLowerCase(Locale.ROOT)); } public static NumericMapping get(JdbcSourceConfig config) { - // We use 'null' as default to be able to check the old config if the new one is unset. if (config.getNumericMapping() != null) { return NumericMapping.valueOf(config.getNumericMapping()); } diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/QuoteMethod.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/QuoteMethod.java similarity index 99% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/QuoteMethod.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/QuoteMethod.java index ccc9d89f3..a759d8c7c 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/QuoteMethod.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/QuoteMethod.java @@ -19,6 +19,11 @@ public enum QuoteMethod { ALWAYS("always"), NEVER("never"); + private final String name; + + QuoteMethod(String name) { + this.name = name; + } public static QuoteMethod get(String name) { for (QuoteMethod method : values()) { @@ -29,12 +34,6 @@ public static QuoteMethod get(String name) { throw new IllegalArgumentException("No matching QuoteMethod found for '" + name + "'"); } - private final String name; - - QuoteMethod(String name) { - this.name = name; - } - @Override public String toString() { return name; diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/TableType.java b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/TableType.java similarity index 79% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/TableType.java rename to connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/TableType.java index 7bbf14b8e..037074e2a 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/main/java/org/apache/rocketmq/connect/jdbc/util/TableType.java +++ b/connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/TableType.java @@ -67,19 +67,4 @@ public static EnumSet parse(Collection values) { return EnumSet.copyOf(types); } - public static String asJdbcTableTypeNames(EnumSet types, String delim) { - return types.stream() - .map(TableType::jdbcName) - .sorted() - .collect(Collectors.joining(delim)); - } - - public static String[] asJdbcTableTypeArray(EnumSet types) { - return types.stream() - .map(TableType::jdbcName) - .sorted() - .collect(Collectors.toList()) - .toArray(new String[types.size()]); - } - } \ No newline at end of file diff --git a/connectors/rocketmq-connect-jdbc/src/main/resources/META-INF/services/org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialectFactory b/connectors/rocketmq-connect-jdbc/src/main/resources/META-INF/services/org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialectFactory new file mode 100644 index 000000000..81f14ddad --- /dev/null +++ b/connectors/rocketmq-connect-jdbc/src/main/resources/META-INF/services/org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialectFactory @@ -0,0 +1,2 @@ +org.apache.rocketmq.connect.jdbc.dialect.mysql.MysqlDatabaseDialectFactory +org.apache.rocketmq.connect.jdbc.dialect.openmldb.OpenMLDBDatabaseDialectFactory diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/resources/mysql-jdbc-sink.conf b/connectors/rocketmq-connect-jdbc/src/main/resources/examples/mysql/mysql-jdbc-sink.conf similarity index 96% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/resources/mysql-jdbc-sink.conf rename to connectors/rocketmq-connect-jdbc/src/main/resources/examples/mysql/mysql-jdbc-sink.conf index 98d1e6c67..39e1ce967 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/resources/mysql-jdbc-sink.conf +++ b/connectors/rocketmq-connect-jdbc/src/main/resources/examples/mysql/mysql-jdbc-sink.conf @@ -1,5 +1,5 @@ { - "connector.class":"org.apache.rocketmq.connect.jdbc.mysql.sink.MysqlJdbcSinkConnector", + "connector.class":"org.apache.rocketmq.connect.jdbc.mysql.sink.JdbcSinkConnector", "max.tasks":"2", "connect.topicnames":"employee-test-topic-json-00002", "connection.url":"jdbc:mysql://localhost:3306/test_database_001", diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/resources/mysql-jdbc-source.conf b/connectors/rocketmq-connect-jdbc/src/main/resources/examples/mysql/mysql-jdbc-source.conf similarity index 94% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/resources/mysql-jdbc-source.conf rename to connectors/rocketmq-connect-jdbc/src/main/resources/examples/mysql/mysql-jdbc-source.conf index 0a6c7625e..5f3b7d3d7 100644 --- a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-mysql/src/main/resources/mysql-jdbc-source.conf +++ b/connectors/rocketmq-connect-jdbc/src/main/resources/examples/mysql/mysql-jdbc-source.conf @@ -1,5 +1,5 @@ { - "connector.class":"org.apache.rocketmq.connect.jdbc.mysql.source.MysqlJdbcSourceConnector", + "connector.class":"org.apache.rocketmq.connect.jdbc.mysql.source.JdbcSourceConnector", "max.tasks":"1", "connect.topicname":"employee-test-topic-json-00002", "connection.url":"jdbc:mysql://localhost:3306", diff --git a/connectors/rocketmq-connect-jdbc/src/main/resources/examples/openmldb/openmldb-jdbc-sink.conf b/connectors/rocketmq-connect-jdbc/src/main/resources/examples/openmldb/openmldb-jdbc-sink.conf new file mode 100644 index 000000000..1f8fe8fa0 --- /dev/null +++ b/connectors/rocketmq-connect-jdbc/src/main/resources/examples/openmldb/openmldb-jdbc-sink.conf @@ -0,0 +1,12 @@ +{ + "connector.class":"org.apache.rocketmq.connect.jdbc.mysql.sink.JdbcSinkConnector", + "max.tasks":"2", + "connect.topicnames":"employee_test", + "connection.url":"jdbc:openmldb:///rocketmq_test?zk=127.0.0.1:2181&zkPath=/openmldb_cluster", + "insert.mode":"INSERT", + "db.timezone":"UTC", + "table.types":"TABLE", + "auto.create":"true", + "key.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter", + "value.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter" +} diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/test/java/org/apache/rocketmq/connect/jdbc/connector/sink/JdbcSinkTest.java b/connectors/rocketmq-connect-jdbc/src/test/java/org/apache/rocketmq/connect/jdbc/connector/sink/JdbcSinkTest.java similarity index 100% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/test/java/org/apache/rocketmq/connect/jdbc/connector/sink/JdbcSinkTest.java rename to connectors/rocketmq-connect-jdbc/src/test/java/org/apache/rocketmq/connect/jdbc/connector/sink/JdbcSinkTest.java diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/test/java/org/apache/rocketmq/connect/jdbc/connector/sink/OpenMLDBJdbcSinkTest.java b/connectors/rocketmq-connect-jdbc/src/test/java/org/apache/rocketmq/connect/jdbc/connector/sink/OpenMLDBJdbcSinkTest.java similarity index 100% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/test/java/org/apache/rocketmq/connect/jdbc/connector/sink/OpenMLDBJdbcSinkTest.java rename to connectors/rocketmq-connect-jdbc/src/test/java/org/apache/rocketmq/connect/jdbc/connector/sink/OpenMLDBJdbcSinkTest.java diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/test/java/org/apache/rocketmq/connect/jdbc/connector/source/JdbcSourceConnectorTest.java b/connectors/rocketmq-connect-jdbc/src/test/java/org/apache/rocketmq/connect/jdbc/connector/source/JdbcSourceConnectorTest.java similarity index 100% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/test/java/org/apache/rocketmq/connect/jdbc/connector/source/JdbcSourceConnectorTest.java rename to connectors/rocketmq-connect-jdbc/src/test/java/org/apache/rocketmq/connect/jdbc/connector/source/JdbcSourceConnectorTest.java diff --git a/connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/test/java/org/apache/rocketmq/connect/jdbc/connector/source/JdbcSourceTaskTest.java b/connectors/rocketmq-connect-jdbc/src/test/java/org/apache/rocketmq/connect/jdbc/connector/source/JdbcSourceTaskTest.java similarity index 100% rename from connectors/rocketmq-connect-jdbc/rocketmq-connect-jdbc-core/src/test/java/org/apache/rocketmq/connect/jdbc/connector/source/JdbcSourceTaskTest.java rename to connectors/rocketmq-connect-jdbc/src/test/java/org/apache/rocketmq/connect/jdbc/connector/source/JdbcSourceTaskTest.java diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSourceTask.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSourceTask.java index c06110971..79317bdb9 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSourceTask.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerSourceTask.java @@ -26,7 +26,6 @@ import io.openmessaging.connector.api.errors.ConnectException; import io.openmessaging.connector.api.errors.RetriableException; import io.openmessaging.connector.api.storage.OffsetStorageReader; - import java.util.HashSet; import java.util.List; import java.util.Optional; @@ -38,7 +37,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; - import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.exception.MQClientException; @@ -125,30 +123,30 @@ public class WorkerSourceTask extends WorkerTask { private final CountDownLatch stopRequestedLatch; private final AtomicReference producerSendException; private final RecordOffsetManagement offsetManagement; + private final Set topicCache; /** * A RocketMQ producer to send message to dest MQ. */ - private DefaultMQProducer producer; + private final DefaultMQProducer producer; private List toSendRecord; private volatile RecordOffsetManagement.CommittableOffsets committableOffsets; - private final Set topicCache; public WorkerSourceTask(WorkerConfig workerConfig, - ConnectorTaskId id, - SourceTask sourceTask, - ClassLoader classLoader, - ConnectKeyValue taskConfig, - PositionManagementService positionManagementService, - RecordConverter keyConverter, - RecordConverter valueConverter, - DefaultMQProducer producer, - AtomicReference workerState, - ConnectStatsManager connectStatsManager, - ConnectStatsService connectStatsService, - TransformChain transformChain, - RetryWithToleranceOperator retryWithToleranceOperator, - WrapperStatusListener statusListener, - ConnectMetrics connectMetrics) { + ConnectorTaskId id, + SourceTask sourceTask, + ClassLoader classLoader, + ConnectKeyValue taskConfig, + PositionManagementService positionManagementService, + RecordConverter keyConverter, + RecordConverter valueConverter, + DefaultMQProducer producer, + AtomicReference workerState, + ConnectStatsManager connectStatsManager, + ConnectStatsService connectStatsService, + TransformChain transformChain, + RetryWithToleranceOperator retryWithToleranceOperator, + WrapperStatusListener statusListener, + ConnectMetrics connectMetrics) { super(workerConfig, id, classLoader, taskConfig, retryWithToleranceOperator, transformChain, workerState, statusListener, connectMetrics); this.sourceTask = sourceTask; @@ -220,7 +218,7 @@ protected void updateCommittableOffsets() { } protected Optional prepareToSendRecord( - ConnectRecord record + ConnectRecord record ) { maybeThrowProducerSendException(); return Optional.of(this.offsetManagement.submitRecord(record.getPosition())); @@ -232,7 +230,6 @@ protected Optional prepareToSendRecord private Boolean sendRecord() throws InterruptedException { int processed = 0; - final CalcSourceRecordWrite counter = new CalcSourceRecordWrite(toSendRecord.size(), sourceTaskMetricsGroup); for (ConnectRecord preTransformRecord : toSendRecord) { retryWithToleranceOperator.sourceRecord(preTransformRecord); @@ -284,7 +281,7 @@ public void onException(Throwable throwable) { } catch (RetriableException e) { log.warn("{} Failed to send record to topic '{}'. Backing off before retrying: ", - this, sourceMessage.getTopic(), e); + this, sourceMessage.getTopic(), e); // Intercepted as successfully sent, used to continue sending next time toSendRecord = toSendRecord.subList(processed, toSendRecord.size()); // remove pre submit position, for retry @@ -314,17 +311,17 @@ private void prepareToPollTask() { private void maybeThrowProducerSendException() { if (producerSendException.get() != null) { throw new ConnectException( - "Unrecoverable exception from producer send callback", - producerSendException.get() + "Unrecoverable exception from producer send callback", + producerSendException.get() ); } } private void recordSendFailed( - boolean synchronous, - Message sourceMessage, - ConnectRecord preTransformRecord, - Throwable e) { + boolean synchronous, + Message sourceMessage, + ConnectRecord preTransformRecord, + Throwable e) { if (synchronous) { throw new ConnectException("Unrecoverable exception trying to send", e); } @@ -332,16 +329,16 @@ private void recordSendFailed( if (retryWithToleranceOperator.getErrorToleranceType() == ToleranceType.ALL) { // ignore all error log.trace( - "Ignoring failed record send: {} failed to send record to {}: ", - WorkerSourceTask.this, - topic, - e + "Ignoring failed record send: {} failed to send record to {}: ", + WorkerSourceTask.this, + topic, + e ); retryWithToleranceOperator.executeFailed( - ErrorReporter.Stage.ROCKETMQ_PRODUCE, - WorkerSourceTask.class, - preTransformRecord, - e); + ErrorReporter.Stage.ROCKETMQ_PRODUCE, + WorkerSourceTask.class, + preTransformRecord, + e); commitTaskRecord(preTransformRecord, null); } else { log.error("{} failed to send record to {}: ", WorkerSourceTask.this, topic, e); @@ -367,9 +364,9 @@ private void recordFailed(ConnectRecord record) { * @param result */ private void recordSent( - ConnectRecord preTransformRecord, - Message sourceMessage, - SendResult result) { + ConnectRecord preTransformRecord, + Message sourceMessage, + SendResult result) { commitTaskRecord(preTransformRecord, result); } @@ -400,10 +397,10 @@ protected Message convertTransformedRecord(final String topic, ConnectRecord rec Message sourceMessage = new Message(); sourceMessage.setTopic(topic); byte[] key = retryWithToleranceOperator.execute(() -> keyConverter.fromConnectData(topic, record.getKeySchema(), record.getKey()), - ErrorReporter.Stage.CONVERTER, keyConverter.getClass()); + ErrorReporter.Stage.CONVERTER, keyConverter.getClass()); byte[] value = retryWithToleranceOperator.execute(() -> valueConverter.fromConnectData(topic, record.getSchema(), record.getData()), - ErrorReporter.Stage.CONVERTER, valueConverter.getClass()); + ErrorReporter.Stage.CONVERTER, valueConverter.getClass()); if (value.length > ConnectorConfig.MAX_MESSAGE_SIZE) { log.error("Send record, message size is greater than {} bytes, record: {}", ConnectorConfig.MAX_MESSAGE_SIZE, JSON.toJSONString(record)); } @@ -485,7 +482,6 @@ protected void initializeAndStart() { log.info("{} Source task finished initialization and start", this); } - /** * execute poll and send record */ @@ -507,7 +503,6 @@ protected void execute() { } } - if (CollectionUtils.isEmpty(toSendRecord)) { try { prepareToPollTask(); @@ -548,8 +543,8 @@ protected void execute() { protected void finalOffsetCommit(boolean b) { offsetManagement.awaitAllMessages( - workerConfig.getOffsetCommitTimeoutMsConfig(), - TimeUnit.MILLISECONDS + workerConfig.getOffsetCommitTimeoutMsConfig(), + TimeUnit.MILLISECONDS ); updateCommittableOffsets(); commitOffsets(); @@ -570,9 +565,9 @@ public boolean commitOffsets() { if (committableOffsets.isEmpty()) { log.debug("{} Either no records were produced by the task since the last offset commit, " - + "or every record has been filtered out by a transformation " - + "or dropped due to transformation or conversion errors.", - this + + "or every record has been filtered out by a transformation " + + "or dropped due to transformation or conversion errors.", + this ); // We continue with the offset commit process here instead of simply returning immediately // in order to invoke SourceTask::commit and record metrics for a successful offset commit @@ -580,17 +575,17 @@ public boolean commitOffsets() { log.info("{} Committing offsets for {} acknowledged messages", this, committableOffsets.numCommittableMessages()); if (committableOffsets.hasPending()) { log.debug("{} There are currently {} pending messages spread across {} source partitions whose offsets will not be committed. " - + "The source partition with the most pending messages is {}, with {} pending messages", - this, - committableOffsets.numUncommittableMessages(), - committableOffsets.numDeques(), - committableOffsets.largestDequePartition(), - committableOffsets.largestDequeSize() + + "The source partition with the most pending messages is {}, with {} pending messages", + this, + committableOffsets.numUncommittableMessages(), + committableOffsets.numDeques(), + committableOffsets.largestDequePartition(), + committableOffsets.largestDequeSize() ); } else { log.debug("{} There are currently no pending messages for this offset commit; " - + "all messages dispatched to the task's producer since the last commit have been acknowledged", - this + + "all messages dispatched to the task's producer since the last commit have been acknowledged", + this ); } } @@ -604,7 +599,7 @@ public boolean commitOffsets() { long durationMillis = System.currentTimeMillis() - started; recordCommitSuccess(durationMillis); log.debug("{} Finished offset commitOffsets successfully in {} ms", - this, durationMillis); + this, durationMillis); commitSourceTask(); return true; } @@ -637,7 +632,7 @@ public boolean commitOffsets() { long durationMillis = System.currentTimeMillis() - started; recordCommitSuccess(durationMillis); log.debug("{} Finished commitOffsets successfully in {} ms", - this, durationMillis); + this, durationMillis); commitSourceTask(); return true; } @@ -650,7 +645,6 @@ protected void commitSourceTask() { } } - protected void recordPollReturned(int numRecordsInBatch, long millTime) { sourceTaskMetricsGroup.recordPoll(numRecordsInBatch, millTime); } @@ -662,13 +656,13 @@ static class SourceTaskMetricsGroup implements AutoCloseable { private final Sensor pollTime; private int activeRecordCount; - private MetricGroup metricGroup; + private final MetricGroup metricGroup; public SourceTaskMetricsGroup(ConnectorTaskId id, ConnectMetrics connectMetrics) { ConnectMetricsTemplates templates = connectMetrics.templates(); metricGroup = connectMetrics.group( - templates.connectorTagName(), id.connector(), - templates.taskTagName(), Integer.toString(id.task())); + templates.connectorTagName(), id.connector(), + templates.taskTagName(), Integer.toString(id.task())); sourceRecordPoll = metricGroup.sensor(); sourceRecordPoll.addStat(new Rate(connectMetrics.registry(), metricGroup.name(templates.sourceRecordPollRate))); diff --git a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractConfigManagementService.java b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractConfigManagementService.java index 1ce8940ff..4e312e29b 100644 --- a/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractConfigManagementService.java +++ b/rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/AbstractConfigManagementService.java @@ -248,7 +248,7 @@ public void deleteConnectorConfig(String connectorName) { struct.put(FIELD_EPOCH, System.currentTimeMillis()); struct.put(FIELD_DELETED, true); byte[] config = converter.fromConnectData(topic, CONNECTOR_DELETE_CONFIGURATION_V1, struct); - notify(DELETE_CONNECTOR_KEY(connectorName), config); + notify(TARGET_STATE_KEY(connectorName), config); } /**