提交 d8356d8c 编写于 作者: Y Yi Tang 提交者: Sijie Guo

[pulsar-io-jdbc] not set action as insert (#4862)

### Motivation

jdbc sink treat all record as INSERT before #4358 , now it requires an indispensable action property which seems to be a break change, and we can deal records without any action property as INSERT.

### Modifications

treat action not set as INSERT action like before.
上级 434c3956
...@@ -182,18 +182,29 @@ public abstract class JdbcAbstractSink<T> implements Sink<T> { ...@@ -182,18 +182,29 @@ public abstract class JdbcAbstractSink<T> implements Sink<T> {
// bind each record value // bind each record value
for (Record<T> record : swapList) { for (Record<T> record : swapList) {
String action = record.getProperties().get(ACTION); String action = record.getProperties().get(ACTION);
if (action != null && action.equals(DELETE)) { if (action == null) {
bindValue(deleteStatment, record, action); action = INSERT;
count += 1; }
deleteStatment.execute(); switch (action) {
} else if (action != null && action.equals(UPDATE)) { case DELETE:
bindValue(updateStatment, record, action); bindValue(deleteStatment, record, action);
count += 1; count += 1;
updateStatment.execute(); deleteStatment.execute();
} else if (action != null && action.equals(INSERT)){ break;
bindValue(insertStatement, record, action); case UPDATE:
count += 1; bindValue(updateStatment, record, action);
insertStatement.execute(); count += 1;
updateStatment.execute();
break;
case INSERT:
bindValue(insertStatement, record, action);
count += 1;
insertStatement.execute();
break;
default:
String msg = String.format("Unsupported action %s, can be one of %s, or not set which indicate %s",
action, Arrays.asList(INSERT, UPDATE, DELETE), INSERT);
throw new IllegalArgumentException(msg);
} }
} }
connection.commit(); connection.commit();
......
...@@ -19,9 +19,13 @@ ...@@ -19,9 +19,13 @@
package org.apache.pulsar.io.jdbc; package org.apache.pulsar.io.jdbc;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
import java.util.Map; import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import lombok.Data; import lombok.Data;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.ToString; import lombok.ToString;
...@@ -40,6 +44,7 @@ import org.testng.annotations.AfterMethod; ...@@ -40,6 +44,7 @@ import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
...@@ -95,7 +100,6 @@ public class JdbcSinkTest { ...@@ -95,7 +100,6 @@ public class JdbcSinkTest {
// change batchSize to 1, to flush on each write. // change batchSize to 1, to flush on each write.
conf.put("batchSize", 1); conf.put("batchSize", 1);
jdbcSink = new JdbcAutoSchemaSink();
jdbcSink = new JdbcAutoSchemaSink(); jdbcSink = new JdbcAutoSchemaSink();
// open should success // open should success
...@@ -109,8 +113,7 @@ public class JdbcSinkTest { ...@@ -109,8 +113,7 @@ public class JdbcSinkTest {
jdbcSink.close(); jdbcSink.close();
} }
@Test private void testOpenAndWriteSink(Map<String, String> actionProperties) throws Exception {
public void TestOpenAndWriteSink() throws Exception {
Message<GenericRecord> insertMessage = mock(MessageImpl.class); Message<GenericRecord> insertMessage = mock(MessageImpl.class);
GenericSchema<GenericRecord> genericAvroSchema; GenericSchema<GenericRecord> genericAvroSchema;
// prepare a foo Record // prepare a foo Record
...@@ -121,19 +124,16 @@ public class JdbcSinkTest { ...@@ -121,19 +124,16 @@ public class JdbcSinkTest {
AvroSchema<Foo> schema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build()); AvroSchema<Foo> schema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
byte[] insertBytes = schema.encode(insertObj); byte[] insertBytes = schema.encode(insertObj);
CompletableFuture<Void> future = new CompletableFuture<>();
Record<GenericRecord> insertRecord = PulsarRecord.<GenericRecord>builder() Record<GenericRecord> insertRecord = PulsarRecord.<GenericRecord>builder()
.message(insertMessage) .message(insertMessage)
.topicName("fake_topic_name") .topicName("fake_topic_name")
.ackFunction(() -> {}) .ackFunction(() -> future.complete(null))
.build(); .build();
genericAvroSchema = new GenericAvroSchema(schema.getSchemaInfo()); genericAvroSchema = new GenericAvroSchema(schema.getSchemaInfo());
Map<String, String> insertProperties = Maps.newHashMap();
insertProperties.put("ACTION", "INSERT");
when(insertMessage.getValue()).thenReturn(genericAvroSchema.decode(insertBytes)); when(insertMessage.getValue()).thenReturn(genericAvroSchema.decode(insertBytes));
when(insertMessage.getProperties()).thenReturn(insertProperties); when(insertMessage.getProperties()).thenReturn(actionProperties);
log.info("foo:{}, Message.getValue: {}, record.getValue: {}", log.info("foo:{}, Message.getValue: {}, record.getValue: {}",
insertObj.toString(), insertObj.toString(),
insertMessage.getValue().toString(), insertMessage.getValue().toString(),
...@@ -143,7 +143,7 @@ public class JdbcSinkTest { ...@@ -143,7 +143,7 @@ public class JdbcSinkTest {
jdbcSink.write(insertRecord); jdbcSink.write(insertRecord);
log.info("executed write"); log.info("executed write");
// sleep to wait backend flush complete // sleep to wait backend flush complete
Thread.sleep(1000); future.get(1, TimeUnit.SECONDS);
// value has been written to db, read it out and verify. // value has been written to db, read it out and verify.
String querySql = "SELECT * FROM " + tableName + " WHERE field3=3"; String querySql = "SELECT * FROM " + tableName + " WHERE field3=3";
...@@ -156,6 +156,25 @@ public class JdbcSinkTest { ...@@ -156,6 +156,25 @@ public class JdbcSinkTest {
} }
@Test
public void TestInsertAction() throws Exception {
testOpenAndWriteSink(ImmutableMap.of("ACTION", "INSERT"));
}
@Test
public void TestNoAction() throws Exception {
testOpenAndWriteSink(ImmutableMap.of());
}
@Test
public void TestUnknownAction() throws Exception {
Record<GenericRecord> recordRecord = mock(Record.class);
when(recordRecord.getProperties()).thenReturn(ImmutableMap.of("ACTION", "UNKNOWN"));
CompletableFuture<Void> future = new CompletableFuture<>();
doAnswer(a -> future.complete(null)).when(recordRecord).fail();
jdbcSink.write(recordRecord);
future.get(1, TimeUnit.SECONDS);
}
@Test @Test
public void TestUpdateAction() throws Exception { public void TestUpdateAction() throws Exception {
...@@ -169,10 +188,11 @@ public class JdbcSinkTest { ...@@ -169,10 +188,11 @@ public class JdbcSinkTest {
byte[] updateBytes = schema.encode(updateObj); byte[] updateBytes = schema.encode(updateObj);
Message<GenericRecord> updateMessage = mock(MessageImpl.class); Message<GenericRecord> updateMessage = mock(MessageImpl.class);
CompletableFuture<Void> future = new CompletableFuture<>();
Record<GenericRecord> updateRecord = PulsarRecord.<GenericRecord>builder() Record<GenericRecord> updateRecord = PulsarRecord.<GenericRecord>builder()
.message(updateMessage) .message(updateMessage)
.topicName("fake_topic_name") .topicName("fake_topic_name")
.ackFunction(() -> {}) .ackFunction(() -> future.complete(null))
.build(); .build();
GenericSchema<GenericRecord> updateGenericAvroSchema; GenericSchema<GenericRecord> updateGenericAvroSchema;
...@@ -188,8 +208,7 @@ public class JdbcSinkTest { ...@@ -188,8 +208,7 @@ public class JdbcSinkTest {
updateRecord.getValue().toString()); updateRecord.getValue().toString());
jdbcSink.write(updateRecord); jdbcSink.write(updateRecord);
future.get(1, TimeUnit.SECONDS);
Thread.sleep(1000);
// value has been written to db, read it out and verify. // value has been written to db, read it out and verify.
String updateQuerySql = "SELECT * FROM " + tableName + " WHERE field3=4"; String updateQuerySql = "SELECT * FROM " + tableName + " WHERE field3=4";
...@@ -210,10 +229,11 @@ public class JdbcSinkTest { ...@@ -210,10 +229,11 @@ public class JdbcSinkTest {
byte[] deleteBytes = schema.encode(deleteObj); byte[] deleteBytes = schema.encode(deleteObj);
Message<GenericRecord> deleteMessage = mock(MessageImpl.class); Message<GenericRecord> deleteMessage = mock(MessageImpl.class);
CompletableFuture<Void> future = new CompletableFuture<>();
Record<GenericRecord> deleteRecord = PulsarRecord.<GenericRecord>builder() Record<GenericRecord> deleteRecord = PulsarRecord.<GenericRecord>builder()
.message(deleteMessage) .message(deleteMessage)
.topicName("fake_topic_name") .topicName("fake_topic_name")
.ackFunction(() -> {}) .ackFunction(() -> future.complete(null))
.build(); .build();
GenericSchema<GenericRecord> deleteGenericAvroSchema = new GenericAvroSchema(schema.getSchemaInfo()); GenericSchema<GenericRecord> deleteGenericAvroSchema = new GenericAvroSchema(schema.getSchemaInfo());
...@@ -228,8 +248,7 @@ public class JdbcSinkTest { ...@@ -228,8 +248,7 @@ public class JdbcSinkTest {
deleteRecord.getValue().toString()); deleteRecord.getValue().toString());
jdbcSink.write(deleteRecord); jdbcSink.write(deleteRecord);
future.get(1, TimeUnit.SECONDS);
Thread.sleep(1000);
// value has been written to db, read it out and verify. // value has been written to db, read it out and verify.
String deleteQuerySql = "SELECT * FROM " + tableName + " WHERE field3=5"; String deleteQuerySql = "SELECT * FROM " + tableName + " WHERE field3=5";
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册