Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
apache
pulsar
提交
f88ea9df
pulsar
项目概览
apache
/
pulsar
通知
129
Star
40
Fork
3
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Wiki
1
Wiki
分析
仓库
DevOps
项目成员
Pages
pulsar
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Pages
分析
分析
仓库分析
DevOps
Wiki
1
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
提交
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
f88ea9df
编写于
7月 25, 2019
作者:
M
Matteo Merli
提交者:
Boyang Jerry Peng
7月 25, 2019
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Reuse ManagedLedgerFactory instances across SQL queries (#4813)
上级
88c85447
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
36 addition
and
66 deletion
+36
-66
pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnector.java
...in/java/org/apache/pulsar/sql/presto/PulsarConnector.java
+0
-5
pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java
...va/org/apache/pulsar/sql/presto/PulsarConnectorCache.java
+4
-1
pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java
...java/org/apache/pulsar/sql/presto/PulsarSplitManager.java
+30
-59
pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java
...ava/org/apache/pulsar/sql/presto/TestPulsarConnector.java
+2
-1
未找到文件。
pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnector.java
浏览文件 @
f88ea9df
...
...
@@ -86,11 +86,6 @@ public class PulsarConnector implements Connector {
}
catch
(
Exception
e
)
{
log
.
error
(
e
,
"Failed to close pulsar connector"
);
}
try
{
PulsarConnectorCache
.
shutdown
();
}
catch
(
Exception
e
)
{
log
.
error
(
"Failed to shutdown pulsar connector cache"
);
}
try
{
lifeCycleManager
.
stop
();
}
catch
(
Exception
e
)
{
...
...
pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java
浏览文件 @
f88ea9df
...
...
@@ -39,11 +39,14 @@ import java.util.Map;
import
static
com
.
google
.
common
.
base
.
Preconditions
.
checkNotNull
;
import
com.google.common.annotations.VisibleForTesting
;
public
class
PulsarConnectorCache
{
private
static
final
Logger
log
=
Logger
.
get
(
PulsarConnectorCache
.
class
);
private
static
PulsarConnectorCache
instance
;
@VisibleForTesting
static
PulsarConnectorCache
instance
;
private
final
ManagedLedgerFactory
managedLedgerFactory
;
...
...
pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java
浏览文件 @
f88ea9df
...
...
@@ -133,16 +133,6 @@ public class PulsarSplitManager implements ConnectorSplitManager {
return
new
FixedSplitSource
(
splits
);
}
@VisibleForTesting
ManagedLedgerFactory
getManagedLedgerFactory
()
throws
Exception
{
ClientConfiguration
bkClientConfiguration
=
new
ClientConfiguration
()
.
setZkServers
(
this
.
pulsarConnectorConfig
.
getZookeeperUri
())
.
setClientTcpNoDelay
(
false
)
.
setStickyReadsEnabled
(
false
)
.
setUseV2WireProtocol
(
true
);
return
new
ManagedLedgerFactoryImpl
(
bkClientConfiguration
);
}
@VisibleForTesting
Collection
<
PulsarSplit
>
getSplitsPartitionedTopic
(
int
numSplits
,
TopicName
topicName
,
PulsarTableHandle
tableHandle
,
SchemaInfo
schemaInfo
,
TupleDomain
<
ColumnHandle
>
tupleDomain
)
throws
Exception
{
...
...
@@ -165,59 +155,40 @@ public class PulsarSplitManager implements ConnectorSplitManager {
int
splitRemainder
=
actualNumSplits
%
numPartitions
;
ManagedLedgerFactory
managedLedgerFactory
=
getManagedLedgerFactory
();
try
{
List
<
PulsarSplit
>
splits
=
new
LinkedList
<>();
for
(
int
i
=
0
;
i
<
numPartitions
;
i
++)
{
int
splitsForThisPartition
=
(
splitRemainder
>
i
)
?
splitsPerPartition
+
1
:
splitsPerPartition
;
splits
.
addAll
(
getSplitsForTopic
(
topicName
.
getPartition
(
i
).
getPersistenceNamingEncoding
(),
managedLedgerFactory
,
splitsForThisPartition
,
tableHandle
,
schemaInfo
,
topicName
.
getPartition
(
i
).
getLocalName
(),
tupleDomain
)
);
}
return
splits
;
}
finally
{
if
(
managedLedgerFactory
!=
null
)
{
try
{
managedLedgerFactory
.
shutdown
();
}
catch
(
Exception
e
)
{
log
.
error
(
e
);
}
}
ManagedLedgerFactory
managedLedgerFactory
=
PulsarConnectorCache
.
getConnectorCache
(
pulsarConnectorConfig
)
.
getManagedLedgerFactory
();
List
<
PulsarSplit
>
splits
=
new
LinkedList
<>();
for
(
int
i
=
0
;
i
<
numPartitions
;
i
++)
{
int
splitsForThisPartition
=
(
splitRemainder
>
i
)
?
splitsPerPartition
+
1
:
splitsPerPartition
;
splits
.
addAll
(
getSplitsForTopic
(
topicName
.
getPartition
(
i
).
getPersistenceNamingEncoding
(),
managedLedgerFactory
,
splitsForThisPartition
,
tableHandle
,
schemaInfo
,
topicName
.
getPartition
(
i
).
getLocalName
(),
tupleDomain
));
}
return
splits
;
}
@VisibleForTesting
Collection
<
PulsarSplit
>
getSplitsNonPartitionedTopic
(
int
numSplits
,
TopicName
topicName
,
PulsarTableHandle
tableHandle
,
SchemaInfo
schemaInfo
,
TupleDomain
<
ColumnHandle
>
tupleDomain
)
throws
Exception
{
ManagedLedgerFactory
managedLedgerFactory
=
null
;
try
{
managedLedgerFactory
=
getManagedLedgerFactory
();
return
getSplitsForTopic
(
topicName
.
getPersistenceNamingEncoding
(),
managedLedgerFactory
,
numSplits
,
tableHandle
,
schemaInfo
,
tableHandle
.
getTableName
(),
tupleDomain
);
}
finally
{
if
(
managedLedgerFactory
!=
null
)
{
try
{
managedLedgerFactory
.
shutdown
();
}
catch
(
Exception
e
)
{
log
.
error
(
e
);
}
}
}
Collection
<
PulsarSplit
>
getSplitsNonPartitionedTopic
(
int
numSplits
,
TopicName
topicName
,
PulsarTableHandle
tableHandle
,
SchemaInfo
schemaInfo
,
TupleDomain
<
ColumnHandle
>
tupleDomain
)
throws
Exception
{
ManagedLedgerFactory
managedLedgerFactory
=
PulsarConnectorCache
.
getConnectorCache
(
pulsarConnectorConfig
)
.
getManagedLedgerFactory
();
return
getSplitsForTopic
(
topicName
.
getPersistenceNamingEncoding
(),
managedLedgerFactory
,
numSplits
,
tableHandle
,
schemaInfo
,
tableHandle
.
getTableName
(),
tupleDomain
);
}
@VisibleForTesting
...
...
pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java
浏览文件 @
f88ea9df
...
...
@@ -920,7 +920,8 @@ public abstract class TestPulsarConnector {
}
});
doReturn
(
managedLedgerFactory
).
when
(
this
.
pulsarSplitManager
).
getManagedLedgerFactory
();
PulsarConnectorCache
.
instance
=
mock
(
PulsarConnectorCache
.
class
);
when
(
PulsarConnectorCache
.
instance
.
getManagedLedgerFactory
()).
thenReturn
(
managedLedgerFactory
);
for
(
Map
.
Entry
<
TopicName
,
PulsarSplit
>
split
:
splits
.
entrySet
())
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录