Merge pull request #198 from didi/master

merge master

Releases_Notes.md

+
+---
+
+![kafka-manager-logo](./docs/assets/images/common/logo_name.png)
+
+**一站式`Apache Kafka`集群指标监控与运维管控平台**
+
+--- 
+
+## v2.3.0
+
+版本上线时间：2021-02-08
+
+
+### 能力提升
+
+- 新增支持docker化部署
+- 可指定Broker作为候选controller
+- 可新增并管理网关配置
+- 可获取消费组状态
+- 增加集群的JMX认证  
+
+### 体验优化
+
+- 优化编辑用户角色、修改密码的流程
+- 新增consumerID的搜索功能
+- 优化“Topic连接信息”、“消费组重置消费偏移”、“修改Topic保存时间”的文案提示
+- 在相应位置增加《资源申请文档》链接 
+
+### bug修复
+
+- 修复Broker监控图表时间轴展示错误的问题
+- 修复创建夜莺监控告警规则时，使用的告警周期的单位不正确的问题
+
+
+
+## v2.2.0
+
+版本上线时间：2021-01-25
+
+
+
+### 能力提升
+
+- 优化工单批量操作流程 
+- 增加获取Topic75分位/99分位的实时耗时数据
+- 增加定时任务，可将无主未落DB的Topic定期写入DB
+
+### 体验优化
+
+- 在相应位置增加《集群接入文档》链接
+- 优化物理集群、逻辑集群含义
+- 在Topic详情页、Topic扩分区操作弹窗增加展示Topic所属Region的信息
+- 优化Topic审批时，Topic数据保存时间的配置流程
+- 优化Topic/应用申请、审批时的错误提示文案
+- 优化Topic数据采样的操作项文案
+- 优化运维人员删除Topic时的提示文案
+- 优化运维人员删除Region的删除逻辑与提示文案
+- 优化运维人员删除逻辑集群的提示文案
+- 优化上传集群配置文件时的文件类型限制条件
+
+### bug修复
+
+- 修复填写应用名称时校验特殊字符出错的问题
+- 修复普通用户越权访问应用详情的问题
+- 修复由于Kafka版本升级，导致的数据压缩格式无法获取的问题
+- 修复删除逻辑集群或Topic之后，界面依旧展示的问题
+- 修复进行Leader rebalance操作时执行结果重复提示的问题
+
+
+## v2.1.0
+
+版本上线时间：2020-12-19
+
+
+
+### 体验优化
+
+- 优化页面加载时的背景样式
+- 优化普通用户申请Topic权限的流程
+- 优化Topic申请配额、申请分区的权限限制
+- 优化取消Topic权限的文案提示
+- 优化申请配额表单的表单项名称
+- 优化重置消费偏移的操作流程
+- 优化创建Topic迁移任务的表单内容
+- 优化Topic扩分区操作的弹窗界面样式
+- 优化集群Broker监控可视化图表样式
+- 优化创建逻辑集群的表单内容
+- 优化集群安全协议的提示文案
+
+### bug修复
+
+- 修复偶发性重置消费偏移失败的问题
+
+
+
+

docs/user_guide/faq.md

@@ -18,6 +18,7 @@
 - 6、如何使用`MySQL 8`？
 - 7、`Jmx`连接失败如何解决？
 - 8、`topic biz data not exist`错误及处理方式
+- 9、进程启动后，如何查看API文档
 
 ---
 
@@ -76,7 +77,7 @@
 
 - 3、数据库时区问题。
 
-检查MySQL的topic表，查看是否有数据，如果有数据，那么再检查设置的时区是否正确。
+检查MySQL的topic_metrics表，查看是否有数据，如果有数据，那么再检查设置的时区是否正确。
 
 ---
 
@@ -109,3 +110,7 @@
 可以在`运维管控->集群列表->Topic信息`下面，编辑申请权限的Topic，为Topic选择一个应用即可。
 
 以上仅仅只是针对单个Topic的场景，如果你有非常多的Topic需要进行初始化的，那么此时可以在配置管理中增加一个配置，来定时的对无主的Topic进行同步，具体见：[动态配置管理 - 1、Topic定时同步任务](../dev_guide/dynamic_config_manager.md)
+
+### 9、进程启动后，如何查看API文档
+
+- 滴滴Logi-KafkaManager采用Swagger-API工具记录API文档。Swagger-API地址： [http://IP:PORT/swagger-ui.html#/](http://IP:PORT/swagger-ui.html#/) 

kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ResultStatus.java

@@ -106,6 +106,7 @@ public enum ResultStatus {
     STORAGE_UPLOAD_FILE_FAILED(8050, "upload file failed"),
     STORAGE_FILE_TYPE_NOT_SUPPORT(8051, "File type not support"),
     STORAGE_DOWNLOAD_FILE_FAILED(8052, "download file failed"),
+    LDAP_AUTHENTICATION_FAILED(8053, "LDAP authentication failed"),
 
     ;
 

kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/ResultStatus.java

+package com.xiaojukeji.kafka.manager.common.entity;
+
+import com.xiaojukeji.kafka.manager.common.constant.Constant;
+
+/**
+ * 返回状态
+ * @author zengqiao
+ * @date 20/4/16
+ */
+public enum ResultStatus {
+    GATEWAY_INVALID_REQUEST(-1, "invalid request"),
+
+    SUCCESS(Constant.SUCCESS, "success"),
+
+    FAIL(1, "操作失败"),
+
+    /**
+     * 操作错误[1000, 2000)
+     * ------------------------------------------------------------------------------------------
+     */
+    OPERATION_FAILED(1401, "operation failed"),
+    OPERATION_FORBIDDEN(1402, "operation forbidden"),
+    API_CALL_EXCEED_LIMIT(1403, "api call exceed limit"),
+    USER_WITHOUT_AUTHORITY(1404, "user without authority"),
+    CHANGE_ZOOKEEPER_FORBIDDEN(1405, "change zookeeper forbidden"),
+
+
+    TOPIC_OPERATION_PARAM_NULL_POINTER(1450, "参数错误"),
+    TOPIC_OPERATION_PARTITION_NUM_ILLEGAL(1451, "分区数错误"),
+    TOPIC_OPERATION_BROKER_NUM_NOT_ENOUGH(1452, "Broker数不足错误"),
+    TOPIC_OPERATION_TOPIC_NAME_ILLEGAL(1453, "Topic名称非法"),
+    TOPIC_OPERATION_TOPIC_EXISTED(1454, "Topic已存在"),
+    TOPIC_OPERATION_UNKNOWN_TOPIC_PARTITION(1455, "Topic未知"),
+    TOPIC_OPERATION_TOPIC_CONFIG_ILLEGAL(1456, "Topic配置错误"),
+    TOPIC_OPERATION_TOPIC_IN_DELETING(1457, "Topic正在删除"),
+    TOPIC_OPERATION_UNKNOWN_ERROR(1458, "未知错误"),
+
+    /**
+     * 参数错误[2000, 3000)
+     * ------------------------------------------------------------------------------------------
+     */
+    PARAM_ILLEGAL(2000, "param illegal"),
+    CG_LOCATION_ILLEGAL(2001, "consumer group location illegal"),
+    ORDER_ALREADY_HANDLED(2002, "order already handled"),
+    APP_ID_OR_PASSWORD_ILLEGAL(2003, "app or password illegal"),
+    SYSTEM_CODE_ILLEGAL(2004, "system code illegal"),
+    CLUSTER_TASK_HOST_LIST_ILLEGAL(2005, "主机列表错误，请检查主机列表"),
+    JSON_PARSER_ERROR(2006, "json parser error"),
+
+    BROKER_NUM_NOT_ENOUGH(2050, "broker not enough"),
+    CONTROLLER_NOT_ALIVE(2051, "controller not alive"),
+    CLUSTER_METADATA_ERROR(2052, "cluster metadata error"),
+    TOPIC_CONFIG_ERROR(2053, "topic config error"),
+
+    /**
+     * 参数错误 - 资源检查错误
+     * 因为外部系统的问题, 操作时引起的错误, [7000, 8000)
+     * ------------------------------------------------------------------------------------------
+     */
+    RESOURCE_NOT_EXIST(7100, "资源不存在"),
+    CLUSTER_NOT_EXIST(7101, "cluster not exist"),
+    BROKER_NOT_EXIST(7102, "broker not exist"),
+    TOPIC_NOT_EXIST(7103, "topic not exist"),
+    PARTITION_NOT_EXIST(7104, "partition not exist"),
+    ACCOUNT_NOT_EXIST(7105, "account not exist"),
+    APP_NOT_EXIST(7106, "app not exist"),
+    ORDER_NOT_EXIST(7107, "order not exist"),
+    CONFIG_NOT_EXIST(7108, "config not exist"),
+    IDC_NOT_EXIST(7109, "idc not exist"),
+    TASK_NOT_EXIST(7110, "task not exist"),
+    AUTHORITY_NOT_EXIST(7111, "authority not exist"),
+    MONITOR_NOT_EXIST(7112, "monitor not exist"),
+    QUOTA_NOT_EXIST(7113, "quota not exist, please check clusterId, topicName and appId"),
+    CONSUMER_GROUP_NOT_EXIST(7114, "consumerGroup not exist"),
+    TOPIC_BIZ_DATA_NOT_EXIST(7115, "topic biz data not exist, please sync topic to db"),
+
+
+    // 资源已存在
+    RESOURCE_ALREADY_EXISTED(7200, "资源已经存在"),
+    TOPIC_ALREADY_EXIST(7201, "topic already existed"),
+
+    // 资源重名
+    RESOURCE_NAME_DUPLICATED(7300, "资源名称重复"),
+
+    // 资源已被使用
+    RESOURCE_ALREADY_USED(7400, "资源早已被使用"),
+
+
+    /**
+     * 因为外部系统的问题, 操作时引起的错误, [8000, 9000)
+     * ------------------------------------------------------------------------------------------
+     */
+    MYSQL_ERROR(8010, "operate database failed"),
+
+    ZOOKEEPER_CONNECT_FAILED(8020, "zookeeper connect failed"),
+    ZOOKEEPER_READ_FAILED(8021, "zookeeper read failed"),
+    ZOOKEEPER_WRITE_FAILED(8022, "zookeeper write failed"),
+    ZOOKEEPER_DELETE_FAILED(8023, "zookeeper delete failed"),
+
+    // 调用集群任务里面的agent失败
+    CALL_CLUSTER_TASK_AGENT_FAILED(8030, " call cluster task agent failed"),
+
+    // 调用监控系统失败
+    CALL_MONITOR_SYSTEM_ERROR(8040, " call monitor-system failed"),
+
+    // 存储相关的调用失败
+    STORAGE_UPLOAD_FILE_FAILED(8050, "upload file failed"),
+    STORAGE_FILE_TYPE_NOT_SUPPORT(8051, "File type not support"),
+    STORAGE_DOWNLOAD_FILE_FAILED(8052, "download file failed"),
+
+    ;
+
+    private int code;
+    private String message;
+
+    ResultStatus(int code, String message) {
+        this.code = code;
+        this.message = message;
+    }
+
+    public int getCode() {
+        return code;
+    }
+
+    public void setCode(int code) {
+        this.code = code;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public void setMessage(String message) {
+        this.message = message;
+    }
+}

kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/rd/ResultStatus.java

+package com.xiaojukeji.kafka.manager.common.entity;
+
+import com.xiaojukeji.kafka.manager.common.constant.Constant;
+
+/**
+ * 返回状态
+ * @author zengqiao
+ * @date 20/4/16
+ */
+public enum ResultStatus {
+    GATEWAY_INVALID_REQUEST(-1, "invalid request"),
+
+    SUCCESS(Constant.SUCCESS, "success"),
+
+    FAIL(1, "操作失败"),
+
+    /**
+     * 操作错误[1000, 2000)
+     * ------------------------------------------------------------------------------------------
+     */
+    OPERATION_FAILED(1401, "operation failed"),
+    OPERATION_FORBIDDEN(1402, "operation forbidden"),
+    API_CALL_EXCEED_LIMIT(1403, "api call exceed limit"),
+    USER_WITHOUT_AUTHORITY(1404, "user without authority"),
+    CHANGE_ZOOKEEPER_FORBIDDEN(1405, "change zookeeper forbidden"),
+
+
+    TOPIC_OPERATION_PARAM_NULL_POINTER(1450, "参数错误"),
+    TOPIC_OPERATION_PARTITION_NUM_ILLEGAL(1451, "分区数错误"),
+    TOPIC_OPERATION_BROKER_NUM_NOT_ENOUGH(1452, "Broker数不足错误"),
+    TOPIC_OPERATION_TOPIC_NAME_ILLEGAL(1453, "Topic名称非法"),
+    TOPIC_OPERATION_TOPIC_EXISTED(1454, "Topic已存在"),
+    TOPIC_OPERATION_UNKNOWN_TOPIC_PARTITION(1455, "Topic未知"),
+    TOPIC_OPERATION_TOPIC_CONFIG_ILLEGAL(1456, "Topic配置错误"),
+    TOPIC_OPERATION_TOPIC_IN_DELETING(1457, "Topic正在删除"),
+    TOPIC_OPERATION_UNKNOWN_ERROR(1458, "未知错误"),
+
+    /**
+     * 参数错误[2000, 3000)
+     * ------------------------------------------------------------------------------------------
+     */
+    PARAM_ILLEGAL(2000, "param illegal"),
+    CG_LOCATION_ILLEGAL(2001, "consumer group location illegal"),
+    ORDER_ALREADY_HANDLED(2002, "order already handled"),
+    APP_ID_OR_PASSWORD_ILLEGAL(2003, "app or password illegal"),
+    SYSTEM_CODE_ILLEGAL(2004, "system code illegal"),
+    CLUSTER_TASK_HOST_LIST_ILLEGAL(2005, "主机列表错误，请检查主机列表"),
+    JSON_PARSER_ERROR(2006, "json parser error"),
+
+    BROKER_NUM_NOT_ENOUGH(2050, "broker not enough"),
+    CONTROLLER_NOT_ALIVE(2051, "controller not alive"),
+    CLUSTER_METADATA_ERROR(2052, "cluster metadata error"),
+    TOPIC_CONFIG_ERROR(2053, "topic config error"),
+
+    /**
+     * 参数错误 - 资源检查错误
+     * 因为外部系统的问题, 操作时引起的错误, [7000, 8000)
+     * ------------------------------------------------------------------------------------------
+     */
+    RESOURCE_NOT_EXIST(7100, "资源不存在"),
+    CLUSTER_NOT_EXIST(7101, "cluster not exist"),
+    BROKER_NOT_EXIST(7102, "broker not exist"),
+    TOPIC_NOT_EXIST(7103, "topic not exist"),
+    PARTITION_NOT_EXIST(7104, "partition not exist"),
+    ACCOUNT_NOT_EXIST(7105, "account not exist"),
+    APP_NOT_EXIST(7106, "app not exist"),
+    ORDER_NOT_EXIST(7107, "order not exist"),
+    CONFIG_NOT_EXIST(7108, "config not exist"),
+    IDC_NOT_EXIST(7109, "idc not exist"),
+    TASK_NOT_EXIST(7110, "task not exist"),
+    AUTHORITY_NOT_EXIST(7111, "authority not exist"),
+    MONITOR_NOT_EXIST(7112, "monitor not exist"),
+    QUOTA_NOT_EXIST(7113, "quota not exist, please check clusterId, topicName and appId"),
+    CONSUMER_GROUP_NOT_EXIST(7114, "consumerGroup not exist"),
+    TOPIC_BIZ_DATA_NOT_EXIST(7115, "topic biz data not exist, please sync topic to db"),
+
+    // 资源已存在
+    RESOURCE_ALREADY_EXISTED(7200, "资源已经存在"),
+    TOPIC_ALREADY_EXIST(7201, "topic already existed"),
+
+    // 资源重名
+    RESOURCE_NAME_DUPLICATED(7300, "资源名称重复"),
+
+    // 资源已被使用
+    RESOURCE_ALREADY_USED(7400, "资源早已被使用"),
+
+
+    /**
+     * 因为外部系统的问题, 操作时引起的错误, [8000, 9000)
+     * ------------------------------------------------------------------------------------------
+     */
+    MYSQL_ERROR(8010, "operate database failed"),
+
+    ZOOKEEPER_CONNECT_FAILED(8020, "zookeeper connect failed"),
+    ZOOKEEPER_READ_FAILED(8021, "zookeeper read failed"),
+    ZOOKEEPER_WRITE_FAILED(8022, "zookeeper write failed"),
+    ZOOKEEPER_DELETE_FAILED(8023, "zookeeper delete failed"),
+
+    // 调用集群任务里面的agent失败
+    CALL_CLUSTER_TASK_AGENT_FAILED(8030, " call cluster task agent failed"),
+
+    // 调用监控系统失败
+    CALL_MONITOR_SYSTEM_ERROR(8040, " call monitor-system failed"),
+
+    // 存储相关的调用失败
+    STORAGE_UPLOAD_FILE_FAILED(8050, "upload file failed"),
+    STORAGE_FILE_TYPE_NOT_SUPPORT(8051, "File type not support"),
+    STORAGE_DOWNLOAD_FILE_FAILED(8052, "download file failed"),
+
+    ;
+
+    private int code;
+    private String message;
+
+    ResultStatus(int code, String message) {
+        this.code = code;
+        this.message = message;
+    }
+
+    public int getCode() {
+        return code;
+    }
+
+    public void setCode(int code) {
+        this.code = code;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public void setMessage(String message) {
+        this.message = message;
+    }
+}

kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ldap/LDAPAuthentication.java

 package com.xiaojukeji.kafka.manager.common.utils.ldap;
 
+import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
@@ -25,6 +26,9 @@ public class LDAPAuthentication {
     @Value(value = "${ldap.factory}")
     private String ldapFactory;
 
+    @Value(value = "${ldap.filter}")
+    private String ldapfilter;
+
     @Value(value = "${ldap.auth-user-registration-role}")
     private String authUserRegistrationRole;
 
@@ -61,7 +65,9 @@ public class LDAPAuthentication {
         try {
             SearchControls constraints = new SearchControls();
             constraints.setSearchScope(SearchControls.SUBTREE_SCOPE);
-            NamingEnumeration<SearchResult> en = ctx.search("", "account=" + account, constraints);
+            String filter = "(&(objectClass=*)("+ldapfilter+"=" + account + "))";
+
+            NamingEnumeration<SearchResult> en = ctx.search("", filter, constraints);
             if (en == null || !en.hasMoreElements()) {
                 return "";
             }
@@ -95,7 +101,9 @@ public class LDAPAuthentication {
 
         try {
             String userDN = getUserDN(account,ctx);
-
+            if(ValidateUtils.isBlank(userDN)){
+                return    valide;
+            }
             ctx.addToEnvironment(Context.SECURITY_PRINCIPAL, userDN);
             ctx.addToEnvironment(Context.SECURITY_CREDENTIALS, password);
             ctx.reconnect(null);

kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/sso/BaseSessionSignOn.java

@@ -5,6 +5,7 @@ import com.xiaojukeji.kafka.manager.account.component.AbstractSingleSignOn;
 import com.xiaojukeji.kafka.manager.common.bizenum.AccountRoleEnum;
 import com.xiaojukeji.kafka.manager.common.constant.LoginConstant;
 import com.xiaojukeji.kafka.manager.common.entity.Result;
+import com.xiaojukeji.kafka.manager.common.entity.ResultStatus;
 import com.xiaojukeji.kafka.manager.common.entity.dto.normal.LoginDTO;
 import com.xiaojukeji.kafka.manager.common.entity.pojo.AccountDO;
 import com.xiaojukeji.kafka.manager.common.utils.EncryptUtil;
@@ -44,7 +45,7 @@ public class BaseSessionSignOn extends AbstractSingleSignOn {
     @Override
     public Result<String> loginAndGetLdap(HttpServletRequest request, HttpServletResponse response, LoginDTO dto) {
         if (ValidateUtils.isBlank(dto.getUsername()) || ValidateUtils.isNull(dto.getPassword())) {
-            return null;
+            return Result.buildFailure("Missing parameters");
         }
 
         Result<AccountDO> accountResult = accountService.getAccountDO(dto.getUsername());
@@ -54,7 +55,7 @@ public class BaseSessionSignOn extends AbstractSingleSignOn {
         if(ldapEnabled){
             //去LDAP验证账密
             if(!ldapAuthentication.authenricate(dto.getUsername(),dto.getPassword())){
-                return null;
+                return Result.buildFrom(ResultStatus.LDAP_AUTHENTICATION_FAILED);
             }
 
             if((ValidateUtils.isNull(accountResult) || ValidateUtils.isNull(accountResult.getData())) && authUserRegistration){

kafka-manager-web/src/main/resources/application.yml

@@ -89,6 +89,7 @@ ldap:
   url: ldap://127.0.0.1:389/
   basedn: dc=tsign,dc=cn
   factory: com.sun.jndi.ldap.LdapCtxFactory
+  filter: sAMAccountName
   security:
     authentication: simple
     principal: cn=admin,dc=tsign,dc=cn