增加子库，增加自动化下载脚本，补充app_preference.hpp文件

.gitmodules

+[submodule "inner-depend/links/protobuf_zcmf_reply"]
+	path = inner-depend/links/protobuf_zcmf_reply
+	url = git@codechina.csdn.net:xiaonuo911teamo/protobuf_zcmf_reply.git
+[submodule "inner-depend/links/zeromq_zcmf_reply"]
+	path = inner-depend/links/zeromq_zcmf_reply
+	url = git@codechina.csdn.net:xiaonuo911teamo/zeromq_zcmf_reply.git

build.sh

@@ -41,9 +41,9 @@ function upload() {
                 cp ../scripts/version_depend_loc.json version_depend.json
                 ./integration.py -c u -$TYPE_CHAR
         elif [ "$1" == "thd" ]; then
-                cd inner-depend/Localization_3rdParty
+                cd inner-depend/3rdParty
                 if [ ! -d "./output" ]; then
-                        ln -s ../Localization_3rdParty output
+                        ln -s ../3rdParty output
                 fi
                 cp $current_dir/integration.py integration.py
                 cp $current_dir/scripts/version_depend_thd.json version_depend.json
@@ -85,7 +85,7 @@ function build_all() {
 }
 
 function gen_proto() {
-        protoc=$current_dir/inner-depend/Localization_3rdParty/linux-x86_64/protobuf/bin/protoc
+        protoc=$current_dir/inner-depend/3rdParty/linux-x86_64/protobuf/bin/protoc
         protofile=`ls $current_dir/src/proto_data/proto/*.proto`
         input=" -I=$current_dir/src/proto_data/proto "
         output="--cpp_out=$current_dir/src/proto_data/data"

code/CMakeLists.txt

@@ -16,7 +16,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../cmake)
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../opt/${BUILD_TYPE}/bin)
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../opt/${BUILD_TYPE}/lib)
 
-set(THIRD_PARTY_DIR ${CMAKE_CURRENT_LIST_DIR}/../inner-depend/Localization_3rdParty/${BUILD_TYPE} CACHE INTERNAL "THIRD_PARTY_DIR")
+set(THIRD_PARTY_DIR ${CMAKE_CURRENT_LIST_DIR}/../inner-depend/3rdParty/${BUILD_TYPE} CACHE INTERNAL "THIRD_PARTY_DIR")
 
 set(CORELIB_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/src/corelib/include/ CACHE INTERNAL "CORELIB_INCLUDE_DIR")
 set(DATA_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/src/proto_data/ CACHE INTERNAL "DATA_INCLUDE_DIR")

code/src/corelib/include/utils/app_preference.hpp

+#pragma once
+#include <mutex>
+#include <map>
+#include <log/logging.h>
+#define appPref AppPreference::get_instance()
+
+class AppPreference {
+public:
+    static AppPreference& get_instance() {
+        static AppPreference app_preference;
+        return app_preference;
+    }
+
+protected:
+    AppPreference(){
+    };
+private:
+    std::map<std::string, std::string> strings;
+    std::map<std::string, int> ints;
+    std::map<std::string, long> longs;
+    std::map<std::string, float> floats;
+    std::map<std::string, double> doubles;
+    std::map<std::string, std::vector<double> > vec_doubles;
+public:
+    std::string get_string_data(const std::string& key) {
+        if (strings.find(key) == strings.end()) {
+            ERROR() << "String data has no key named " << key;
+            throw std::runtime_error("key_not_found");
+        }
+
+        return strings[key];
+    }
+    int get_int_data(const std::string& key) {
+        if (ints.find(key) == ints.end()) {
+            ERROR() << "Int data has no key named " << key;
+            throw std::runtime_error("key_not_found");
+        }
+
+        return ints[key];
+    }
+    long get_long_data(const std::string& key) {
+        if (longs.find(key) == longs.end()) {
+            ERROR() << "Long data has no key named " << key;
+            throw std::runtime_error("key_not_found");
+        }
+
+        return longs[key];
+    }
+
+    float get_float_data(const std::string& key) {
+        if (floats.find(key) == floats.end()) {
+            ERROR() << "Float data has no key named " << key;
+            throw std::runtime_error("key_not_found");
+        }
+
+        return floats[key];
+    }
+
+    double get_double_data(const std::string& key) {
+        if (doubles.find(key) == doubles.end()) {
+            ERROR() << "Double data has no key named " << key;
+            throw std::runtime_error("key_not_found");
+        }
+
+        return doubles[key];
+    }
+
+    std::vector<double> get_double_vector_data(const std::string& key) {
+        if (vec_doubles.find(key) == vec_doubles.end()) {
+            ERROR() << "Double vector data has no key named " << key;
+            throw std::runtime_error("key_not_found");
+        }
+        return vec_doubles[key];
+    }
+
+    void set_string_data(const std::string& key, const std::string& value) {
+        strings[key] = value;
+    }
+    void set_int_data(const std::string& key, int value) {
+        ints[key] = value;
+    }
+    void set_long_data(const std::string& key, long value) {
+        longs[key] = value;
+    }
+    void set_float_data(const std::string& key, float value) {
+        floats[key] = value;
+    }
+    void set_double_data(const std::string& key, double value) {
+        doubles[key] = value;
+    }
+    void set_double_vector_data(const std::string& key, std::vector<double> value) {
+        vec_doubles[key] = value;
+    }
+
+    bool has_string_key(std::string key) {
+        return strings.find(key) != strings.end();
+    }
+    bool has_int_key(std::string key) {
+        return ints.find(key) != ints.end();
+    }
+    bool has_long_key(std::string key) {
+        return longs.find(key) != longs.end();
+    }
+    bool has_float_key(std::string key) {
+        return floats.find(key) != floats.end();
+    }
+    bool has_double_key(std::string key) {
+        return doubles.find(key) != doubles.end();
+    }
+    bool has_double_vector_key(std::string key) {
+        return vec_doubles.find(key) != vec_doubles.end();
+    }
+};
+

code/src/corelib/include/utils/dl_utils.hpp

@@ -144,14 +144,5 @@ private:
         } else {
             WARNING() << "using default log level: INFO";
         }
-        if (appPref.has_string_key("ime_log.ime_log_level")){
-            int log_level = std::stoi(appPref.get_string_data("ime_log.ime_log_level"));
-            bool level_check = log_level < LogLevel::INFO && log_level > LogLevel::FATAL;
-            log_level = level_check ? LogLevel::DIRECT : log_level;
-            ImeLogInterface::set_log_level((LogLevel)log_level);
-            DIRECT() << "using log level: " << ImeLogInterface::log_level_to_string((LogLevel)log_level);
-        } else {
-            WARNING() << "using default log level: INFO";
-        }
     }
 };

code/src/server_proc/src/plugin.cpp

@@ -6,7 +6,6 @@ PipeController g_controllor;
 extern "C"{
 
 void LOAD_PLUGIN(){
-    ServerData::initial();
     g_controllor.add_element<ServerProcTask>();
 }
 

protobuf_zcmf_reply @ f96e3fc2

+Subproject commit f96e3fc2991ed7f04fb1c9485ebcce6c143cdfd0

zeromq_zcmf_reply @ 963767d5

+Subproject commit 963767d5c11f776b42b6366c2f794024928405aa

integration.py

+#!/usr/bin/python
+########################################################
+#
+# Copyright 2019 Baidu Inc. All Rights Reserved.
+#
+########################################################
+"""
+integration
+"""
+import os
+import sys
+import tarfile
+import socket
+import urllib
+# global setting
+g_root_path = os.path.split(os.path.realpath(__file__))[0]
+g_tool_file = os.path.split(os.path.realpath(__file__))[1]
+
+def error_message(msg):
+    """Print the error message in red."""
+    print("\033[1;31m[ERROR] %s\033[0m" % msg)
+def info_message(msg):
+    """Print the info message in green."""
+    print("\033[1;32m[INFO] %s\033[0m" % msg)
+    # -1:failed 0:success 1:no such file
+
+class Downloader(object):
+    """Downloader"""
+
+    def __init__(self):
+        """init"""
+        self.__root_path = ""
+        self.__download_path = ""
+        self.__download_platforms = []
+        self.__ftp_client = FtpClient(g_ftp_host, g_ftp_port, g_ftp_username, g_ftp_password)
+
+    def __del__(self):
+        """delete"""
+        del self.__ftp_client
+    
+    def download(self):
+        """download depends"""
+        ret = 0
+        # parse arg
+        if self.__parse_arg() != 0:
+            ret = -1
+        else:
+            self.__print_env()
+        # get need modules
+        parent_module_name = [""]
+        parent_version_id = [""]
+        need_modules = {}
+        if ret == 0:
+            if self.__valid_version_depend(self.__root_path + "/" + g_json_file_name) != 0:
+                ret = -1
+            else:
+                self.__get_need_modules(parent_module_name, parent_version_id, need_modules)
+        # make downlaod path
+        if ret == 0:
+            if self.__make_dir(self.__download_path) != 0:
+                ret = -1
+        # download modules
+        if ret == 0:
+            for platform in self.__download_platforms:
+                info_message("**** download " + platform + " ****")
+                self.downloaded_map = {}
+                self.downloading_statck = []
+                self.downloading_statck.append(parent_module_name[0])
+                has_error = 0
+                for key in need_modules:
+                    if self.__download_module(key, need_modules[key],
+                        parent_module_name[0], parent_version_id, platform) != 0:
+                        has_error = 1
+                        break
+                if has_error == 1:
+                    ret = -1
+                    break
+        # last log
+        if ret == 0:
+            info_message("==== success ====")
+        else:
+            info_message("==== failed ====")
+        return ret
+
+    def __get_need_modules(self, parent_module_name, parent_version_id, need_modules):
+        parent_module_name[0] = self.version_depend_obj["module"]
+        parent_version_id[0] = "v_" + self.version_depend_obj["version"] + "_" + self.version_depend_obj["branch"]
+        module_depends = self.version_depend_obj["depends"]
+        for i in module_depends:
+            need_modules[i["name"]] = i["link"]
+
+    def __valid_module_name(self, module_name):
+        ret = 0
+        if module_name not in g_supported_modules:
+            ret = -1
+        if ret != 0:
+            supported_modules = ""
+            for i in g_supported_modules:
+                supported_modules += " " + i
+            error_message("module name error: " + module_name + " (supported modules:" + supported_modules + ")")
+        return ret
+
+    def __valid_version(self, version):
+        ret = 0
+        number_list = version.split(".")
+        if len(number_list) == 4:
+            for i in number_list:
+                if not i.isdigit():
+                    ret = -1
+                    break
+        else:
+            ret = -1
+        if ret != 0:
+            error_message("version format error: " + version + " (eg: 1.0.0.0)")
+        return ret
+
+    def __valid_branch(self, branch):
+        ret = 0
+        if branch == "":
+            ret = -1
+        else:
+            search_obj = re.search("[^a-zA-Z0-9_-]", branch)
+            if search_obj:
+                ret = -1
+        if ret != 0:
+            error_message("branch format error: " + branch + " (regular expression: [a-zA-z0-9_-])")
+        return ret
+
+    def __valid_version_id(self, version_id):
+        ret = 0
+        if version_id not in g_branch_list:
+            part_list = version_id.split("_")
+            if len(part_list) < 3:
+                ret = -1
+            else:
+                if part_list[0] != "v" or self.__valid_version(part_list[1]) != 0:
+                    ret = -1
+                else:
+                    index = len(part_list[0]) + len(part_list[1]) + 2
+                    if self.__valid_branch(version_id[index:]) != 0:
+                        ret = -1
+        if ret != 0:
+            error_message("version_id format error: " + version_id + " (eg: v_1.0.0.0_MyBranch, develop or stable)")
+        return ret
+
+    def __valid_version_depend(self, json_file):
+        ret = 0
+        info_message("read " + json_file + " ...")
+        json_str = [""]
+        if self.__read_json_file(json_file, json_str) != 0:
+            ret = -1
+        else:
+            try:
+                json_obj = json.loads(json_str[0])
+                if self.__valid_module_name(json_obj["module"]) != 0:
+                    ret = -1
+                if self.__valid_version(json_obj["version"]) != 0:
+                    ret = -1
+                if self.__valid_branch(json_obj["branch"]) != 0:
+                    ret = -1
+                if self.__valid_branch(json_obj["link"]) != 0:
+                    ret = -1
+                module_depends = json_obj["depends"]
+                for i in module_depends:
+                    if self.__valid_module_name(i["name"]) != 0:
+                        ret = -1
+                    if self.__valid_version_id(i["version_id"]) != 0:
+                        ret = -1
+                if ret == 0:
+                    self.version_depend_obj = json_obj
+            except:
+                ret = -1
+                error_message("version_depend format error: " + json_file)
+        return ret
+
+    def __make_dir(self, path):
+        ret = 0
+        try:
+            if not os.path.exists(path):
+                os.makedirs(path)
+        except:
+            ret = -1
+            error_message("make directory failed: " + path)
+        return ret
+
+    def __remove_path(self, path):
+        ret = 0
+        try:
+            if os.path.exists(path):
+                info_message("remove " + path + " ...")
+                shutil.rmtree(path)
+        except:
+            ret = -1
+            error_message("remove " + path + " failed")
+        return ret
+
+    def __remove_file(self, file_path):
+        if os.path.isfile(file_path):
+            info_message("remove " + file_path + " ...")
+            os.remove(file_path)
+
+
+    def __get_module_depends(self, json_file, module_depends):
+        ret = 0
+        info_message("read " + json_file + " ...")
+        # to read json file
+        json_str = [""]
+        if self.__read_json_file(json_file, json_str) != 0:
+            ret = -1
+        # parse json data
+        if ret == 0:
+            try:
+                json_obj = json.loads(json_str[0])
+                module_depends += json_obj["depends"]
+            except:
+                ret = -1
+                error_message("json format error: " + json_file)
+        return ret
+
+    def __download_module_depends(self, json_file, parent_module_name, parent_version_id, platform):
+        ret = 0
+        module_depends = []
+        if self.__get_module_depends(json_file, module_depends) != 0:
+            ret = -1
+        if ret == 0:
+            for module in module_depends:
+                if self.__download_module(module["name"], \
+                    module["version_id"], parent_module_name, parent_version_id, platform) != 0:
+                    ret = -1
+                    break
+        return ret
+    def __download_platform(self, module_name, link, platform):
+            file_path = # TODO: 通过version_depend.json获取
+            print "downloading " + g_depends[moudle_name]
+            download_file(file_path, link)
+            print "done. saved to " + file_path
+            target_path = # TODO: 
+            print "tar file ..."
+            __tar_file(file_path, target_path)
+
+    def __download_module(self, module_name, link, parent_module_name, parent_version_id, platform):
+        ret = 0
+        info_message("download " + module_name + ": " + link + " " + platform + " ...")
+        # check is downloaded
+        is_download = 0
+        if ret == 0:
+            if module_name in self.downloaded_map:
+                is_download = 1
+                info_message(module_name + "-" + actual_version_id[0] + " already downloaded")
+            else:
+                self.downloading_statck.append(module_name)
+        # download
+        if ret == 0 and is_download == 0:
+            if self.__download_platform(module_name, link, platform) != 0:
+                ret = -1
+            else:
+                json_file = self.__download_path + "/" + module_name + "/" + platform + "/" + g_json_file_name
+                ret = self.__download_module_depends(json_file, module_name, actual_version_id[0], platform) # recursion
+            # update downloading statck
+            if ret == 0:
+                self.downloaded_map[module_name] = module
+                self.downloading_statck.pop()
+        return ret
+
+    def __get_actual_version_id(self, module_name, version_id, actual_version_id):
+        ret = 0
+        actual_version_id[0] = version_id
+        if version_id in g_branch_list:
+            local_file = self.__download_path + "/" + module_name + "-" + g_tag_file
+            remote_file = g_ftp_root + "/" + module_name + "/" + module_name + "-" + g_tag_file
+            if self.__ftp_client.download_file(local_file, remote_file) != 0:
+                ret = -1
+                error_message("download failed: no such file \"" + remote_file + "\"")
+                self.__remove_file(local_file)
+            if ret == 0:
+                json_str = [""]
+                if self.__read_json_file(local_file, json_str) != 0:
+                    ret = -1
+                else:
+                    try:
+                        json_obj = json.loads(json_str[0])
+                        actual_version_id[0] = "v_" + json_obj[version_id] + "_" + version_id
+                    except:
+                        ret = -1
+                        error_message("json format error: " + local_file)
+            if ret == 0:
+                info_message("current " + module_name + "`s " + version_id + " version is " + actual_version_id[0])
+        return ret
+
+    def __check_module_conflict(self, module):
+        ret = 0
+        curr_module = module.module_name
+        # check recycle depends
+        recycle_list = []
+        for i in self.downloading_statck:
+            if i == curr_module:
+                ret = -1
+            if ret != 0:
+                recycle_list.append(i)
+        if ret != 0:
+            err_info = "recycle depend: "
+            for i in recycle_list:
+                err_info += "->" + i
+            error_message(err_info)
+        # check version conflict
+        if ret == 0:
+            if curr_module in self.downloaded_map:
+                if self.downloaded_map[curr_module].version_id != module.version_id:
+                    ret = -1
+                    old_module = self.downloaded_map[curr_module]
+                    error_message("version conflict: " + \
+                        module.parent_module_name + "->" + \
+                        curr_module + "_" + module.version_id + ", " + \
+                        old_module.parent_module_name + "->" + \
+                        old_module.module_name + "_" + old_module.version_id)
+        return ret
+
+    def __tar_file(self, file_path, target_path):
+        ret = 0
+        info_message("tar " + file_path + " ...")
+        try:
+            tar = tarfile.open(file_path, "r:gz")
+            try:
+                tar.extractall(target_path)
+            except:
+                ret = -1
+                error_message("tar " + file_path + " failed")
+            finally:
+                tar.close()
+        except:
+            ret = -1
+            error_message("open " + file_path + " failed")
+        return ret
+
+    def __read_json_file(self, json_file, json_str):
+        ret = 0
+        try:
+            file = open(json_file, "r")
+            try:
+                json_str[0] = file.read()
+            except:
+                ret = -1
+                error_message("read " + json_file + " failed")
+            finally:
+                file.close()
+        except:
+            ret = -1
+            error_message("open " + json_file + " failed")
+        return ret
+
+    def __print_env(self):
+        info_message("================ env bengin ================")
+        info_message("root path: " + self.__root_path)
+        info_message("download path: " + self.__download_path)
+        tmp_str = ",".join(self.__download_platforms)
+        info_message("download platforms: " + tmp_str)
+        info_message("================ env end    ================")
+
+    def __parse_arg(self):
+        ret = 0
+        try:
+            opts, args = getopt.getopt(sys.argv[1:], "R:npqlvahA")
+            for op, value in opts:
+                if op == "-R":
+                    self.__root_path = value
+                    self.__download_path = self.__root_path + "/inner-depend"
+                elif op == "-n":
+                    self.__download_platforms.append("linux-x86_64")
+                elif op == "-p":
+                    self.__download_platforms.append("linux-aarch64")
+                elif op == "-q":
+                    self.__download_platforms.append("qnx-aarch64")
+                elif op == "-l":
+                    self.__download_platforms.append("linaro-aarch64")
+                elif op == "-v":
+                    self.__download_platforms.append("qnx-x86_64")
+                elif op == "-a":
+                    self.__download_platforms.append("poky-aarch32")
+                elif op == "-A":
+                    self.__download_platforms.append("poky-aarch64")
+                elif op == "-h":
+                    ret = -1
+                    break
+        except getopt.GetoptError as error:
+            ret = -1
+            error_message(error)
+        # check arguments
+        if ret == 0:
+            if self.__root_path == "":
+                ret = -1
+                error_message("argument error: root path is empty")
+            if len(self.__download_platforms) == 0:
+                ret = -1
+                error_message("argument error: download platforms is empty")
+        if ret != 0:
+            self.__usage()
+        return ret
+
+    def __usage(self):
+        """print usage"""
+        info_message("================ usage bengin ================")
+        info_message("-R: root path")
+        info_message("-n: platform option, linux-x86_64,  n = native")
+        info_message("-p: platform option, linux-aarch64, p = px2")
+        info_message("-q: platform option, qnx-aarch64,   q = qnx")
+        info_message("-l: platform option, linaro-aarch64,   l = linaro")
+        info_message("-v: platform option, qnx-x86_64,    v = vmware")
+        info_message("-a: platform option, poky-aarch32,  a = poky")
+        info_message("-h: help")
+        info_message("example:")
+        info_message("    ./download-depend.py -R . -n")
+        info_message("    ./download-depend.py -R . -npqv")
+        info_message("================ usage end    ================")
+
+
+# entry
+inst = Downloader()
+inst.download()
+
+
+def download_file(self, local_file, remote_file):
+    """download file"""
+    ret = 0
+    urllib.urlretrieve(remote_file, local_file)
+    return ret
+
+class Integration(object):
+    """integration"""
+    def __init__(self):
+        """init"""
+        self.__version = "1.0.0.0"
+    def integrate(self):
+        # print tips
+        self.__print_tips()
+        """integrate"""
+        # update
+        log_failed = 1
+        # download depends
+        for moudle_name in g_depends:
+            file_path = # TODO: 通过version_depend.json获取
+            print "downloading " + g_depends[moudle_name]
+            download_file(file_path, g_depends[moudle_name])
+            print "done. saved to " + file_path
+            target_path = # TODO: 
+            print "tar file ..."
+            __tar_file(file_path, target_path)
+    def __remove_file(self, file_path):
+        if os.path.isfile(file_path):
+            os.remove(file_path)
+    def __make_dir(self, path):
+        ret = 0
+        try:
+            if not os.path.exists(path):
+                os.makedirs(path)
+        except:
+            ret = -1
+            error_message("make directory failed: " + path)
+        return ret
+    def __tar_file(self, file_path, target_path):
+        ret = 0
+        try:
+            tar = tarfile.open(file_path, "r:gz")
+            try:
+                tar.extractall(target_path)
+            except:
+                ret = -1
+                error_message("tar " + file_path + " failed")
+            finally:
+                tar.close()
+        except:
+            ret = -1
+            error_message("open " + file_path + " failed")
+        return ret
+    def __print_tips(self):
+        info_message("**** " + g_tool_file + " (" + self.__version + ") ****")
+# entry
+inst = Integration()
+inst.integrate()