diff --git a/advisors/check_source_url.py b/advisors/check_source_url.py new file mode 100755 index 0000000000000000000000000000000000000000..4a2510b45b4e3ab7c70dbe3b4f73de72af26b8a2 --- /dev/null +++ b/advisors/check_source_url.py @@ -0,0 +1,212 @@ +#!/usr/bin/python3 +#****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# +# ******************************************************************************/ +""" +This is an automatic script for checking source url of package +""" + +from pyrpm.spec import Spec, replace_macros +import argparse +import gitee +import os +import re +import sys +import yaml +import subprocess +import threading +import math + + +def check_repo(repo, branch, batch_num): + """ + Check source url of multi-packages in repo like src-openeuler. + batch_num is batch num of one thread + """ + gt = gitee.Gitee() + repo_info = gt.get_community(repo) + if not repo_info: + print("WARNING: {repo}.yaml can't be found in community.".format(repo=repo)) + sys.exit(1) + repo_yaml = yaml.load(repo_info, Loader=yaml.Loader) + repo_list = repo_yaml.get("repositories") + thread_num = math.ceil(len(repo_list)/batch_num) + threads = [] + lock = threading.Lock() + for number in range(thread_num): + thread= threading.Thread(target=check_batch, args=(repo_list, branch, number, batch_num, lock)) + threads.append(thread) + for t in threads: + t.start() + for t in threads: + t.join() + print("Ending check.") + + +def create_issue(repo, title, body): + """ + Create issue for repo + """ + gt = gitee.Gitee() + created_issue = False + issues = gt.get_issues(repo) + for issue in issues: + if issue["title"] == title: + created_issue = True + break + if not created_issue: + gt.post_issue(repo, title, body) + + +def check_batch(repo_list, branch, number, batch_num, lock): + """ + Check source url in one batch + """ + gt = gitee.Gitee() + body_source = """Dear Maintainer: + Due to source url problem checked by openEuler-Advisor, please solve it as soon as + possible and check other branch too. + If any problem, please create issue in https://gitee.com/openeuler/openEuler-Advisor. + Thanks. + Yours openEuler-Advisor""" + body_spec = """Dear Maintainer: + Due to spec can't be found by openEuler-Advisor, please add it as soon as possible + and check other branch too. + If any problem, please create issue in https://gitee.com/openeuler/openEuler-Advisor. + Thanks. + Yours openEuler-Advisor""" + file_str = "result_" + "{num}.log".format(num=number) + file_name = open(file_str, "w+") + for n in range(batch_num): + index = number * batch_num + n + if index < len(repo_list): + repo_name = repo_list[index].get("name") + else: + break + file_name.writelines("\n-----------------------Checking {repo}-----------------------".format(repo=repo_name)) + lock.acquire() + spec_str = gt.get_spec(repo_name, branch) + lock.release() + if not spec_str: + file_name.writelines("WARNING: {repo}.spec can't be found on {br}".format(repo=repo_name, br=branch)) + title = "Submit spec file into this repository" + ####create_issue(repo_name, title, body_spec) + continue + + repo_spec = Spec.from_string(spec_str) + if repo_spec.sources_dict: + source = replace_macros(repo_spec.sources[0], repo_spec) + else: + title = "Source url can't be found in spec on {br}".format(br=branch) + file_name.writelines("WARNING: {content}".format(content=title)) + create_issue(repo_name, title, body_source) + continue + + if re.search(r"%{.*?}", source): + title = "Source url can't be parsed with extra macros in spec on {br}.".format(br=branch) + file_name.writelines("WARNING: {content}".format(content=title)) + create_issue(repo_name, title, body_source) + continue + elif source.startswith("http") or source.startswith("ftp"): + fn = os.path.basename(source) + n = 0 + lock.acquire() + while n < 2: + n += 1 + if subprocess.call(["curl -m 600 -L {url} -o {name}".format(url=source, name=fn)], shell=True): + continue + else: + break + lock.release() + title = "Source url may be wrong in spec on {br}.".format(br=branch) + if os.path.exists(fn): + if subprocess.call(["tar -tvf {file_name} &>/dev/null".format(file_name=fn)], shell=True): + file_name.writelines("WARNING: {content}".format(content=title)) + create_issue(repo_name, title, body_source) + else: + file_name.writelines("Check successfully.") + subprocess.call(["rm -rf {file_name}".format(file_name=fn)], shell=True) + else: + file_name.writelines("WARNING: {content}".format(content=title)) + create_issue(repo_name, title, body_source) + else: + title = "Source url is invalid in spec on {br}.".format(br=branch) + file_name.writelines("WARNING: {content}".format(content=title)) + create_issue(repo_name, title, body_source) + continue + + +def check_pkg(pkg, branch): + """ + Check source url of single package + """ + gt = gitee.Gitee() + print("\n-----------------------Checking {repo}-----------------------".format(repo=pkg)) + spec_str = gt.get_spec(pkg, branch) + if not spec_str: + print("WARNING: {repo}.spec can't be found on {branch}".format(repo=pkg, branch=branch)) + created_issue = False + issues = gt.get_issues(pkg) + for issue in issues: + if issue["title"] == "Submit spec file into this repository": + created_issue = True + break + if not created_issue: + title = "Submit spec file into this repository" + print(title) + sys.exit(1) + repo_spec = Spec.from_string(spec_str) + if repo_spec.sources_dict: + source = replace_macros(repo_spec.sources[0], repo_spec) + else: + print("No source url") + sys.exit(1) + if re.search(r"%{.*?}", source): + print("WARNING: Extra macros in source url which failed to be expanded.") + sys.exit(1) + elif source.startswith("http") or source.startswith("ftp"): + fn = os.path.basename(source) + n = 0 + while n < 2: + n += 1 + if subprocess.call(["curl -m 600 -L {url} -o {name}".format(url=source, name=fn)], shell=True): + continue + else: + break + + if os.path.exists(fn): + if subprocess.call(["tar -tvf {file_name} &>/dev/null".format(file_name=fn)], shell=True): + print("WARNING: source url of {repo} may be wrong.".format(repo=pkg)) + else: + print("Check successfully.") + else: + print("WARNING: source url of {repo} may be wrong.".format(repo=pkg)) + else: + print("WARNING: Not valid URL for Source code.") + sys.exit(1) + + +if __name__ == "__main__": + pars = argparse.ArgumentParser() + pars.add_argument("-r", "--repo", type=str, help="The repository to be check.") + pars.add_argument("-p", "--pkg", type=str, help="The package to be check.") + pars.add_argument("-b", "--batch_num", type=int, default=100, help="The batch num of one task.") + pars.add_argument("branch", type=str, help="The branch to be check.") + args = pars.parse_args() + + if args.repo: + check_repo(args.repo, args.branch, args.batch_num) + elif args.pkg: + check_pkg(args.pkg, args.branch) + else: + print("WARNING: Please specify what to be checkd.") + sys.exit(1) diff --git a/advisors/gitee.py b/advisors/gitee.py index 9732435f799064bf0b0350da50d9194ec8252056..740c2879c9863b665a70b40d952bea47f1667442 100755 --- a/advisors/gitee.py +++ b/advisors/gitee.py @@ -13,6 +13,7 @@ import re import sys import os.path import json +import base64 import pprint from datetime import datetime @@ -33,7 +34,6 @@ class Gitee(object): self.yamlfile_url_template = self.src_openeuler_url + "{package}.yaml" #self.advisor_url_template = "https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/{package}.yaml" self.advisor_url_template = self.advisor_url + "upstream-info/{package}.yaml" - self.community_url_template = self.gitee_url + "openeuler/community/raw/master/repository/{repository}.yaml" #self.specfile_exception_url = "https://gitee.com/openeuler/openEuler-Advisor/raw/master/helper/specfile_exceptions.yaml" self.specfile_exception_url = self.advisor_url + "advisors/helper/specfile_exceptions.yaml" self.version_exception_url = self.advisor_url + "advisors/helper/version_exceptions.yaml" @@ -127,11 +127,13 @@ class Gitee(object): """ Get and load gitee json response """ + json_resp = [] headers = self.headers.copy() - #headers = {} headers["Content-Type"] = "application/json;charset=UTF-8" resp = self.get_gitee(url, headers) - return json.loads(resp) + if resp: + json_resp = json.loads(resp) + return json_resp def get_spec_exception(self): """ @@ -185,9 +187,11 @@ class Gitee(object): """ Get yaml data from community repo """ - yamlurl = self.community_url_template.format(repository=repo) - resp = self.get_gitee(yamlurl) - return resp + yamlurl = "https://gitee.com/api/v5/repos/openeuler/community/contents/"\ + "repository/{repo}.yaml".format(repo=repo) + resp = self.get_gitee_json(yamlurl) + resp_str = base64.b64decode(resp["content"]) + return resp_str def get_issues(self, pkg, prj="src-openeuler"): """ diff --git a/advisors/simple-update-robot.py b/advisors/simple-update-robot.py index 8cc4b48f89cab06b5c30566c697c768e73ab94aa..35e95b6a78f71f026c949c386b3eb5eede1f4982 100755 --- a/advisors/simple-update-robot.py +++ b/advisors/simple-update-robot.py @@ -31,7 +31,7 @@ def download_source_url(spec, o_ver, n_ver): """ Download source file from Source or Source0 URL """ - source = replace_macros(spec.sources[0], spec).replace(o_ver, n_ver) + source = replace_macros(spec.sources[0], spec).replace(o_ver, n_ver) if re.match(r"%{.*?}", source): print("WARNING: Extra macros in URL which failed to be expanded") return False @@ -268,7 +268,7 @@ def auto_update_repo(gt, u_repo, u_branch): else: print("WARNING: Please check branch to upgrade.") sys.exit(1) - + pkg_info = yaml.load(repo_yaml, Loader=yaml.Loader) pkg_list = pkg_info.get("repositories") for pkg in pkg_list: