diff --git a/pyporter b/pyporter index 9303caf28b4422219ced9fd6f9f283cb37352073..e37c9e32835696a2fc11ae7d71c70abb7c204897 100755 --- a/pyporter +++ b/pyporter @@ -28,10 +28,12 @@ import datetime import argparse import subprocess import os +import platform from pathlib import Path # python3-wget is not default available on openEuler yet. # import wget + url_template = 'https://pypi.org/pypi/{pkg_name}/json' json_file_template = '{pkg_name}.json' name_tag_template = 'Name:\t\tpython-{pkg_name}' @@ -88,19 +90,24 @@ def transform_module_name(n): """ # remove () ns = re.split("[()]", n) - if len(ns) > 1: - m = re.match("([<>=]+)( *)(\d.*)", ns[1]) - ns[1] = m[1] + " " + m[3] + ver_constrain = [] ns[0] = ns[0].strip() if ns[0].startswith("python-"): ns[0] = ns[0].replace("python-", "python3-") - return " ".join(ns) else: ns[0] = "python3-" + ns[0] if ns[0].find("/") != -1 or ns[0].find(".") != -1: return "" - else: - return " ".join(ns) + """ + if len(ns) > 1: + vers = ns[1].split(",") + for ver in vers: + m = re.match("([<>=]+)( *)(\d.*)", ver.strip()) + ver_constrain.append(ns[0] + " " + m[1] + " " + m[3]) + return ", ".join(ver_constrain) + else: + """ + return ns[0] def get_requires(j): @@ -216,8 +223,9 @@ def get_pkg_json(pkg): """ url = url_template.format(pkg_name=pkg) - u = urllib.request.urlopen(url) - resp = json.loads(u.read().decode('utf-8')) + resp = "" + with urllib.request.urlopen(url) as u: + resp = json.loads(u.read().decode('utf-8')) return resp @@ -233,31 +241,30 @@ def download_source(j, tgtpath): return subprocess.call(["wget", s_url, "-P", tgtpath]) -def prepare_rpm_build_env(buildroot): +def prepare_rpm_build_env(root): """ prepare environment for rpmbuild """ + if (os.path.exists(root) == False): + print("Root path %s does not exist\n" & buildroot) + return "" + + buildroot = os.path.join(root, "rpmbuild") if (os.path.exists(buildroot) == False): - print("Build Root path %s does not exist\n", buildroot) - return False + os.mkdir(buildroot) for sdir in ['SPECS', 'BUILD', 'SOURCES', 'SRPMS', 'RPMS', 'BUILDROOT']: bpath = os.path.join(buildroot, sdir) if (os.path.exists(bpath) == False): os.mkdir(bpath) - return True + return buildroot -def try_install_package(pkg): +def try_pip_install_package(pkg): """ install packages listed in build requires """ - print(pkg) - ret = subprocess.call(["rpm", "-qi", pkg]) - if ret == 0: - return True - # try pip installation pip_name = pkg.split("-") if len(pip_name) == 2: @@ -273,10 +280,21 @@ def try_install_package(pkg): # return True -def prepare_dependencies(req_list): - for req in req_list: - if (try_install_package(req) == False): - return req +def package_installed(pkg): + print(pkg) + ret = subprocess.call(["rpm", "-qi", pkg]) + if ret == 0: + return True + + return False + + +def dependencies_ready(req_list): + """ + TODO: do not need to do dependency check here, do it in pyporter_run + """ + # if (try_pip_install_package(req) == False): + # return req return "" def build_package(specfile): @@ -287,17 +305,34 @@ def build_package(specfile): return ret -def build_rpm(j, buildroot): +def build_install_rpm(j, rootpath): + ret = build_rpm(j, rootpath) + if (ret != ""): + return ret + + arch = "noarch" + if (build_noarch == False): + arch = platform.machine() + + pkgname = os.path.join(rootpath, "rpmbuild", "RPMS", arch, "python3-" + j["info"]["name"] + "*") + ret = subprocess.call(["rpm", "-ivh", pkgname]) + if (ret != 0): + return "Install failed\n" + + return "" + +def build_rpm(j, rootpath): """ full process to build rpm """ - if(prepare_rpm_build_env(buildroot) == False): + buildroot = prepare_rpm_build_env(rootpath) + if (buildroot == ""): return False specfile = os.path.join(buildroot, "SPECS", "python-" + j["info"]["name"] + ".spec") req_list = build_spec(j, specfile) - ret = prepare_dependencies(req_list) + ret = dependencies_ready(req_list) if ret != "": print("%s can not be installed automatically, Please handle it" % ret) return ret @@ -320,7 +355,8 @@ def build_spec(resp, output): print() else: sys.stdout = open(output, 'w+') - + + print("%global _empty_manifest_terminate_build 0") print(name_tag_template.format(pkg_name=resp["info"]["name"])) print(version_tag_template.format(pkg_ver=resp["info"]["version"])) print(release_tag_template) @@ -384,8 +420,13 @@ def build_spec(resp, output): print("if [ -d usr/sbin ]; then") print("\tfind usr/sbin -type f -printf \"/%h/%f\\n\" >> filelist.lst") print("fi") + print("touch doclist.lst") + print("if [ -d usr/share/man ]; then") + print("\tfind usr/share/man -type f -printf \"/%h/%f.gz\\n\" >> doclist.lst") + print("fi") print("popd") print("mv %{buildroot}/filelist.lst .") + print("mv %{buildroot}/doclist.lst .") print("") print("%files -n python3-{name} -f filelist.lst".format(name=resp["info"]["name"])) # print("%{python3_sitelib}/*.egg-info/") @@ -397,7 +438,7 @@ def build_spec(resp, output): print("%dir %{python3_sitearch}/*") print("") - print("%files help") + print("%files help -f doclist.lst") print("%{_pkgdocdir}") print("") print("%changelog") @@ -412,12 +453,14 @@ def build_spec(resp, output): if __name__ == "__main__": - dft_root_path=os.path.join(str(Path.home()), "rpmbuild") + dft_root_path=os.path.join(str(Path.home())) parser = argparse.ArgumentParser() parser.add_argument("-s", "--spec", help="Create spec file", action="store_true") + parser.add_argument("-R", "--requires", help="Get required python modules", action="store_true") parser.add_argument("-b", "--build", help="Build rpm package", action="store_true") + parser.add_argument("-B", "--buildinstall", help="Build&Install rpm package", action="store_true") parser.add_argument("-r", "--rootpath", help="Build rpm package in root path", type=str, default=dft_root_path) parser.add_argument("-d", "--download", help="Download source file indicated path", action="store_true") parser.add_argument("-p", "--path", help="indicated path to store files", type=str, default=os.getcwd()) @@ -427,6 +470,16 @@ if __name__ == "__main__": args = parser.parse_args() response = get_pkg_json(args.pkg) + if (response == ""): + print("Get %s module json info failed\n" % args.pkg) + sys.exit(1) + + if (args.requires): + reqlist = get_build_requires(response) + if reqlist is not None: + for req in reqlist: + print(req) + sys.exit(0) if (args.spec): build_spec(response, args.output) @@ -434,7 +487,14 @@ if __name__ == "__main__": if (args.build): ret = build_rpm(response, args.rootpath) if ret != "": - print("BuildRequire : %s" % ret) + print("build failed : BuildRequire : %s\n" % ret) + sys.exit(1) + + if (args.buildinstall): + ret = build_install_rpm(response, args.rootpath) + if ret != "": + print("Build & install failed\n") + sys.exit(1) if (args.download): download_source(response, args.path) diff --git a/pyporter_run b/pyporter_run new file mode 100755 index 0000000000000000000000000000000000000000..26d9cecf21d158a4c775d9ab2c91c21cdbb6a197 --- /dev/null +++ b/pyporter_run @@ -0,0 +1,143 @@ +#!/usr/bin/python3 +""" +This is a packager bot for python modules from pypi.org +""" +#****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: Shinwell_Hu Myeuler +# Create: 2020-05-07 +# Description: This is a helper tools for pyporter +# ******************************************************************************/ + +from pprint import pprint +from os import path +import json +import sys +import re +import datetime +import argparse +import subprocess +import os +from pathlib import Path +import queue + + +def pkg_installed(pkg): + #ret = subprocess.call(["rpm", "-qi", pkg], stdout=subprocess.PIPE) + ret = subprocess.call(["rpm", "-qi", pkg]) + if (ret == 0): + return True + + # try to install it + ret = subprocess.call(["yum", "install", "-y", pkg]) + if (ret == 0): + return True + + return False + +def circle_dep(pkg, prepare): + for ppg in prepare: + if (ppg == pkg): + return True + return False + +def issue_analysis(prepare): + while (len(prepare) != 0): + pkg = prepare.pop(0) + bpkg = pkg.replace("python3-", "") + deps = get_deps(bpkg) + if (deps[1] == False): + print("Get module %s failed\n" % pkg) + return False + if (len(deps[0]) == 0): + ret = build_install_pkg(pkg) + if (ret == False): + return False + else: + # + # push back to stack + # + prepare.insert(0, pkg) + for req in deps: + if (circle_dep(req, prepare)): + print("There is circle dependency") + print(prepare) + print(bpkg) + return False + else: + prepare.insert(0, req) + + return True + + + +def do_prepare_job(pkgs): + pkg_prepare = [] + + for pkg in pkgs: + pkg_prepare.append(pkg) + + if (len(pkg_prepare) == 0): + return True + + return issue_analysis(pkg_prepare) + +def build_install_pkg(pkg): + print("Build&Install : %s\n" % pkg) + + bpkg = pkg.replace("python3-", "") +# ret = subprocess.call(["./pyporter", "-B", bpkg], stdout=subprocess.PIPE) + ret = subprocess.call(["./pyporter", "-B", bpkg]) + if (ret != 0): + print(" Build&Install package %s failed\n" % pkg) + return False + + return True + +def get_deps(pkg): + needed = [] + proc = subprocess.Popen(["./pyporter", "-R", pkg], stdout=subprocess.PIPE) + + #if (proc.returncode != 0): + # return (needed, False) + + while (True): + line = proc.stdout.readline() + if not line: + break; + newpkg = str(line.strip().decode()) + if (pkg_installed(newpkg) == False): + needed.append(newpkg) + proc.stdout.close() + proc.wait() + + return (needed, True) + + +if __name__ == "__main__": + ret = True + + parser = argparse.ArgumentParser() + + parser.add_argument("pkg", type=str, help="The python module name") + + args = parser.parse_args() + + reqs = get_deps(args.pkg) + if (reqs[1] == False): + print("Get deps failed\n") + sys.exit(1) + if (len(reqs[0]) != 0): + ret = do_prepare_job(reqs[0]) + + if (ret == True): + build_install_pkg(args.pkg) +