data_import.py

#!/usr/bin/python3
"""
Description: Initialization of data import
            Import the data in the sqlite database into the mysql database
Class: InitDataBase,MysqlDatabaseOperations,SqliteDatabaseOperations
"""
import os
import pathlib
import yaml
from sqlalchemy.exc import SQLAlchemyError, InternalError
from packageship.libs.dbutils.sqlalchemy_helper import DBHelper
from packageship.libs.exception import ContentNoneException
from packageship.libs.exception import DatabaseRepeatException
from packageship.libs.exception import Error
from packageship.libs.exception import ConfigurationException
from packageship.libs.configutils.readconfig import ReadConfig
from packageship.libs.log import Log
from packageship.application.models.package import SrcPack
from packageship.application.models.package import BinPack
from packageship.application.models.package import BinRequires
from packageship.application.models.package import SrcRequires
from packageship.application.models.package import BinProvides
from packageship.application.models.package import BinFiles
from packageship.application.models.package import Packages
from packageship import system_config

LOGGER = Log(__name__)


class InitDataBase():
    """
    Description: Database initialization, generate multiple databases and data
                 based on configuration files

    Attributes:
        config_file_path: configuration file path
        config_file_datas: initialize the configuration content of the database
        db_type: type of database
    """

    def __init__(self, config_file_path=None):
        """
        Description: Class instance initialization

        Args:
            config_file_path: Configuration file path
        """
        self.config_file_path = config_file_path

        if self.config_file_path:
            # yaml configuration file content
            self.config_file_datas = self.__read_config_file()
        self._read_config = ReadConfig(system_config.SYS_CONFIG_PATH)
        self.db_type = 'sqlite'
        self.sql = None
        self._database = None
        self._sqlite_db = None
        self._database_engine = {
            'sqlite': SqliteDatabaseOperations,
            'mysql': MysqlDatabaseOperations
        }
        self.database_name = None
        self._tables = ['src_pack', 'bin_pack',
                        'bin_requires', 'src_requires', 'bin_provides', 'bin_files']
        # Create life cycle related databases and tables
        if not self.create_database(db_name='lifecycle',
                                    tables=['packages_issue',
                                            'packages_maintainer'],
                                    storage=True):
            raise SQLAlchemyError(
                'Failed to create the specified database and table：lifecycle')

    def __read_config_file(self):
        """
        Read the contents of the configuration file load each
        node data in the yaml configuration file as a list to return

        Returns:
            Initialize the contents of the database configuration file
        Raises:
            FileNotFoundError: The specified file does not exist
            TypeError: Wrong type of data
        """

        if not os.path.exists(self.config_file_path):
            raise FileNotFoundError(
                'system initialization configuration file \
                    does not exist: %s' % self.config_file_path)
        # load yaml configuration file
        with open(self.config_file_path, 'r', encoding='utf-8') as file_context:
            try:
                init_database_config = yaml.load(
                    file_context.read(), Loader=yaml.FullLoader)
            except yaml.YAMLError as yaml_error:

                raise ConfigurationException(' '.join("The format of the yaml configuration\
                     file is wrong please check and try again:{0}".format(yaml_error).split()))

            if init_database_config is None:
                raise ConfigurationException(
                    'The content of the database initialization configuration file cannot be empty')
            if not isinstance(init_database_config, list):
                raise ConfigurationException(
                    ' '.join('The format of the initial database configuration file\
                        is incorrect.When multiple databases need to be initialized, \
                        it needs to be configured in the form of multiple \
                        nodes:{}'.format(self.config_file_path).split()))
            for config_item in init_database_config:
                if not isinstance(config_item, dict):
                    raise ConfigurationException(' '.join('The format of the initial database\
                        configuration file is incorrect, and the value in a single node should\
                        be presented in the form of key - val pairs: \
                        {}'.format(self.config_file_path).split()))
            return init_database_config

    def init_data(self):
        """
        Initialization of the database

        Raises:
            IOError: An error occurred while deleting the database information file
        """
        if getattr(self, 'config_file_datas', None) is None or \
                self.config_file_datas is None:
            raise ContentNoneException('The content of the database initialization \
                configuration file is empty')

        if self.__exists_repeat_database():
            raise DatabaseRepeatException(
                'There is a duplicate database configuration')
        if not InitDataBase.delete_settings_file():
            raise IOError(
                'An error occurred while deleting the database configuration file')

        for database_config in self.config_file_datas:
            if not database_config.get('dbname'):
                LOGGER.logger.error(
                    'The database name in the database initialization configuration file is empty')
                continue
            priority = database_config.get('priority')
            if not isinstance(priority, int) or priority < 0 or priority > 100:
                LOGGER.logger.error('The priority value type in the database initialization \
                    configuration file is incorrect')
                continue
            lifecycle_status_val = database_config.get('lifecycle')
            if lifecycle_status_val not in ('enable', 'disable'):
                LOGGER.logger.error('The status value of the life cycle in the initialization\
                    configuration file can only be enable or disable')
                continue
            # Initialization data
            self._init_data(database_config)

    def _create_database(self, db_name, tables, storage=False):
        """
        create related databases

        Args:
            database_config: Initialize the configuration content of the database_config
        Returns:
            The generated mysql database or sqlite database
        Raises:
            SQLAlchemyError: Abnormal database operation
        """
        _database_engine = self._database_engine.get(self.db_type)
        if not _database_engine:
            raise Error('The database engine is set incorrectly, \
            currently only the following engines are supported: %s '
                        % '、'.join(self._database_engine.keys()))
        _create_table_result = _database_engine(
            db_name=db_name, tables=tables, storage=storage).create_database(self)
        return _create_table_result

    def _init_data(self, database_config):
        """
        data initialization operation

        Args:
            database: Initialize the configuration content of the database
        Returns:

        Raises:
            ContentNoneException: Exception with empty content
            TypeError: Data type error
            SQLAlchemyError: Abnormal database operation
            IOError: An error occurred while deleting the database information file
        """

        try:
            # 1. create a database and related tables in the database
            _db_name = database_config.get('dbname')
            _create_database_result = self._create_database(
                _db_name, self._tables)
            if not _create_database_result:
                raise SQLAlchemyError(
                    'Failed to create the specified database and table：%s'
                    % database_config['dbname'])
            # 2. get the data of binary packages and source packages
            src_db_file = database_config.get('src_db_file')
            bin_db_file = database_config.get('bin_db_file')

            if src_db_file is None or bin_db_file is None:
                raise ContentNoneException(
                    'The path to the sqlite file in the database initialization configuration \
                    is incorrect ')
            if not os.path.exists(src_db_file) or not os.path.exists(bin_db_file):
                raise FileNotFoundError("sqlite file {src} or {bin} does not exist, please \
                    check and try again".format(src=src_db_file, bin=bin_db_file))
            # 3. Obtain temporary source package files and binary package files
            if self.__save_data(database_config,
                                self.database_name):
                # Update the configuration file of the database
                database_content = {
                    'database_name': _db_name,
                    'priority': database_config.get('priority'),
                }
                InitDataBase.__updata_settings_file(
                    database_content=database_content)

        except (SQLAlchemyError, ContentNoneException, TypeError,
                Error, FileNotFoundError) as error_msg:
            LOGGER.logger.error(error_msg)
            # Delete the specified database
            self.__del_database(_db_name)

    def __del_database(self, db_name):
        try:
            _database_engine = self._database_engine.get(self.db_type)
            del_result = _database_engine(db_name=db_name).drop_database()
            return del_result
        except (IOError, Error) as exception_msg:
            LOGGER.logger.error(exception_msg)
            return False

    @staticmethod
    def __columns(cursor):
        """
        functional description:Returns all the column names
        queried by the current cursor

        Args:
            cursor: Cursor
        Returns:
            The first columns
        Raises:

        """
        return [col[0] for col in cursor.description]

    def __get_data(self):
        """
        According to different sql statements, query related table data

        Args:

        Returns:

        Raises:

        """
        if self.sql is None:
            return None
        try:
            src_packages_data = self._database.session.execute(self.sql)
            columns = InitDataBase.__columns(
                src_packages_data.cursor)
            return [dict(zip(columns, row)) for row in src_packages_data.fetchall()]
        except SQLAlchemyError as sql_error:
            LOGGER.logger.error(sql_error)
            return None

    def __save_data(self, database_config, db_name):
        """
        integration of multiple data files

        Args:
            src_package_paths: Source package database file
            bin_package_paths: Binary package database file
        Returns:
            Path of the generated temporary database file
        Raises:

        """
        src_db_file = database_config.get('src_db_file')
        bin_db_file = database_config.get('bin_db_file')
        table_name = database_config.get('dbname')
        lifecycle_status_val = database_config.get('lifecycle')
        try:
            with DBHelper(db_name=src_db_file, db_type='sqlite:///', complete_route_db=True) \
                    as database:
                self._database = database
                # Save data related to source package
                self._save_src_packages(
                    db_name, table_name, lifecycle_status_val)
                self._save_src_requires(db_name)

            with DBHelper(db_name=bin_db_file, db_type='sqlite:///', complete_route_db=True)\
                    as database:
                self._database = database
                # Save binary package related data
                self._save_bin_packages(db_name)
                self._save_bin_requires(db_name)
                self._save_bin_provides(db_name)
                self._save_bin_files(db_name)
        except (SQLAlchemyError, ContentNoneException) as sql_error:
            LOGGER.logger.error(sql_error)
            self.__del_database(db_name)
            return False
        else:
            return True

    def _save_src_packages(self, db_name, table_name, lifecycle_status_val):
        """
        Save the source package data

        Args:
            db_name: Saved database name
        Returns:

        Raises:

        """
        # Query all source packages
        self.sql = " select * from packages "
        packages_datas = self.__get_data()
        if packages_datas is None:
            raise ContentNoneException(
                '{db_name}:There is no relevant data in the source \
                    package provided '.format(db_name=db_name))
        with DBHelper(db_name=db_name) as database:
            database.batch_add(packages_datas, SrcPack)
        if lifecycle_status_val == 'enable':
            InitDataBase._storage_packages(table_name, packages_datas)

    @staticmethod
    def _storage_packages(table_name, package_data):
        """
            Bulk storage of source code package data
        """
        add_packages = []
        cls_model = Packages.package_meta(table_name)
        pkg_keys = ('name', 'url', 'rpm_license', 'version',
                    'release', 'summary', 'description')
        with DBHelper(db_name="lifecycle") as database:
            if table_name not in database.engine.table_names():
                database.create_table([table_name])
            # Query data that already exists in the database
            exist_packages_dict = dict()
            for pkg in database.session.query(cls_model).all():
                exist_packages_dict[pkg.name] = pkg
            _packages = []
            for pkg in package_data:
                _package_dict = {key: pkg[key] for key in pkg_keys}
                _packages.append(_package_dict)

            # Combine all package data, save or update
            for package_item in _packages:
                package_model = exist_packages_dict.get(package_item['name'])
                if package_model:
                    for key, val in package_item.items():
                        setattr(package_model, key, val)
                else:
                    add_packages.append(package_item)

            if add_packages:
                database.batch_add(add_packages, cls_model)
                database.session.commit()

    def _save_src_requires(self, db_name):
        """
        Save the dependent package data of the source package

        Args:
            db_name：Name database
        Returns:

        Raises:

        """
        # Query all source packages
        self.sql = " select * from requires "
        requires_datas = self.__get_data()
        if requires_datas is None:
            raise ContentNoneException('{db_name}: The package data that the source package \
                depends on is empty'.format(db_name=db_name))
        with DBHelper(db_name=db_name) as database:
            database.batch_add(requires_datas, SrcRequires)

    def _save_bin_packages(self, db_name):
        """
        Save binary package data

        Args:
            db_name：Name database
        Returns:

        Raises:

        """
        self.sql = " select * from packages "
        bin_packaegs = self.__get_data()
        if bin_packaegs is None:
            raise ContentNoneException(
                '{db_name}:There is no relevant data in the provided \
                     binary package '.format(db_name=db_name))
        for index, bin_package_item in enumerate(bin_packaegs):
            try:
                src_package_name = bin_package_item.get('rpm_sourcerpm').split(
                    '-' + bin_package_item.get('version'))[0]
            except AttributeError as exception_msg:
                src_package_name = None
                LOGGER.logger.warning(exception_msg)
            finally:
                bin_packaegs[index]['src_name'] = src_package_name

        with DBHelper(db_name=db_name) as database:
            database.batch_add(bin_packaegs, BinPack)

    def _save_bin_requires(self, db_name):
        """
        Save the dependent package data of the binary package

        Args:
            db_name：Name database
        Returns:

        Raises:

        """
        self.sql = " select * from requires "
        requires_datas = self.__get_data()
        if requires_datas is None:
            raise ContentNoneException(
                '{db_name}:There is no relevant data in the provided binary \
                    dependency package'.format(db_name=db_name))

        with DBHelper(db_name=db_name) as database:
            database.batch_add(requires_datas, BinRequires)

    def _save_bin_provides(self, db_name):
        """
        Save the component data provided by the binary package

        Args:
            db_name：Name database
        Returns:

        Raises:

        """
        self.sql = " select * from provides "
        provides_datas = self.__get_data()
        if provides_datas is None:
            raise ContentNoneException(
                '{db_name}:There is no relevant data in the provided \
                    binary component '.format(db_name=db_name))

        with DBHelper(db_name=db_name) as database:
            database.batch_add(provides_datas, BinProvides)

    def _save_bin_files(self, db_name):

        self.sql = " select * from files "
        files_datas = self.__get_data()
        if files_datas is None:
            raise ContentNoneException(
                '{db_name}:There is no relevant binary file installation\
                    path data in the provided database '.format(db_name=db_name))

        with DBHelper(db_name=db_name) as database:
            database.batch_add(files_datas, BinFiles)

    def __exists_repeat_database(self):
        """
        Determine if the same database name exists

        Returns:
            True if there are duplicate databases, false otherwise
        Raises:

        """
        db_names = [name.get('dbname')
                    for name in self.config_file_datas]

        if len(set(db_names)) != len(self.config_file_datas):
            return True

        return False

    @staticmethod
    def __updata_settings_file(**Kwargs):
        """
        update some configuration files related to the database in the system

        Args:
            **Kwargs: data related to configuration file nodes
            database_name: Name database
        Returns:

        Raises:
            FileNotFoundError: The specified file was not found
            IOError: File or network operation io abnormal
        """
        try:
            if not os.path.exists(system_config.DATABASE_FILE_INFO):
                pathlib.Path(system_config.DATABASE_FILE_INFO).touch()
            with open(system_config.DATABASE_FILE_INFO, 'a+', encoding='utf8') as file_context:
                setting_content = []
                if 'database_content' in Kwargs.keys():
                    content = Kwargs.get('database_content')
                    if content:
                        setting_content.append(content)
                yaml.dump(setting_content, file_context)

        except FileNotFoundError as not_found:
            LOGGER.logger.error(not_found)
        except IOError as exception_msg:
            LOGGER.logger.error(exception_msg)

    @staticmethod
    def delete_settings_file():
        """
        Delete the configuration file of the database

        Args:

        Returns:
            True if the deletion is successful, otherwise false
        Raises:
            IOError: File or network operation io abnormal
        """

        try:
            if os.path.exists(system_config.DATABASE_FILE_INFO):
                os.remove(system_config.DATABASE_FILE_INFO)
        except (IOError, Error) as exception_msg:
            LOGGER.logger.error(exception_msg)
            return False
        else:
            return True

    def delete_db(self, db_name):
        """
        delete the database

        Args:
            db_name: The name of the database
        Returns:

        Raises:
            IOError: File or network operation io abnormal
        """
        try:
            del_result = True
            file_read = open(
                system_config.DATABASE_FILE_INFO, 'r', encoding='utf-8')
            _databases = yaml.load(
                file_read.read(), Loader=yaml.FullLoader)
            for database in _databases:
                if database.get('database_name') == db_name:
                    _databases.remove(database)
            # Delete the successfully imported database configuration node
            with open(system_config.DATABASE_FILE_INFO, 'w+', encoding='utf-8') as file_context:
                yaml.safe_dump(_databases, file_context)
        except (IOError, Error) as del_config_error:
            LOGGER.logger.error(del_config_error)
            del_result = False
        finally:
            file_read.close()

        if del_result:
            del_result = self.__del_database(db_name)

        return del_result

    def create_database(self, db_name, tables=None, storage=True):
        """
            Create databases and tables related to the package life cycle

            Args:
                db_name: The name of the database
                tables: Table to be created
        """
        _create_database_result = self._create_database(
            db_name, tables, storage)
        return _create_database_result


class MysqlDatabaseOperations():
    """
    Related to database operations, creating databases, creating tables

    Attributes:
        db_name: The name of the database
        create_database_sql: SQL statement to create a database
        drop_database_sql: Delete the SQL statement of the database
    """

    def __init__(self, db_name, tables=None, storage=False):
        """
        Class instance initialization

        Args:
            db_name: Database name
        """
        self.db_name = db_name
        self.create_database_sql = ''' CREATE DATABASE if not exists `{db_name}` \
                                    DEFAULT CHARACTER SET utf8mb4; '''.format(db_name=self.db_name)
        self.drop_database_sql = '''drop DATABASE if exists `{db_name}` '''.format(
            db_name=self.db_name)
        self.tables = tables
        self.storage = storage

    def create_database(self, init_db):
        """
        create a mysql database

        Returns:
            True if successful, otherwise false
        Raises:
            SQLAlchemyError: An exception occurred while creating the database
        """
        _create_success = True
        if isinstance(init_db, InitDataBase):
            init_db.database_name = self.db_name
        with DBHelper(db_name='mysql') as data_base:

            try:
                # create database
                if not self.storage:
                    data_base.session.execute(self.drop_database_sql)
                data_base.session.execute(self.create_database_sql)
            except InternalError as internal_error:
                LOGGER.logger.info(internal_error)
            except SQLAlchemyError as exception_msg:
                LOGGER.logger.error(exception_msg)
                return False
        if self.tables:
            _create_success = self.__create_tables()
        return _create_success

    def drop_database(self):
        """
        Delete the database according to the specified name

        Args:
            db_name: The name of the database to be deleted
        Returns:
            True if successful, otherwise false
        Raises:
            SQLAlchemyError: An exception occurred while creating the database
        """
        if self.db_name is None:
            raise IOError(
                "The name of the database to be deleted cannot be empty")
        with DBHelper(db_name='mysql') as data_base:
            drop_database = '''  drop DATABASE if exists `{db_name}` '''.format(
                db_name=self.db_name)
            try:
                data_base.session.execute(drop_database)
            except SQLAlchemyError as exception_msg:
                LOGGER.logger.error(exception_msg)
                return False
            else:
                return True

    def __create_tables(self):
        """
        Create the specified data table

        Returns:
            True if successful, otherwise false
        Raises:
            SQLAlchemyError: An exception occurred while creating the database
        """
        try:
            with DBHelper(db_name=self.db_name) as database:
                if self.tables:
                    _tables = list(set(self.tables).difference(
                        set(database.engine.table_names())))
                    database.create_table(_tables)

        except SQLAlchemyError as exception_msg:
            LOGGER.logger.error(exception_msg)
            return False
        else:
            return True


class SqliteDatabaseOperations():
    """
    sqlite database related operations

    Attributes:
        db_name: Name database
        database_file_folder: Database folder path
    """

    def __init__(self, db_name, tables=None, storage=False, ** kwargs):
        """
        Class instance initialization

        Args:
            db_name: Database name
            kwargs: data related to configuration file nodes
        """
        self.db_name = db_name
        self._read_config = ReadConfig(system_config.SYS_CONFIG_PATH)
        if getattr(kwargs, 'database_path', None) is None:
            self._database_file_path()
        else:
            self.database_file_folder = kwargs.get('database_path')
        self.tables = tables
        self.storage = storage

    def _database_file_path(self):
        """
        Database file path

        Returns:

        Raises:
            IOError: File or network operation io abnormal
        """
        self.database_file_folder = self._read_config.get_system(
            'data_base_path')
        if not self.database_file_folder:
            self.database_file_folder = system_config.DATABASE_FOLDER_PATH

        if not os.path.exists(self.database_file_folder):
            try:
                os.makedirs(self.database_file_folder)
            except IOError as makedirs_error:
                LOGGER.logger.error(makedirs_error)
                self.database_file_folder = None

    def create_database(self, init_db):
        """
        create sqlite database and table

        Returns:
            After successful generation, return the database file address,
            otherwise return none
        Raises:
            FileNotFoundError: The specified folder path does not exist
            SQLAlchemyError: An error occurred while generating the database
        """
        _create_success = False
        if self.database_file_folder is None:
            raise FileNotFoundError('Database folder does not exist')

        _db_file = os.path.join(
            self.database_file_folder, self.db_name)

        if not self.storage and os.path.exists(_db_file + '.db'):
            os.remove(_db_file + '.db')

        # create a sqlite database
        with DBHelper(db_name=_db_file) as database:
            try:
                if self.tables:
                    _tables = list(set(self.tables).difference(
                        set(database.engine.table_names())))
                    database.create_table(_tables)
            except (SQLAlchemyError, InternalError) as create_table_err:
                LOGGER.logger.error(create_table_err)
                return _create_success
        if isinstance(init_db, InitDataBase):
            init_db.database_name = _db_file
            _create_success = True
        return _create_success

    def drop_database(self):
        """
        Delete the specified sqlite database

        Returns:
            Return true after successful deletion, otherwise return false
        Raises:
            IOError: An io exception occurred while deleting the specified database file
        """
        try:
            db_path = os.path.join(
                self.database_file_folder, self.db_name + '.db')
            if os.path.exists(db_path):
                os.remove(db_path)
        except IOError as exception_msg:
            LOGGER.logger.error(exception_msg)
            return False
        else:
            return True