JezK
Edit File: apps_manager.py
# coding=utf-8 # Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved # # Licensed under CLOUD LINUX LICENSE AGREEMENT # http://cloudlinux.com/docs/LICENSE.TXT from __future__ import print_function from __future__ import division from __future__ import absolute_import import logging import os import pwd import json from typing import AnyStr, Optional # NOQA from future.utils import iteritems import secureio from clselect.utils import get_abs_rel, run_process_in_cagefs from clcommon.clpwd import drop_privileges from clselect.baseclselect import ( BaseSelectorError, MissingVirtualenvError, MissingAppRootError, ) from clselect.clpassenger import WSGI_PATH from clselect import ClSelectExcept, clpassenger, clselectctl from clselect.baseclselect.apps_manager import BaseApplicationsManager from clselect.clselectpython.python_manager import PythonManager class PythonAppFormatVersion(object): """ Class represents possible python application versions. """ # application that was created in old selector # does not support version change and env variables LEGACY = 1 # new application, supports all selector features STABLE = 2 class PythonVenvFormatVersion(object): """ Class represents possible python application venv path versions. """ # venv that was created in old selector # and was transformed like 'app/test_1' -> 'app_test__1' LEGACY = 1 # any application (old & new) that does not contain # replacements in venv path STABLE = 2 class ApplicationsManager(BaseApplicationsManager): _USER_CONFIG = '.cl.selector/python-selector.json' _IMPORT_STATUS_ADMIN = '/var/lve/.py.migration.status' _IMPORT_STATUS_USER = '.cl.selector/.py.migration.status' _IMPORT_LOG_FILE = '/var/log/cloudlinux/py-selector/app_imports.log' _IMPORT_REQUIREMENTS_FILE = 'imported_pip_requirements.txt' INTERPRETER = 'python' VENV_DIR = 'virtualenv' BINARY_NAME = 'python' def __init__(self): super(ApplicationsManager, self).__init__(PythonManager()) # TODO: replace with real implementation (LVEMAN-1324) def _find_config_files(self, user_name, app_directory, patterns=()): return super(ApplicationsManager, self)._find_config_files(user_name, app_directory, patterns=patterns) @staticmethod def get_interpreter_specific_passenger_lines(binary_path, app_config): """ Return lines for htaccess that are specific to the python interpreter :param binary_path: path to the environment's python binary :param app_config: application's config dictionary """ specific_lines = ['PassengerPython "{}"\n'.format(binary_path)] return specific_lines def _get_legacy_applications(self, user, logger): """ Search server for old applications (created in old python selector) and add them to new config. :param user: user to search applications :param logger: logger to write messages to :return: tuple(list[applications], list[errors]) """ from clselect.clselectctlpython import WSGI_PATTERN applications = {} errors = {} user_config_data = self.get_user_config_data(user) for directory, data in iteritems(clpassenger.summary(user)): if data['interpreter'] != self.INTERPRETER: continue # noinspection PyBroadException try: app_wsgi = os.path.join(data['directory'], clpassenger.WSGI_PATH) # skip app if wsgi file was not found if not os.path.isfile(app_wsgi): logger.warning('Application %s was skipped during import, because wsgi file was not found', directory) continue with open(app_wsgi) as f: wsgi_conf = f.read() match = WSGI_PATTERN.search(wsgi_conf) if match: groupdict = match.groupdict() wsgi_path = groupdict['script'] callable_object = groupdict['callable'] else: wsgi_path = callable_object = '' py_version = os.path.basename(os.path.dirname( os.path.dirname(data['binary']))) domain = data['domains'][0] # LVEMAN-1502. if application already present in apps list use its app_version try: app_ver = user_config_data[directory][u'app_version'] except KeyError: app_ver = PythonAppFormatVersion.LEGACY app_info = { u'python_version': py_version, u'app_version': app_ver, u'domain': domain, u'app_uri': data['alias'], u'app_status': 'started', u'startup_file': wsgi_path or WSGI_PATH, u'config_files': [], u'env_vars': {}, u'entry_point': callable_object or 'application' } applications[directory] = app_info except Exception: logger.exception('Unable to import application %s', directory) errors.setdefault(directory, []).append( 'Unexpected issue during application "%s" import. ' 'Your python app will work as before, but you wo\'t be able ' 'to control it from Python Selector UI. Please, ask your system ' 'administrator to contact CloudLinux support ' 'to resolve this issue.' % directory ) continue return applications, errors def _setup_import_logger(self): """ Setup logger for application import. """ app_logger = logging.getLogger('import_apps') app_logger.setLevel(logging.DEBUG) fh = logging.FileHandler(self._IMPORT_LOG_FILE) fh.formatter = logging.Formatter( '[%(levelname)s | %(asctime)s]: %(message)s') cl = logging.StreamHandler() cl.setLevel(logging.INFO) app_logger.addHandler(fh) # app_logger.addHandler(cl) return app_logger def import_legacy_applications_to_config(self, user=None): """ Scan users for legacy applications (created by old python selector) and import them into config file. Done automatically by spec file. """ app_logger = self._setup_import_logger() users_info = self.get_users_dict(user) logging.debug('Start applications import for users %s', list(users_info.keys())) failed_users = [] skipped_users_count = 0 apps_errors = None for user, pw in iteritems(users_info): # Skip user if '.cl.selector' directory absent in his homedir cl_selector_dir = os.path.join(pw.pw_dir, '.cl.selector') if not os.path.isdir(cl_selector_dir): skipped_users_count += 1 app_logger.warning('User %s is skipped due to %s directory absent' % (user, cl_selector_dir)) continue app_logger.info('Importing user %s', user) try: # disable quota here, because clpassanger.summary creates htaccess_cache file # see clpassenger._summary for details. Will be fixed in LVEMAN-1524 with drop_privileges(user), secureio.disable_quota(): try: config = self.read_user_selector_config_json( pw.pw_dir, pw.pw_uid, pw.pw_gid) except BaseSelectorError: config = {} # take applications from clpassanger.summary and import them apps, apps_errors = self._get_legacy_applications(user, app_logger) if not apps: continue for app_root, app_config in iteritems(apps): if app_root in config: # save fields that were probably set before app_config['config_files'] = config[app_root]['config_files'] app_config['env_vars'] = config[app_root]['env_vars'] # generate pip requirements file for each legacy app self.generate_pip_requirements(user, app_root, app_logger) requirements_file = os.path.join(pw.pw_dir, app_root, self._IMPORT_REQUIREMENTS_FILE) if os.path.isfile(requirements_file) and \ self._IMPORT_REQUIREMENTS_FILE not in app_config['config_files']: # add this newly generated pip requirements file to config files app_config['config_files'].append(self._IMPORT_REQUIREMENTS_FILE) config[app_root] = app_config # case when hoster downgraded package for some time # and user destroyed some of applications # we should remove them from config file destroyed_apps = set(config.keys()) - set(apps.keys()) for app_root in destroyed_apps: config.pop(app_root) with drop_privileges(user), secureio.disable_quota(): self.write_full_user_config_data(user, config) self._set_user_legacy_import_status(pw, is_import_failed=False, apps_errors=apps_errors) except Exception as e: import_error = 'Unable to import user `%s` to new python selector. ' + \ 'Already created applications work as before, ' + \ 'but user won\'t be able to manage them. Error is: `%s`' app_logger.exception(import_error, user, e) with drop_privileges(user), secureio.disable_quota(): self._set_user_legacy_import_status(pw, is_import_failed=True, apps_errors=apps_errors) failed_users.append(user) self._set_admin_legacy_import_status(failed_users) if skipped_users_count != 0: app_logger.warning('Some users skipped... see import log') def _get_legacy_import_status(self, user=None): """ Read import log which contains information about failed users or apps depending on current user. """ if user is None: if not os.path.exists(self._IMPORT_STATUS_ADMIN): return None with open(self._IMPORT_STATUS_ADMIN) as f: return json.load(f) else: user_pw = pwd.getpwnam(user) marker = os.path.join(user_pw.pw_dir, self._IMPORT_STATUS_USER) if not os.path.isfile(marker): return with open(marker) as f: return json.load(f) def _set_user_legacy_import_status(self, user_pw, is_import_failed, apps_errors): """ Save information that some applications were not imported automatically to show warning for user in the future. """ if not os.path.exists(os.path.join(user_pw.pw_dir, '.cl.selector')): return try: marker = os.path.join(user_pw.pw_dir, self._IMPORT_STATUS_USER) secureio.write_file_via_tempfile(json.dumps({ 'is_import_failed': is_import_failed, 'apps_lookup_failed': apps_errors }), marker, 0o600) except (IOError, OSError): # sad, but not critical, go further logging.exception('Unable to save migration status file') def _set_admin_legacy_import_status(self, failed_users): """ Save information that some users were not imported automatically to show warning for admin in the future. """ packed = json.dumps({'failed_users': failed_users}) secureio.write_file_via_tempfile(packed, self._IMPORT_STATUS_ADMIN, 0o600) def _get_admin_legacy_import_warning(self): config = self._get_legacy_import_status() if config is None or not config['failed_users']: return None warning_msg = 'Unexpected issue(s) happened during importing python ' \ 'applications for the following users: "{users}". ' \ 'Everything will work as before, but listed users wo\'t be able ' \ 'to control applications from Python Selector UI. ' \ 'Please, contact CloudLinux support and send them log file located at ' \ '`{log_file_path}` to investigate and ' \ 'resolve this issue. Also you can hide this warning ' \ 'by deleting `{import_warning_marker}` file' \ ''.format(users=','.join(config['failed_users']), log_file_path=self._IMPORT_LOG_FILE, import_warning_marker=self._IMPORT_STATUS_ADMIN) return warning_msg def _get_user_legacy_import_warning(self, username): config = self._get_legacy_import_status(user=username) if config is None: return None what_to_do_msg = \ 'Everything will work as before, but you won\'t be able to control ' \ 'listed applications from Python Selector UI. ' \ 'Please, ask you hoster to contact CloudLinux support ' \ 'to investigate and ' \ 'resolve this issue.\nAlso you can hide this warning ' \ 'by deleting `~/{import_warning_marker}` file.' \ ''.format(import_warning_marker=self._IMPORT_STATUS_USER) if config['is_import_failed']: return 'Unexpected issue(s) happened during importing python ' \ 'applications. ' \ '%s' % what_to_do_msg elif config['apps_lookup_failed']: return 'Unexpected issue(s) happened during importing following python ' \ 'applications: "%s". ' \ '%s' % (','.join(config['apps_lookup_failed']), what_to_do_msg) return None def _get_legacy_import_warning_or_none(self, user=None): if user is None: return self._get_admin_legacy_import_warning() else: return self._get_user_legacy_import_warning(user) def migrate_application(self, user, app_root): """ Update environment of specific application to support features of new python selector """ from clselect.clselectctlpython import _get_environment application = self.get_app_config(user, app_root) if application is None: raise ClSelectExcept.WrongData("Application %s does not exist" % app_root) if application['app_version'] == PythonAppFormatVersion.STABLE: raise ClSelectExcept.WrongData( "Application %s is already new version " "and does not need any updates" % app_root) environment = _get_environment(user, app_root, apps_manager=self) try: environment.configure_environment(auto_restore=True) except Exception as e: raise ClSelectExcept.WrongData( "Unable to migrate application %s. " "Error is '%s', everything restored to work as before migration. " "Try again later or ask your hoster " "to contact CloudLinux support if the issue persists." "" % (app_root, e)) application['app_version'] = PythonAppFormatVersion.STABLE self.add_app_to_config(user, app_root, application) def get_applications_users_info(self, user=None): result = super(ApplicationsManager, self). \ get_applications_users_info(user) warning = self._get_legacy_import_warning_or_none(user) if warning is not None: result['warning'] = warning return result def generate_pip_requirements(self, user, app_root, app_logger): # type: (AnyStr, AnyStr, logging.Logger) -> Optional[AnyStr] """ Generates requirements file from python apps :param user: username :param app_root: app root :param app_logger: app logger """ app_path, _ = get_abs_rel(user, app_root) req_path = os.path.join(app_path, self._IMPORT_REQUIREMENTS_FILE) if os.path.exists(req_path): return from clselect.clselectctlpython import _get_environment with drop_privileges(user): environment = _get_environment( user, app_root, apps_manager=self) pip_path = environment.pip() user_home = pwd.getpwnam(user).pw_dir env_vars = {'HOME': user_home} modules = '' # need to pass users home as env var directly during running as user result = run_process_in_cagefs(user, pip_path, ['freeze', '-l'], env_vars=env_vars) if result['returncode'] != 0: app_logger.warning('Error during generation pip requirements file. ' + str(result['output'])) # like in `check_output` raise ClSelectExcept.ExternalProgramFailed(result['output']) elif not result['failed']: modules = result['output'] # write stdout to file with disabled quota with drop_privileges(user), secureio.disable_quota(): f = open(req_path, 'w') f.write(modules) f.close() def get_app_folders(self, username, app_root, chk_env=True, chk_app_root=True): """ Calculate, check exists and return application folders This method does not check that application exists in config. :raises: NoSuchUserException, MissingVirtualenvError, MissingAppRootError :return: tuple(app_root, app_venv) with absolute paths """ user_home = self._pwd.get_pw_by_name(username).pw_dir _, rel_path = get_venv_rel_path(username, app_root) app_venv = os.path.join(user_home, rel_path) if chk_env and not os.path.exists(app_venv): raise MissingVirtualenvError(app_venv) app_root = os.path.join(user_home, app_root) if chk_app_root and not os.path.exists(app_root): raise MissingAppRootError(app_root) return app_root, app_venv def get_binary_path(self, user, app_root, user_dir, binary_name=None): """ Return a path to the environment's interpreter binary Get interpreter path for application :param user: owner of the application :param app_root: app path relative to user home (app-root) :param user_dir: User's home directory :param binary_name: name of binary in virtual environemnt (python, npm, node) :return: path to interpreter binary in virtual environment """ version = self.get_interpreter_version_for_app(user, app_root) if binary_name is None: binary_name = self.BINARY_NAME _, rel_path = get_venv_rel_path(user, app_root) return os.path.join(user_dir, rel_path, version, 'bin', binary_name) # TODO: we definitely should refactor this one time def get_venv_rel_path(user, directory, version=None): """ Get path to users VENV relative to home dir. Old python selector transforms app_root using get_prefix() method before creating venv directory. We should handle both cases: - when app is from old selector - when app is from new selector and return right env directory. If both old and new vevns exist we use old one. Return tuple with two values: - detected version of venv path (LEGACY for path with replacements "/" -> "_" & "_" => "__" and STABLE in other case) - path to venv relative to user's home dir """ _, new_rel_dir = get_abs_rel(user, os.path.join(ApplicationsManager.VENV_DIR, directory)) old_abs_dir, old_rel_dir = get_abs_rel( user, os.path.join(ApplicationsManager.VENV_DIR, clselectctl.get_prefix(directory))) if version is None: if os.path.exists(old_abs_dir) and directory != clselectctl.get_prefix(directory): return PythonVenvFormatVersion.LEGACY, old_rel_dir return PythonVenvFormatVersion.STABLE, new_rel_dir elif version == PythonVenvFormatVersion.LEGACY: return PythonVenvFormatVersion.LEGACY, old_rel_dir elif version == PythonVenvFormatVersion.STABLE: return PythonVenvFormatVersion.STABLE, new_rel_dir else: raise ValueError("unknown version `%s`" % version)