JezK
Edit File: environments.py
# -*- coding: utf-8 -*- # Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved # # Licensed under CLOUD LINUX LICENSE AGREEMENT # http://cloudlinux.com/docs/LICENSE.TXT from __future__ import print_function from __future__ import absolute_import from __future__ import division import os import sys import pwd import subprocess import shutil import filecmp from datetime import datetime from clselect.clselectctl import get_directory from clselect.utils import check_call, check_output, list_dirs, run_command from .extensions import EXTENSION_PATTERN, ExtensionInfo from .interpreters import Interpreter, interpreters import simplejson as json from simplejson import JSONDecodeError from clselect.clselectexcept import ClSelectExcept DEFAULT_PREFIX = 'virtualenv' BACKUP_PREFIX = '.virtualenv.backup' # only this binary it brought by alt-python-virtualenv package VIRTUALENV_BIN = '/opt/cloudlinux/venv/bin/virtualenv' VERSION_DELIMITER = '#' WRAPPERS_PATH = '/usr/share/l.v.e-manager/utils' PYTHON_WRAPPER = 'python_wrapper' SET_ENV_VARS_SCRIPT = 'set_env_vars.py' class Environment(object): def __init__(self, name, user=None, prefix=None): self.name = name if user: self.user = user else: self.user = pwd.getpwuid(os.getuid()).pw_name self.homepath = pwd.getpwnam(self.user).pw_dir self.pip_logfile = os.path.join(self.homepath, '.pip/pip.log') if prefix is None: self.prefix = DEFAULT_PREFIX else: self.prefix = prefix self.path = os.path.join(_abs_prefix(self.user, self.prefix), name) self.backup_path = os.path.join( _abs_prefix(self.user, BACKUP_PREFIX), self.name) self._requirements = None self._interpreter = None self._pip = None self.interpreter_name = 'python' + name # Create extenstion remap table self._extension_remap = {'MySQLdb': 'MySQL-python'} def __repr__(self): return ("%s.%s(name='%s', user='%s', prefix='%s')" % ( self.__class__.__module__, self.__class__.__name__, self.name, self.user, self.prefix)) def _demote(self): user_pwd = pwd.getpwnam(self.user) def func(): os.setgid(user_pwd.pw_gid) os.setuid(user_pwd.pw_uid) os.environ['USER'] = self.user os.environ['HOME'] = user_pwd.pw_dir return func def as_dict(self, key=None): e = { 'name': self.name, 'interpreter': self.interpreter(), 'extensions': self.extensions(), } if key: del e[key] return {getattr(self, key): e} return e def as_deepdict(self, key=None, with_extensions=True): e = { 'name': self.name, 'interpreter': self.interpreter().as_dict(), } if with_extensions: e.update({ 'extensions': self.extensions(), }) if key: del e[key] return {getattr(self, key): e} return e def create(self, interpreter=None, version=None, wait=None): if not interpreter: interpreter = Interpreter(target_user=self.user) path = self.path if version: path = os.path.join(path, version) prompt = "({}:{})".format( get_directory(os.path.basename(self.prefix)), self.name, ) args = [ VIRTUALENV_BIN, '--prompt', prompt, '--python', interpreter.binary, path, ] kwargs = {"preexec_fn": self._demote(), "cwd": self.homepath, "wait": wait} try: check_call(*args, **kwargs) except ClSelectExcept.ExternalProgramFailed as err: err = str(err) err_trace = None # Change error text and add help if disk quota exceeded if "Disk quota exceeded" in err: err_text = "Disk quota exceeded.\n " \ "Contact system administrator to increase disk quota." elif "Traceback" in err: # Find second ":" character. First is "Traceback :" err_char = err.find(":", err.find(":")+1) # Find last row of trace err_trace_end = err[:err_char].rfind('\n') # pylint: disable=indexing-exception if err_trace_end == -1 or err_char == -1: err_text = err else: # Trace row without error err_trace = err[:err_trace_end] # pylint: disable=indexing-exception # Only error without first trace err_text = err[err_trace_end+1:] # pylint: disable=indexing-exception else: err_text = err raise ClSelectExcept.ExternalProgramFailed( message=err_text, details=err_trace, ) self.configure_environment() def detect_python_binary(self, bin_path): files_to_check = [ 'python', self.interpreter_name.split('.')[0], self.interpreter_name, ] for file in files_to_check: path = os.path.join(bin_path, file) if not os.path.islink(path) or os.readlink(path).startswith('/opt/alt/python'): return path return None def configure_environment(self, auto_restore=False): """ Configures environment: 1. Rename binary to pythonX.Y_bin 2. Makes symlink from python binary to python_wrapper """ bin_path = os.path.join(self.path, 'bin') new_interpreter_path = os.path.join(bin_path, self.interpreter_name) + '_bin' interpreter_path = self.detect_python_binary(bin_path) if interpreter_path is None: return if os.path.exists(new_interpreter_path): os.remove(new_interpreter_path) os.rename(interpreter_path, new_interpreter_path) try: if not os.path.exists(interpreter_path): os.symlink(os.path.join(WRAPPERS_PATH, PYTHON_WRAPPER), interpreter_path) except (IOError, OSError): if auto_restore: os.rename(new_interpreter_path, interpreter_path) raise if not os.path.exists(os.path.join(bin_path, SET_ENV_VARS_SCRIPT)): os.symlink(os.path.join(WRAPPERS_PATH, SET_ENV_VARS_SCRIPT), os.path.join(bin_path, SET_ENV_VARS_SCRIPT)) def destroy(self, version=None): path = self.path if version: path = os.path.join(path, version) if os.path.exists(path): check_call('/bin/rm', '-r', '--interactive=never', path, preexec_fn=self._demote()) def _get_extension_name(self, extension_name): """ Returns extensions name considering extension remap table :param extension_name: Input extension name :return: Result extension name """ if extension_name in self._extension_remap: return self._extension_remap[extension_name] else: return extension_name def _recreate(self, version): """ Recreate python virtual environment with requirements :return: """ # if virtual environment does not exist, just create it # unfortunately, we don't have requirements env_path = os.path.join(self.path, version) if not os.path.exists(self.pip(version=version)): return print('Re-create python virtualenv:', env_path) # pip freeze, save last requirements into the file self._pip_freeze(version) # remember the requirements in the memory requirements_path = self.pip_requirements(version) requirements = [] if os.path.exists(requirements_path): reqs_file = open(requirements_path, 'r') requirements = reqs_file.readlines() reqs_file.close() # destroy python virtual environment self.destroy(version=version) # create python virtual environment self.create(version=version, wait=True) # put remembered requirements into the file reqs_file = open(requirements_path, 'w') reqs_file.writelines(requirements) reqs_file.close() # pip install -r requirements, install requirements check_call( self.pip(version=version), 'install', '-r', self.pip_requirements(version)) def recreate(self): for version in interpreters(key='version').keys(): self._recreate(version) def exists(self): return os.path.exists(self.path) def interpreter(self): if not self._interpreter: self._interpreter = Interpreter(prefix=self.path, target_user=self.user) return self._interpreter def extension_install(self, extension_name): extension_name = self._get_extension_name(extension_name) locked_extensions = ExtensionInfo.get_locked_extensions(self.interpreter_name) t = extension_name.split(VERSION_DELIMITER) extension, version = t[0], t[1:] or '' command = ('/bin/bash', '-l', '-c', self.pip() + ' --log-file=' + self.pip_logfile + ' install ' + extension_name) if version: version = version[0] command = ('/bin/bash', '-l', '-c', self.pip() + ' --log-file=' + self.pip_logfile + ' install ' + extension + '==' + version) if ExtensionInfo.is_extensions_locked(locked_extensions, extension_name, version): raise ValueError("Extension '%s' install is prohibited. System extension" % extension_name) check_call(args=command, preexec_fn=self._demote(), cwd=self.homepath) self._pip_freeze() def extension_install_requirements(self, requirements_path): command = ('/bin/bash', '-l', '-c', self.pip() + ' --log-file=' + self.pip_logfile + ' install -r {}'.format(requirements_path)) check_call(args=command, preexec_fn=self._demote(), cwd=self.homepath) self._pip_freeze() def extension_update(self, extension): check_call(self.pip(), '--log-file='+self.pip_logfile, 'install', '--upgrade', extension, preexec_fn=self._demote(), cwd=self.homepath) self._pip_freeze() def extension_uninstall(self, extension): locked_extensions = ExtensionInfo.get_locked_extensions(self.interpreter_name) t = extension.split(VERSION_DELIMITER) extension, version = t[0], t[1:] or '' if version: version = version[0] if ExtensionInfo.is_extensions_locked(locked_extensions, extension, version): raise ValueError("Extension '%s' removal is prohibited" % extension) p = subprocess.Popen( (self.pip(), '--log-file='+self.pip_logfile, 'uninstall', extension), preexec_fn=self._demote(), stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE, cwd=self.homepath, text=True) stdout, stderr = p.communicate('y') if p.returncode: raise Exception(stderr or stdout) self._pip_freeze() def extensions(self): result = {} locked_extensions = ExtensionInfo.get_locked_extensions(self.interpreter_name) try: output = check_output(self.pip(), 'list', '--log-file='+self.pip_logfile, '--format=json', preexec_fn=self._demote(), cwd=self.homepath) extensions = [(x['name'], x['version']) for x in json.loads(output)] except (JSONDecodeError, KeyError, ValueError, ClSelectExcept.FileProcessError, ClSelectExcept.ExternalProgramFailed): output = check_output(self.pip(), 'list', '--log-file='+self.pip_logfile, preexec_fn=self._demote(), cwd=self.homepath) extensions = EXTENSION_PATTERN.findall(output) docs = (ExtensionInfo().extension_doc(extension) for extension, _ in extensions) for (name, version), doc in zip(extensions, docs): if ExtensionInfo.is_extensions_locked(locked_extensions, name, version): version_diff = list(set([v.strip() for v in version.split(',')]) - set(locked_extensions.get(name))) if version_diff and len(locked_extensions.get(name)) != 0: result[name] = {'doc': doc, 'version': ', '.join(version_diff)} else: result[name] = {'doc': doc, 'version': version} return result def pip(self, version=None): if version is not None: return os.path.join(self.path, version, 'bin', 'pip') if not self._pip: self._pip = os.path.join(self.path, 'bin', 'pip') return self._pip def pip_requirements(self, version=None): if version is not None: return os.path.join(self.path, version, 'requirement.pip') return os.path.join(self.path, 'requirement.pip') def update_python_interpreter(self, backup=False, force=False, verbose=True, _alt_interpreters_dict=None): """ copy binary python from /opt/alt/pythonXY/bin/pythonX.Y to virtualenvdir/bin/pythonX.Y :param backup: make backup old python interpreter :param force: force rewrite python interpreter without check :param verbose: print actions details to stdout :return: True - updating success; False - updating fail """ update_result = False interpreter = self.interpreter() if _alt_interpreters_dict: main_interpreter = _alt_interpreters_dict[interpreter.version] else: main_interpreter = interpreters(key='version')[interpreter.version] # path to original /opt/alt/pythonXY/bin/pythonX.Y updated_list = list() # list updated interpreters if os.path.islink(interpreter.python_bin) and os.readlink(interpreter.python_bin).startswith('/opt/alt/python'): if verbose: print('Nothing to do, binary in your virtual environment is already symlink to global python!') return False # make backup and delete old python binary python_backup = interpreter.python_bin + '.orig_%s' % datetime.now().strftime("%Y-%m-%d_%H-%M") stat_ = os.stat(interpreter.python_bin) shutil.copy(interpreter.python_bin, python_backup) os.chown(python_backup, stat_.st_uid, stat_.st_gid) # preserving owner try: for virtualenv_python_bin in interpreter.binary_list: if filecmp.cmp(main_interpreter.binary, interpreter.python_bin) and not force: update_result = False if verbose: print(" not need updating; skip '%s'" % virtualenv_python_bin) continue if verbose: sys.stdout.write(" copy '%s' -> '%s'..." % (main_interpreter.binary, virtualenv_python_bin)) run_command(cmd=('/bin/cp', '--force', main_interpreter.binary, virtualenv_python_bin)) updated_list.append(virtualenv_python_bin) print("Done") update_result = True except (OSError, IOError) as e: # rollback binaries python if something is wrong print("Fail %s" % str(e)) for updated_python in updated_list: shutil.copyfile(python_backup, updated_python) # safe copy with preserve owner and mode os.unlink(python_backup) if not backup: # delete backup if not need os.unlink(python_backup) return update_result def _pip_freeze(self, version=None): """ Output installed packages in requirements format :return: None """ if not os.path.exists(self.pip(version)): return command = (self.pip(version), 'freeze', '-l') f = open(self.pip_requirements(version), 'w') check_call(args=command, preexec_fn=self._demote(), cwd=self.homepath, output=f) def pip_freeze(self): """ Output installed packages in requirements format :return: None """ for version in interpreters(key='version').keys(): self._pip_freeze(version=version) def _abs_prefix(user=None, prefix=None): if not prefix: prefix = DEFAULT_PREFIX if user: return os.path.join(pwd.getpwnam(user).pw_dir, prefix) else: return os.path.join(pwd.getpwuid(os.getuid()).pw_dir, prefix) def environments(user=None, prefix=None): venv_path = _abs_prefix(user, prefix) try: env_list = list_dirs(venv_path) except OSError: return [] envs = [] for env_name in env_list: envs.append(Environment(env_name, user, prefix)) return envs def environments_dict(key, user=None, prefix=None): return dict(list(e.as_dict(key=key).items()) for e in environments(user, prefix)) def environments_deepdict(key, user=None, prefix=None): return dict(list(e.as_deepdict(key=key).items()) for e in environments(user, prefix))