# keep-py-file: True
""" mergeimporter.py -- Imports modules from a series of directories for each
package similar to how merge filesystems work. This is used to implement patch
loading, so as few modules as possible should be used because any module used
cannot be patched.  Needs to be able to be loaded as a top level module and by
any python version supported by the debugger.

Copyright (c) 2004-2018, Archaeopteryx Software, Inc.  All rights reserved.

Written by Stephan R.A. Deibel and John P. Ehresman

"""

# This is used during bootstrap and should not load any other modules in wingbase

# Compatibility note: this is used when loading the debugger so needs to
# work with all supported Python versions.  The meta path import mechanism
# was introduced in python 2.3.  Some of the details are being deprecated
# in the python 3 series but nothing is abandoned as of 3.5.  As of 3.7
# the code uses importlib and not imp.

import linecache
import marshal
import os
import sys
import traceback
if sys.hexversion >= 0x03070000:
  import importlib.util
  import importlib.machinery
  import types
else:
  import imp

if sys.version_info[:2] < (3, 0):
  import string

if sys.hexversion >= 0x03070000:
  kMagic = importlib.util.MAGIC_NUMBER
else:
  kMagic = imp.get_magic()
  
def _GetModuleSuffixes():
  if sys.hexversion >= 0x03070000:
    return importlib.machinery.all_suffixes()
  else:
    return [i[0] for i in imp.get_suffixes()]
  
def _GetImportableName(file_name):
  """ Return Python name created when given file name is imported or None
  if it can't be imported through the built in import mechanism. """

  for suffix in _GetModuleSuffixes():
    if file_name.endswith(suffix):
      return file_name[:-len(suffix)]
  return None

def _IsIdentifier(s):
  """ Returns whether s is a valid Python identifer """
  
  if s == '':
    return False
  
  for i, c in enumerate(s):
    if i == 0:
      if not (c.isalpha() or c == '_'):
        return False
    else:
      if not (c.isalnum() or c == '_'):
        return False
      
  return True

def _MakeBytes(s):
  
  if sys.version_info[:2] < (3, 0):
    return str(s)
  else:
    return bytes(s, 'ascii')

def _GetTrans():
  
  if sys.version_info[:2] < (3, 0):
    f = ''.join(chr(i) for i in range(256))
    t = ''.join(chr(i) for i in reversed(range(256)))
    return string.maketrans(f, t)
  
  f = bytes(range(256))
  t = bytes(reversed(range(256)))
  # XXX 3.0 is missing .maketrans
  return bytes.maketrans(f, t)

class CMergeDirImporter:
  """ Importer object to load modules and modules in packages from a list of
  directories in preference to sys.path; loads modules from internal list of
  directories even if the import is a local import within a package. This is
  so directories of patch files can be kept in separate directories, but
  loaded even when using relative imports. Designed to be used as a meta_path
  hook, which intercepts all import requests before the internal import
  mechanism attempts to resolve the import.
  
  Supports .wingcode formatting of Python files for mild obfuscation, mainly
  to prevent accidental changes to an installation.
  
  Files in directories added later take precedence over those for the same
  module name in directories added earlier. Note that any file for a named
  module in a directory will take precedence over files for the same named
  module added earlier; this means that a .py in a later directory will be
  used instead of a .so in an earlier directory (ordinarily a .so file takes
  precedence over a .py file)

  Current limitations / implementation quirks:
   * Ignores zip files in patch directory
   * List of files in patch directory is scanned when directory is added
     so files added or removed afterward are ignored
   * Source in .wingcode files must be utf8 encoded (ascii is acceptable,
     of course)
  """
  
  WING_CODE_EXT = '.wingcode'
  _TRANSLATE = _GetTrans()
  USE_OUR_METHODS_FOR_PY_FILES = False
  WRITE_EMPTY_LINECACHE_ENTRIES = True
  MAX_LOG_LEVEL = 1
  
  CACHE_DIR_UMASK = 0x1c0  # 0o700, which python 2.5 doesn't accept
  
  if sys.version_info[:2] < (3, 0):
    CODE_OBJECT_TYPE = type(_IsIdentifier.func_code)
  else:
    CODE_OBJECT_TYPE = type(_IsIdentifier.__code__)
  
  def __init__(self, dir_list = [], logger=None):
    self.__files_for_name = {}
    self._cache_dirs = {}
    self.dir_list = []
    
    self._logger = logger
    
    self._Log("Created meta importer\n")
    
    for dirname in dir_list:
      self.AddDir(dirname)
      
  def _TryCreateCacheDir(self, dirname):
    """ Try to create the given cache directory on disk"""
    
    if os.path.exists(dirname):
      return
  
    try:
      os.makedirs(dirname, self.CACHE_DIR_UMASK)
    except (IOError, OSError):
      self._Log('Could not create cache directory: ' + repr(dirname) + '\n')
    else:
      self._Log('Created cache directory: ' + repr(dirname) + '\n')
    
  def AddCacheDir(self, dirname, cache_dir):
    """ Add cache dir for a given dir.  Subdirectories of dirname
    will map to subdirectories of cache_dir.  Note that cache dirs
    are scanned when dirs are initially scanned """
    
    self._TryCreateCacheDir(cache_dir)
    
    if os.sep != '/':
      dirname = dirname.replace(os.sep, '/')
      cache_dir = cache_dir.replace(os.sep, '/')
    
    self._cache_dirs[dirname] = cache_dir

  def AddDir(self, dirname, parent = '', cache_dir=None):
    """ Add all files in given directory and all subdirectories.  parent 
    is the dotted name of the parent directory.  Optionally, add a cache
    directory to use for the added directory."""
    
    self._Log("  Added dir %s for %s" % (dirname, parent))

    if cache_dir is not None:
      self.AddCacheDir(dirname, cache_dir)
      
    if os.sep != '/':
      dirname = dirname.replace(os.sep, '/')
    
    self.dir_list.append(dirname)
    
    if parent != '' and not parent.endswith('.'):
      parent += '.'
    
    try:
      name_list = os.listdir(dirname)
    except (IOError, OSError):
      return

    for name in name_list:
      # Note os.path is safe since name is never absolute
      assert not os.path.isabs(name)
      full_path = os.path.join(dirname, name)
      if os.sep != '/':
        full_path = full_path.replace(os.sep, '/')
        
      if os.path.isdir(full_path):
        if _IsIdentifier(name):
          self.AddDir(full_path, parent + name + '.')
          
      elif os.path.isfile(full_path):
        if full_path.endswith(self.WING_CODE_EXT):
          imp_name = name[:-len(self.WING_CODE_EXT)]
        else:
          imp_name = _GetImportableName(name)
        if imp_name is not None:
          self._AddSingleFile(full_path, imp_name, parent)

  def _Log(self, msg, exc=False, level=1):
    
    if self._logger is None or level > self.MAX_LOG_LEVEL:
      return
    
    if not msg.endswith('\n'):
      msg += '\n'
    self._logger.write(msg)
      
    if exc:
      traceback.print_exc(file=self._logger)
      
  def _Log2(self, msg, exc=False):
    
    self._Log(msg, exc, 2)

  def _AddSingleFile(self, full_path, imp_name, parent):
    """ Add a single file to __files_for_name data.  The imp_name argument
    is the local name of the file in the import namespace and the parent
    is the full name of the containing package ('' if no containing package)
    """
    
    pkg_namespace = (parent != '' and imp_name == '__init__')
    if not pkg_namespace:
      full_mod_name = parent + imp_name
    else:
      full_mod_name = parent[:-1]
  
    dirname = os.path.dirname(full_path)
  
    existing = self.__files_for_name.get(full_mod_name)
    if existing is not None and existing[0] == dirname:
      file_list = existing[2]
    else:
      file_list = []
    self._AddFileToOrderedList(full_path, file_list)
    self.__files_for_name[full_mod_name] = (dirname, pkg_namespace, file_list)

  def _AddFileToOrderedList(self, full_path, file_list):
    """ Add / insert file in list -- the first file that can be used will be """

    def key(full_path):
      compiled_exts = ('.so', '.pyd')
      pyc_exts = ('.pyc', '.pyo')
      if full_path.lower().endswith(compiled_exts):
        return 1
      if full_path.lower().endswith('.py'):
        return 2
      if full_path.lower().endswith(self.WING_CODE_EXT):
        return 3
      
      return 4
    
    file_list.append(full_path)
    file_list.sort(key=key)
      
  def find_module(self, full_name, path = None):

    self._Log2('find_module called, full_name = %s' % (full_name, ))
    data = self.__files_for_name.get(full_name)
    if data is not None:
      return self
    else:
      return None

  if sys.hexversion >= 0x03070000:
    def find_spec(self, full_name, path, target):
      """ find_spec method needed when importlib is used.  full_name is
      the module name including the package prefix.  path is None for
      top-level modules or the value of __path__ in the enclosing package
      namespace for modules in a package. """
      
      self._Log2("find_spec(%s, %s, %s)" % (full_name, path, target))
      data = self.__files_for_name.get(full_name)
      if data is None:
        self._Log2("  spec returned: None (no data)")
        return None
      
      dirname, is_pkg, full_path_list = data
      if is_pkg:
        dirname = os.path.dirname(dirname)

      spec = importlib.machinery.ModuleSpec(full_name, self, is_package=is_pkg)
      self._Log2("  spec returned: %r" % (spec,))
      return spec
    
  def load_module(self, full_name):

    data = self.__files_for_name.get(full_name)
    if data is None:
      raise ImportError('Unknown module: %s' % full_name)

    self._Log2('load_module found data: %s' % (data, ))

    last_import_exc = None

    dirname, is_pkg, full_path_list = data
    for full_path in full_path_list:
      if os.sep != '/':
        full_path = full_path.replace('/', os.sep)

      if full_path.endswith(self.WING_CODE_EXT):
        mod = self._LoadWingCode(full_name, full_path, is_pkg)
      elif self.USE_OUR_METHODS_FOR_PY_FILES and full_path.endswith('.py'):
        mod = self._LoadWingCode(full_name, full_path, is_pkg)
      elif sys.version_info >= (3, 7):
        mod, last_import_exc = self._LoadFileWithImportLib(full_name, full_path,
                                                              is_pkg)
      else:
        self._Log2("Non .wingcode file using imp: %r" % (full_path, ))
        
        parts = full_name.split('.')
        path = os.path.dirname(full_path)
        if is_pkg:
          path = os.path.dirname(path)

        try:
          f, p, d = imp.find_module(parts[-1], [path])
          try:
            mod = imp.load_module(full_name, f, p, d)
          finally:
            if f is not None:
              f.close()
        except ImportError:
          last_import_exc = sys.exc_info()
          mod = None
        
      if mod is not None:
        self._Log2('Returning module for %s using file: %r' % (full_name, full_path))
        return mod
    
    if last_import_exc is not None:
      etype, exc, tb = last_import_exc
      raise exc
    
    raise ImportError('Unknown module: %s' % full_name)
  
  def _LoadFileWithImportLib(self, full_name, full_path, is_pkg):
    """ Load file using importlib's functions -- handles .so & .pyd and .py,
    .pyc, & .pyo if USE_OUR_METHODS_FOR_PY_FILES is false.  Returns mod, exc_info
    pair; 1 or the other should be None """
    
    parts = full_name.split('.')
    path = os.path.dirname(full_path)
    if is_pkg:
      path = os.path.dirname(path)
   
    try:
      spec = importlib.machinery.PathFinder.find_spec(full_name, [path])
      self._Log2("spec = {}".format(spec))
      if spec is None:
        raise ImportError("Failed to find spec for %s" % full_name)
      mod = importlib.util.module_from_spec(spec)
      if mod is None:
        raise ImportError("Failed to get module from spec for %s" % full_name)
      spec.loader.exec_module(mod)
      
      if '.' in full_name:
        parent_mod = sys.modules['.'.join(parts[:-1])]
        setattr(parent_mod, parts[-1], mod)
      sys.modules[full_name] = mod
    
    except (ImportError, ModuleNotFoundError):
      last_import_exc = sys.exc_info()
      mod = None
    else:
      last_import_exc = None
      
    return mod, last_import_exc
      
  def _LoadWingCode(self, mod_name, filename, is_pkg):
    """ Load a module from a .wingcode or .py file, using a .pyc / .pyo
    file if possible """
    
    self._Log2("_load_wing_code(mod_name=%r, filename=%r, is_pkg=%r)" % (mod_name, filename, is_pkg))

    cache_pyc = self._GetPycFilenameInCache(filename)
    
    def update_linecache(co):
      """ Insert a dummy entry into linecache so it doesn't try to
      read the file; it tries over and over """
      
      if not self.WRITE_EMPTY_LINECACHE_ENTRIES:
        return
      
      linecache.cache[co.co_filename] = (None, None, [], co.co_filename)

    def try_pyc_file(src_mtime):
      """ Try loading mod from .pyc file.  Return None if not successful """
      
      if cache_pyc is None or not os.path.exists(cache_pyc):
        return None
      
      co = self._LoadFromPyc(mod_name, cache_pyc, filename, mtime)
      if co is None:
        return None

      update_linecache(co)
      
      # Pretend that the .pyc file is in the same dir as the src
      basename = os.path.basename(filename)
      if cache_pyc.endswith(('.pyc', '.pyo')):
        pos = basename.rfind('.')
        if pos > 0:
          basename = basename[:pos]
        basename += cache_pyc[-4:]
        
      file_in_mod = os.path.join(os.path.dirname(filename), basename)

      mod = self._ExecMod(mod_name, file_in_mod, co, is_pkg)
      return mod
    
    try:
      f = open(filename, 'rb')
      try:
        mtime = int(os.fstat(f.fileno()).st_mtime)
        mod = try_pyc_file(mtime)
        if mod is not None:
          self._Log2('Using mod loaded from .pyc: %r' % (mod_name, ))
          return mod
        
        data = f.read()
      finally:
        f.close()
    except (IOError, OSError):
      return None
    
    if data.startswith(_MakeBytes('\0')):
      data = data[1:].translate(self._TRANSLATE)

    # XXX This is now done before creating the wingcode file
    ## Cheap universal newline translating
    #data = data.replace(_MakeBytes('\r\n'), _MakeBytes('\n'))
    #data = data.replace(_MakeBytes('\r'), _MakeBytes('\n'))
      
    if filename.endswith(self.WING_CODE_EXT):
      compile_filename = filename[:-len(self.WING_CODE_EXT)] + '.py'      
    else:
      compile_filename = filename
    
    co = self._CompileMod(mod_name, data, filename, compile_filename)
    update_linecache(co)

    if cache_pyc is not None:
      self._WriteToPyc(cache_pyc, co, mtime)
    
    return self._ExecMod(mod_name, compile_filename, co, is_pkg)
    
  def _ExecMod(self, mod_name, filename, co, is_pkg):
    """ Exec co in a new module.  The module gets added to sys.modules before
    the co is exec'd """
    
    # Convert to byte strings in 2.x; use 'replace' so exceptions aren't thrown
    # Filenames that don't decode to fs encoding are likely to cause problems
    if sys.version_info < (3, 0):
      fs_encoding = sys.getfilesystemencoding()
      if fs_encoding is None:
        fs_encoding = sys.getdefaultencoding()
        
      if isinstance(filename, unicode):
        filename = filename.encode(fs_encoding, 'replace')
      if isinstance(mod_name, unicode):
        mod_name = mod_name.encode('ascii', 'replace')
    
    if sys.hexversion >= 0x03070000:
      mod = types.ModuleType(mod_name)
    else:
      mod = imp.new_module(mod_name)
    mod.__loader__ = self
    mod.__file__ = filename

    if is_pkg:
      mod.__path__ = [os.path.dirname(filename)]
      mod.__package__ = mod_name
    else:
      name_parts = mod_name.split('.')
      if len(name_parts) > 1:
        mod.__package__ = '.'.join(name_parts[:-1])
    
    sys.modules[mod_name] = mod
    exec(co, mod.__dict__)
    return mod

  def _CompileMod(self, name, source, filename, compile_filename=None):
    """ Compile & return code object.  Assumes encoding is utf8, which is valid for
    all Wing files """
    
    self._Log2("_compile_mod called(name=%r, filename=%r)" % (name, filename))
   
    if compile_filename is None:
      compile_filename = filename

    if sys.version_info[:2] >= (3, 0):
      source = str(source, 'utf8')
      
    if not source.endswith('\n'):
      source += '\n'

    co = compile(source, compile_filename, 'exec', 0, True)
    return co
  
  def _LoadFromPyc(self, name, pyc_name, src_full_path=None, src_mtime=None):
    """ Try to load code object from .pyc file.  Returns None if unable to """
    
    pyc_suffix = None
    for suffix in _GetModuleSuffixes():
      if suffix in ('.pyc', '.pyo'):
        pyc_suffix = suffix
        break
    if pyc_suffix is None:
      return None

    if src_mtime is None and src_full_path is not None:
      try:
        src_mtime = int(os.stat(src_full_path).st_mtime)
      except (IOError, OSError):
        src_mtime = None
        
    try:
      f = open(pyc_name, 'rb')
    except (IOError, OSError):
      return None
    
    try:
      try:
        magic = f.read(4)
        if magic != kMagic:
          return None
        if src_mtime is not None:
          pyc_mtime = _MarshalRead4ByteInt(f)
          if pyc_mtime != int(src_mtime):
            return None
        else:
          f.seek(4)
          
        co = marshal.load(f)

      except:
        self._Log('Unable to load code object from pyc file: %r' % (pyc_name, ), exc=True)
        return None

    finally:
      f.close()
      
    if not isinstance(co, self.CODE_OBJECT_TYPE):
      self._Log('Object loaded from .pyc not a code object: %r' % (repr(co), ))
      return None
    
    self._Log2('Loaded code object from .pyc file: %r' % (pyc_name, ))
    return co
      
  def _WriteToPyc(self, pyc_name, co, mtime):
    
    # Create directory if needed and the registered cache directory
    # exists
    dirname = os.path.dirname(pyc_name)
    if not os.path.exists(dirname):
      found_parent = None
      for registered_dir in self._cache_dirs.values():
        if os.path.normpath(dirname).startswith(os.path.normpath(registered_dir)):
          found_parent = registered_dir
          break
      if found_parent is None:
        self._Log('Unable to find registered cache directory for: %r'
                  % (pyc_name, ))
        return False
      if not os.path.exists(found_parent):
        self._Log('Registered cache directory for .pyc does not exist: %r'
                  % (pyc_name, ))
        self._Log('  Registered cache directory: %r' % (found_parent, ))
        return False
        
      try:
        os.makedirs(dirname, self.CACHE_DIR_UMASK)
      except (IOError, OSError):
        self._Log('Failed to create directory: %r' % (dirname, ))
        return False
      
      self._Log('Create directory: %r' % (dirname, ))
    
    try:
      f = open(pyc_name, 'wb')
    except (IOError, OSError):
      self._Log('Writing .pyc file failed: %r' % (pyc_name, ), exc=True)
      return
    
    try:
      try:
        f.write(kMagic)
        _MarshalWrite4ByteInt(0, f)
        marshal.dump(co, f)
        
        f.seek(4)
        _MarshalWrite4ByteInt(mtime, f)
        failed = False
        self._Log('Successfully wrote .pyc file: %r' % (pyc_name, ))
      except:
        self._Log('Writing .pyc file failed: %r' % (pyc_name, ), exc=True)
        failed = True
        
    finally:
      f.close()
  
    if failed:
      try:
        os.remove(pyc_name)
      except (IOError, OSError):
        # May be removed by another process
        self._Log('Removing .pyc file failed (may be removed by other process): %r'
                  % (pyc_name, ), exc=True)
      
    return (not failed)
  
  def _GetPycFilenameInCache(self, full_path):
    
    full_path = os.path.normpath(full_path)
    
    dirname = os.path.dirname(full_path)
    cache_dir = self._GetCacheDir(dirname)
    if cache_dir is None:
      return None
    
    name = os.path.basename(full_path)
    if sys.version_info[:2] < (3, 0):
      name += '.cpython-%s%s' % sys.version_info[:2]
    else:
      if sys.hexversion >= 0x03070000:
        name += '.' + sys.implementation.cache_tag
      else:
        name += '.' + imp.get_tag()
      
    if __debug__:
      name += '.pyc'
    else:
      name += '.pyo'
      
    cache_full_path = cache_dir + '/' + name
    return cache_full_path
    
  def _GetCacheDir(self, dirname):
    """ Get cache dir for source directory.  The cache directory
    may be a sub directory of one of the registered directories """

    if os.sep != '/':
      dirname = dirname.replace(os.sep, '/')
    
    parts = dirname.split('/')
    for i in range(len(parts), -1, -1):
      cache_dir = self._cache_dirs.get('/'.join(parts[:i]))
      if cache_dir is not None:
        if i < len(parts):
          cache_dir += '/' + '/'.join(parts[i:])
        return cache_dir
      
    return None

def _MarshalWrite4ByteInt(val, f):
  
  byte_list = [
    val & 0xff,
    (val >> 8) & 0xff, 
    (val >> 16) & 0xff, 
    (val >> 24) & 0xff, 
  ]
  
  if sys.version_info[:2] < (3, 0):
    s = ''.join(map(chr, byte_list))
  else:
    s = bytes(byte_list)
    
  f.write(s)
  
def _MarshalRead4ByteInt(f):
  
  byte_str = f.read(4)
  
  if sys.version_info[:2] < (3, 0):
    val_list = map(ord, byte_str)
  else:
    val_list = byte_str

  val = (val_list[0] 
         | (val_list[1] << 8)
         | (val_list[2] << 16)
         | (val_list[3] << 24))
  return val
