# keep-py-file: True
#########################################################################
""" updateutils.py  -- Utilities for updating Wingware software

Copyright (c) 1999-2018, Archaeopteryx Software, Inc.  All rights reserved.

Written by Stephan R.A. Deibel and John P. Ehresman

"""

# This is used during bootstrap and should not load any other modules in wingbase

import os
import time
import collections
import tempfile
import shutil
import stat
import tarfile
import fnmatch

try:
  import gettext
  _ = gettext.translation('src_wingbase', fallback=1).ugettext
except:
  def _(a):
    return a
  
#----------------------------------------------------------------------
def GetSHA1Digest(filenames, normalize_eol=False):
  """Get SHA1 digest for given files.  If multiple files are given, a single
  digest is computed for all of them appended in alphabetical order."""

  all_txt = []
  for filename in sorted(filenames):
    f = open(filename, 'rb')
    txt = f.read()
    f.close()

    if normalize_eol:
      txt = txt.replace('\r\n', '\n').replace('\r', '\n')
      
    all_txt.append(txt)

  digest = ComputeSHA1Digest(''.join(all_txt))
  
  return digest

#----------------------------------------------------------------------
def ReadSHA1SourceDigest(filename):
  """Read the source digest for given file.  It is stored in a filenames
  named xxx.shash for a filename of xxx.*"""
  
  sha_file = filename[:filename.rfind('.')] + '.shash'
  if not os.path.isfile(sha_file):
    return ''
  
  f = open(sha_file, 'r')
  txt = f.read()
  f.close()
  
  digest = txt[:txt.find(':')]
  return digest
  
#----------------------------------------------------------------------
def ComputeSHA1Digest(txt):
  """Compute SHA1 digest for given text"""

  try:
    import hashlib
    s = hashlib.sha1()
  except:
    print("Failed to import hashlib.  Exception is:")
    import traceback
    traceback.print_exc()
    import sha
    s = sha.sha()
  s.update(txt)
  digest = s.hexdigest()
  
  return digest

#-----------------------------------------------------------------------
def BuildDirectoryCatalog(dirname, omit_names=set(), omit_exts=set()):
  """Build catalog of all files in install_root so we know which files are
  for which OS and have an SHA1 hash of every file.  The partial paths
  in the catalog always use / as separator."""

  def partial_path(root, fn):
    fp = os.path.normpath(os.path.join(root, fn))
    return fp[len(dirname)+1:].replace(os.sep, '/')

  catalog = {}
  for root, subdirs, files in os.walk(dirname):
    for dn in subdirs:
      catalog[partial_path(root, dn)] = ('D', None, None)
    for fn in files:
      fp = os.path.join(root, fn)
      if fn in omit_names:
        continue
      if any(fn.endswith(ext) for ext in omit_exts):
        continue
      digest = GetSHA1Digest([fp], normalize_eol=True)
      source_digest = ReadSHA1SourceDigest(fp)
      catalog[partial_path(root, fn)] = ('F', digest, source_digest)

  return catalog

#-----------------------------------------------------------------------
def WriteDirectoryCatalog(catalog, catfile):
  """Convert given catalog to text and write to given file"""
  
  txt = []
  for fp in catalog:
    ftype, digest, source_digest = catalog[fp]
    if ftype == 'F':
      txt.append('%s|%s|%s|%s' % (ftype, digest, source_digest, fp))
    else:
      txt.append('%s|%s' % (ftype, fp))

  f = open(catfile, mode='w')
  f.write('\n'.join(txt) + '\n')
  f.close()

#-----------------------------------------------------------------------
def ReadDirectoryCatalog(fn):
  """Read given directory catalog from disk and parse it"""
  
  f = open(fn)
  txt = f.read()
  f.close()

  return ParseDirectoryCatalog(txt)

#----------------------------------------------------------------------
def ParseDirectoryCatalog(txt):
  """Parse the given directory catalog"""
  
  lines = txt.splitlines()
  
  catalog = {}
  for line in lines:
    line = line.strip()
    if not '|'in line:
      return _("Corrupt catalog: Missing | in line: ") + line, None
    ltype = line[:line.find('|')]
    line = line[2:]
    if ltype == 'F':
      digest = line[:line.find('|')]
      line = line[line.find('|')+1:]
      source_digest = line[:line.find('|')]
      line = line[line.find('|')+1:]
      catalog[line] = (ltype, digest, source_digest)
    elif ltype == 'D':
      catalog[line] = (ltype, None, None)
    else:
      return _("Corrupt catalog: Invalid line type: ") + line, None

  return None, catalog

#----------------------------------------------------------------------
def CompareDirectoryCatalogs(install_root, old, new, osname, whole_dirs=[], omitted={},
                             compute_deleted=True):
  """Find changed and deleted files in the update from old to new catalog.
  Set whole_dirs to a list of prefixes for which any file change will include
  that whole directory.  Set omitted to a dict from directory name to
  list of file name patterns to omit.  Set compute_deleted to False to
  prevent computing the set of files that were deleted and removing those
  from the new catalog.  This is done when staging because we're rebuilding
  a catalog from scratch and merging the OSes.  It should never be set to False
  when computing an update from completed install roots since then the
  catalog is also complete and contains all OSes."""
  
  def whole_dir_item(fp):
    for wp in whole_dirs:
      if fp.startswith(wp):
        return wp[:-1]
    return None
  
  omitted_but_changed = set()
  def ok_to_include(fp):
    for rdir in omitted:
      if fp.startswith(rdir):
        for omit in omitted[rdir]:
          if fnmatch.fnmatch(fp, omit):
            omitted_but_changed.add(fp)
            return False
    return True
    
  # Compute things that were added or changed since the old version
  changed_files = set()
  delete_first = set()
  added_whole_dirs = set()
  for fp in new:
    ftype, digest, source_digest= new[fp]
    old_ftype, old_digest, old_source_digest = old.get(fp, (None, None, None))

    # Need to delete anything that changes file type
    if old_ftype is not None and old_ftype != ftype:
        delete_first.add(fp)
        
    # Include files with changed digest or directories not in the old version
    # (to handle the case of a new empty dir); we compare the source digest,
    # if there is one, since digest of the file itself often changes even if
    # the source does not.
    if source_digest is not None:
      changed = source_digest != old_source_digest
    else:
      changed = digest != old_digest
    if changed or fp not in old:
      
      # Include all of any runtime with a changed file
      wp = whole_dir_item(fp)
      if wp is not None:
        if wp not in added_whole_dirs:
          changed_files.add(wp)
          delete_first.add(wp)
          added_whole_dirs.add(wp)

      # Other things can be patched file by file
      elif ok_to_include(fp):
        changed_files.add(fp)
  
  # Compute things that were deleted since the old version unless disabled by caller; note
  # that 'new' contains just items for the OS we're building or updating for.
  if compute_deleted:
    deleted = set(old)
    deleted.difference_update(new)
    for ditem in old:

      if ditem not in deleted:
        continue

      # Omit other OS's files since we can't know if they were deleted (__os__ files
      # and a few miscellaneous ones that are OS-specific but not in __os__ dirs)
      if '__os__' in ditem and '__os__/' + osname not in ditem:
        deleted.remove(ditem)
      elif ditem in ('src/wingapi.pyc', 'build-files/wing-install.py', 'wingdb.exe'):
        deleted.remove(ditem)
      elif ditem.startswith('bin/') and ditem[len('bin/'):].find('/') == -1 and \
           ditem.endswith(('.exe', '.dll')):
        deleted.remove(ditem)
        
      # Omit un-updateable files which should never be deleted
      elif not ok_to_include(ditem):
        deleted.remove(ditem)
  else:
    deleted = set()
    
  # Compute new catalog, which should change only entries which
  # we determined to be different.  This is needed to prevent changing
  # the digest of items that did not change according to source digest
  # (the file itself always changes, but the source has not so we are
  # keeping the old build target).  Also remove any deleted items.
  new_catalog = old.copy()
  for fp in changed_files:
    new_catalog[fp] = new[fp]
  for fp in deleted:
    del new_catalog[fp]
  
  return changed_files, delete_first, deleted, new_catalog, omitted_but_changed

#----------------------------------------------------------------------
def ParseVersionString(ver):
  """Convert #.#.#.# string into (#, #, #, #) tuple of ints"""
  
  ver = ver.split('.')
  if len(ver) != 4:
    return None
  try:
    return tuple([int(v) for v in ver])
  except:
    return None
    
#----------------------------------------------------------------------
def ParseReleases(txt, from_version, to_version, include_from=False):
  """Parse RELEASES.txt formatted release list for all releases between from_version
  and to_version, not including from_version."""

  summaries = collections.defaultdict(list)
  version_info = {}
  version = None
  skip_version = False
  for i, line in enumerate(txt.splitlines()):
    
    if line.startswith('#') or not line.strip():
      continue
    
    elif line.startswith('==='):
      parts = line.split()
      if len(parts) < 3:
        return _("Version spec on line %i is too short") % i, None, None
      version = parts[1]
      version_tuple = ParseVersionString(version)
      if version_tuple is None:
        return _("Bad version %s on line %i") % (version, i), None, None
      if not include_from and version_tuple <= from_version:
        return None, summaries, version_info
      if include_from and version_tuple < from_version:
        return None, summaries, version_info
      if version_tuple > to_version:
        skip_version = True
        continue
      else:
        skip_version = False
      rtype = parts[2].lower()
      if rtype not in ['alpha', 'beta', 'rc', 'final']:
        return _("Bad release type %s on line %i") % (rtype, i), None, None
      flags = parts[3:]
      for flag in flags:
        if flag not in ['force-download', 'patch']:
          return _("Bad release flag %s on line %i") % (flag, i), None, None
      version_info[version_tuple] = (rtype, flags)
        
    elif not skip_version:
      if version is None:
        return _("Line %i lists summary before version") % i, None, None
      parts = line.split()
      if len(parts) < 4:
        return _("Bad change summary on line %i") % i, None, None
      prod = parts[0].lower()
      if prod not in ['all', 'per', 'pro']:
        return _("Bad product type on line %i") % i, None, None
      ctype = parts[1].lower()
      if ctype not in ('f', 'c', 'b'):
        return _("Bad change type on line %i") % i, None, None
      change = ' '.join(parts[2:])
      summaries[version_tuple].append((parts[0], parts[1], change))
      
  return None, summaries, version_info

#----------------------------------------------------------------------
def FindAvailableReleases(prod, from_version, releases_fn, max_version=None):
  """Find all available releases for the given version by reading the given RELEASES.txt
  file. This takes into account whether or not a release can be applied with the update
  manager, and whether a release has installers. Returns latest_update, newer_release."""
  
  def relevant_changes(prod, changes):
    for cprod, ctype, change in changes:
      cprod = cprod.lower()
      if cprod == 'all' or prod == 'wingpro' or ((prod == 'wing-personal' and cprod in ('per', '101')) or
                                                 (prod == 'wing-101' and cprod == '101')):
        return True
      
    return False
      
  f = open(releases_fn)
  txt = f.read()
  f.close()

  if max_version is None:
    max_version = (1000, 1000, 1000, 1000)
    
  err, releases, version_info = ParseReleases(txt, from_version, max_version,
                                              include_from=True)
  if not releases:
    return None, None
  
  try:
    from_rtype, from_flags = version_info[from_version]
  except KeyError:
    from_rtype = 'alpha'
    from_flags = []
  
  update_candidates = []
  newer_release = None
  force_found = False
  for ver in reversed(sorted(releases)):
    if ver == from_version:
      break
    rtype, flags = version_info[ver]
    if 'force-download' in flags:
      force_found = True
    # Don't consider prereleases if not running one
    if rtype in ('alpha', 'beta', 'rc') and from_rtype not in ('alpha', 'beta', 'rc'):
      continue
    # Ignore releases with no changes for selected product
    if not relevant_changes(prod, releases[ver]):
      continue
    # Save latest release with installers
    if newer_release is None and 'patch' not in flags:
      newer_release = ver
    # Find latest release to which we can update without an installer
    if 'force-download' not in flags:
      update_candidates.append(ver)

  if force_found:
    update_candidates = []
    
  update_candidates.sort()
  if update_candidates:
    update_version = update_candidates[-1]
  else:
    update_version = None

  if newer_release == update_version:
    newer_release = None
  
  return update_version, newer_release
  
#----------------------------------------------------------------------
def GenerateUpdate(from_version, to_version, install_root, osname, from_catalog,
                   to_catalog, target_file, build_utils):
  """Generate an update file from the given install root"""

  # Never update anything in bootstrap or selected files at top level
  # Updating these requires making a new release with force-download
  # set in the RELEASES.txt file
  omitted = {
    'bootstrap/': ('*',),
    '/': ('wing', 'wing-personal', 'wing-101'),
    'bin': ('*.exe', '*.dll'),
  }
  
  # Include whole directories for runtimes if anything changes
  whole_dirs = []
  runtimes_dir = install_root+'/bin/__os__/%s/' % osname
  if os.path.exists(runtimes_dir):
    for fn in os.listdir(install_root+'/bin/__os__/%s/' % osname):
      if fn.startswith('runtime-'):
        whole_dirs.append('bin/__os__/%s/%s/' % (osname, fn))

  # Find changed or new files and deleted files
  changed_or_new, delete_first, deleted, new_catalog, omitted_but_changed = \
    CompareDirectoryCatalogs(install_root, from_catalog, to_catalog, osname, whole_dirs, omitted)
  if omitted_but_changed:
    warning = _("Some unpatchable files changed:"), ', '.join(omitted_but_changed)
  else:
    warning = None
  if not changed_or_new and not deleted:
    return _("No changes for the update"), warning
  
  # Make a temporary directory with the changed files.  We skip over files for a
  # different OS, except if they are for the debugger / remote agent since those
  # are used by the IDE to install on a remote host.
  upload_tmp = tempfile.mkdtemp() + '/' + to_version
  build_utils.EnsureDirExists(upload_tmp+'/xxx')
  for fp in changed_or_new:
    if '__os__' in fp and not '__os__/' + osname in fp and 'bin/dbg/src' not in fp:
      continue
    ifp = install_root + '/' + fp
    tfp = upload_tmp + '/' + fp
    build_utils.EnsureDirExists(tfp)
    build_utils.CopyFileIfNewer(ifp, tfp, force=1, recursive=1)
    
  # Also write the generated catalog from CompareDirectoryCatalogs, which changes
  # entries only for files that actually changed (as opposed to just being
  # different; for many files this is determined by looking at source hashes)
  WriteDirectoryCatalog(new_catalog, upload_tmp+'/CATALOG.txt')

  # Add a registry of deleted files
  f = open(upload_tmp+'/DELETED.txt', 'w')
  f.write('\n'.join(sorted(deleted)))
  f.close()

  # Create update tarfile
  temp_tar = tempfile.mktemp(suffix='.tar.bz2')
  f = tarfile.TarFile.open(temp_tar, 'w:bz2')
  f.add(upload_tmp, to_version)
  f.close()
  build_utils.Delete(upload_tmp, force=1)
  if not os.path.exists(temp_tar):
    return _("Could not create update tarfile:"), warning

  # Copy it into place
  build_utils.EnsureDirExists(target_file)
  build_utils.CopyFileIfNewer(temp_tar, target_file)
  os.unlink(temp_tar)
  
  return None, warning

#-----------------------------------------------------------------------
def Delete(name, recursive=True, force=True):
  """ Delete files or directories (if recursive is true).  Returns whether it
  was able to delete everything; it tries to delete as much as it can even
  if it encounters errors.  This avoids bugs in shutil.rmtree that prevent
  removing read-only files."""
  
  deleted_all = True
  
  if os.path.isfile(name) or os.path.islink(name):
    if force and not os.path.islink(name):
      try:
        os.chmod(name, 438)  # octal 0666
      except OSError:
        pass
    try:  
      os.remove(name)
    except OSError:
      deleted_all = False
      
  elif os.path.isdir(name) and recursive:
    if force:
      try:
        os.chmod(name, 511)  # octal 0777
      except OSError:
        pass

    try:
      children = os.listdir(name)
    except OSError:
      children = ()
      deleted_all = False
    for child in children:
      sub_all = Delete(os.path.join(name, child), recursive, force)
      if not sub_all:
        deleted_all = False
        
    try:
      os.rmdir(name)
    except OSError:
      deleted_all = False
      
  return deleted_all

#----------------------------------------------------------------------
def RemoveOldUpdates(update_dirs, install_version):
  """Try to remove all old update directories"""
  
  failed = []
  for update_dir in update_dirs:
    
    try:
      name_list = os.listdir(update_dir)
    except:
      continue
    
    for name in name_list:
      fp = update_dir + '/' + name
      ver = NameToVersion(name)
      if ver is not None and ver < install_version:
        
        try:
          # Try rename first so it fails on win32 if any file is in use by another instance;
          # this also prevents leaving a partially deleted update that Wing may try to load
          os.rename(fp, fp+'-old')
          fp += '-old'
          Delete(fp)
        except:
          failed.append(fp)
    
  return failed
  
#-----------------------------------------------------------------------
def MarkUpdatesCorrupt(update_dirs):

  all_marked = True
  
  for update_dir in update_dirs:
    if not os.path.exists(update_dir):
      continue
    try:
      f = open(os.path.join(update_dir, 'CORRUPT.txt'), 'w')
      f.write(_("Updates are corrupt and need to be removed and reinstalled") + '\n')
      f.close()
    except:
      all_marked = False
      
  return all_marked
        
#----------------------------------------------------------------------
kMaxUpdateDirs = 3
def MergeUpdateDirs(code_dirs):
  """Try to merge older update directories.  Returns (fatal, error_list).
  When fatal is true, the updates are corrupted and should not be loaded."""
  
  errors = []
  
  # Omit WINGHOME and change to oldest first sorting
  code_dirs = code_dirs[:-1]
  code_dirs.reverse()

  # Assumption made by the implementation
  assert kMaxUpdateDirs >= 2
  
  if len(code_dirs) <= kMaxUpdateDirs:
    return False, []

  fatal_errors = []
  def try_fatal_rename(old, new):
    """Try a rename that on failure will corrupt the update area.  If it fails,
    we start without updates, although subsequent starts will likely fail entirely."""
    try:
      os.rename(old, new)
      return True
    except:
      fatal_errors.append((old, new))
      return False
    
  # Rename all the directories we're going to merge from/to all at once to prevent
  # concurrent update/merge by multiple instances of Wing.
  renamed = set()
  for from_ver, from_dir in code_dirs[:-kMaxUpdateDirs+1]:
    from_dir_busy = from_dir+'-busy'
    try:
      
      # Wait up to 2 hours before removing an existing -busy directory in case
      # it's due to a concurrent update/merge process
      if os.path.exists(from_dir_busy):
        mtime = os.stat(from_dir_busy)[stat.ST_MTIME]
        if mtime < time.time() - 2 * 60 * 60:
          if not Delete(from_dir_busy):
            errors.append(_("Failed to delete old busy update: %s") % from_dir_busy)
            raise RuntimeError("failed-delete")
        else:
          errors.append(_("Possibly concurrent update/merge on %s") % from_dir_busy)
          raise RuntimeError('concurrent-update-merge')
        
      # Try to rename it; fails on Windows if in use by another process
      os.rename(from_dir, from_dir_busy)
      renamed.add(from_dir)
      
    except:
      errors.append(_("Renaming %s failed; skipping update merge"))
        
      # Try to put back the dirs we already renamed; fatal if it fails
      for dirname in renamed:
        try_fatal_rename(dirname+'-busy', dirname)
      
      return bool(fatal_errors), errors
  
  # Merge oldest update directories one at a time; any that fail are left alone
  # if possible under their original name (w/o -busy)
  while len(code_dirs) > kMaxUpdateDirs:
    
    # These were all renamed above
    from_dir = code_dirs[0][1] + '-busy'
    to_dir = code_dirs[1][1] + '-busy'
    tmp_to_dir = to_dir + '-merge'
    
    try:
        
      # Copy the target directory to a temp directory so we can replace it atomically
      shutil.copytree(to_dir, tmp_to_dir)
      
      # Merge into temp dir and remove from_dir if is succeeds
      MergeUpdateDir(from_dir, tmp_to_dir)
      
      # Move temp merge dir into place and remove old dir
      if try_fatal_rename(to_dir, to_dir+'-old'):
        if try_fatal_rename(tmp_to_dir, to_dir):
          Delete(to_dir+'-old')
        else:
          try_fatal_rename(to_dir+'-old', to_dir)
      
    except:
      import traceback
      errors.append(_("Failed to merge update directories: %s -> %s; aborting merge") % (from_dir, to_dir))
      errors.append(traceback.format_exc())
      Delete(tmp_to_dir)
      if os.path.exists(from_dir):
        try_fatal_rename(from_dir, from_dir[:-len('-busy')])
    
    if fatal_errors:
      return True, errors
    
    code_dirs = code_dirs[1:]
    
  # Rename the final target directory; this is fatal if it fails
  if try_fatal_rename(to_dir, to_dir[:-len('-busy')]):
    return False, errors
  else:
    return True, errors

#----------------------------------------------------------------------
def MergeUpdateDir(from_dir, to_dir):
  """Merge one update directory into another, updating files as
  necessary"""

  if from_dir.endswith(os.sep):
    from_dir = from_dir[:-1]
  if to_dir.endswith(os.sep):
    to_dir = to_dir[:-1]
    
  # Read deleted files for newer version
  try:
    f = open(os.path.join(to_dir, 'DELETED.txt'))
    to_del = f.readlines()
    f.close()
  except:
    to_del = []
  to_del_set = set(to_del)
  
  # Get list of files in newer version
  newer_files = set()
  for dirname, dirs, files in os.walk(to_dir):
    for fn in files:
      fullpath = os.path.join(dirname, fn)
      relpath = fullpath[len(to_dir)+1:].replace(os.sep, '/')
      newer_files.add(relpath)
    
  # Copy over all files except: DELETED.txt, any since-deleted files,
  # and any files already present in newer update
  for dirname, dirs, files in os.walk(from_dir):
    for fn in files:
      if fn == 'DELETED.txt' and dirname == from_dir:
        continue
      fullpath = os.path.join(dirname, fn)
      relpath = fullpath[len(from_dir)+1:]
      if relpath in to_del_set:
        continue
      target = os.path.join(to_dir, relpath)
      if os.path.exists(target):
        continue
      if not os.path.exists(os.path.dirname(target)):
        # mode=0755 octal; plain int form works in all Python versions:
        os.makedirs(os.path.dirname(target), mode=493)
      shutil.copy2(fullpath, os.path.join(to_dir, relpath))
    
  # Read old DELETED.txt
  try:
    f = open(os.path.join(from_dir, 'DELETED.txt'))
    from_del = f.readlines()
    f.close()
  except:
    from_del = []

  # Remove any items that have been added again
  filtered_from_del = set()
  for relpath in from_del:
    if relpath not in newer_files:
      filtered_from_del.add(relpath)
      
  # Write merged DELETED.txt into newer dir
  filtered_from_del.update(to_del)
  f = open(os.path.join(to_dir, 'DELETED.txt'), 'w')
  f.write('\n'.join(sorted(filtered_from_del)))
  f.close()
  
  # Remove old update dir
  Delete(from_dir)
  
#----------------------------------------------------------------------
def ValidateInstall(code_dirs, ignores, progress_cb=None):
  """Validate that all the files in updates match the checksums in the latest catalog
  for the given code directories."""

  # This is currently used only in the IDE
  import bootstrap_utils
  osname = bootstrap_utils.NP_GetPlatform()
  
  # Some files to ignore because the installer changes them or they get rewritten
  # at runtime
  ignores.extend([
    'wingdbstub.py',
    'src/wingapi.pyo'
  ])
  
  def get_file_os(fn):
    
    # Determine OS for the file based on standard layout
    if '__os__' in fn:
      fn = fn[fn.find('__os__'):][len('__os__') + 1:]
      if not fn:
        return None
      fos = fn[fn.find('/')]
      return fos

    # A few files are placed at top level or in bin or bootstrap but are OS-specific
    # so need to be ignored on other OSes
    osfiles = [
      ('', ('wing', 'wing-personal', 'wing-101', 'wingdb-suid'), ('linux', 'osx')), 
      ('', ('wingdb.exe', ), ('win32',)), 
      ('', ('wing.icns',), ('osx',)), 
      ('bin', ('wing*.exe', 'console*.exe', '*.dll'), ('win32',)),
      ('bootstrap', ('run-wing-osx.sh',), ('osx')), 
      ('bootstrap', ('run-wing.sh', ), ('linux', 'osx')), 
    ]
    for prefix, patterns, oses in osfiles:
      if fn.startswith(prefix):
        nfn = fn[len(prefix):]
        if nfn.startswith('/'):
          nfn = nfn[1:]
        parts = nfn.split('/')
        for pat in patterns:
          if fnmatch.fnmatch(parts[0], pat):
            return oses
          
    # File is for all OSes
    return None
            
  latest_ver, latest_dir = code_dirs[0]
  fn = os.path.join(latest_dir, 'CATALOG.txt')
  error, catalog = ReadDirectoryCatalog(fn)

  if callable(progress_cb):
    progress_cb(0, len(catalog))
    
  if error:
    return [error]

  def ignore(pp):
    for pat in ignores:
      if fnmatch.fnmatch(pp, pat):
        return True
    return False
  
  def find_file(pp):
    assert not os.path.isabs(pp)
    for cv, cdir in code_dirs:
      fn = os.path.join(cdir, pp)
      if os.path.exists(fn):
        return fn
    return None

  errors = []
  start_time = time.time()
  for i, partial_path in enumerate(catalog):
    if callable(progress_cb) and time.time() > start_time + 1.0:
      start_time = time.time()
      progress_cb(i, len(catalog))
    file_type, digest, source_digest = catalog[partial_path]
    fn = find_file(partial_path)
    if fn is None:
      # Ignore missing files for other OSes
      foses = get_file_os(partial_path)
      if foses is None or osname in foses:
        errors.append((partial_path, _("Missing file: %s") % partial_path))
    elif ignore(partial_path):
      continue
    elif file_type == 'F' and not os.path.isfile(fn):
      errors.append((partial_path, _("Invalid Type: %s (expected file but found non-file)") % partial_path))
    elif file_type == 'D' and not os.path.isdir(fn):
      errors.append((partial_path, _("Invalid Type: %s (expected directory but found non-directory)") % partial_path))
    elif file_type == 'F':
      try:
        file_digest = GetSHA1Digest([fn], normalize_eol=True)
      except:
        errors.append((partial_path, _("Read failed:  %s") % partial_path))
      else:
        if file_digest != digest:
          errors.append((partial_path, _("Changed file: %s") % partial_path))
        
  errors.sort()
  errors = [e[1] for e in errors]
  
  return errors
    
#-----------------------------------------------------------------------
def NameToVersion(name):
  """Try to convert given directory name to a 4-int version tuple.  Returns
  None if it's not a version directory."""

  parts = name.split('.')
  if len(parts) not in (3, 4):
    return None
  try:
    retval = tuple([int(p) for p in parts])
    if len(retval) == 3:
      return retval + (0, )
    else:
      return retval
  except:
    return None

