python3 3.2.3-6 / usr / bin / dh_python3

#! /usr/bin/python3
# vim: et ts=4 sw=4

# Copyright © 2010-2012 Piotr Ożarowski <piotr@debian.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import logging
import os
import re
import sys
from filecmp import cmp as cmpfile
from optparse import OptionParser, SUPPRESS_HELP
from os.path import isdir, islink, exists, join, split
from shutil import rmtree, copy as fcopy
from stat import ST_MODE, S_IXUSR, S_IXGRP, S_IXOTH
sys.path.insert(1, '/usr/share/python3/')
from debpython.debhelper import DebHelper
from debpython.depends import Dependencies
from debpython.version import SUPPORTED, DEFAULT, \
    getver, vrepr, parse_pycentral_vrange, \
    vrange_str
from debpython.pydist import validate as validate_pydist, \
                             PUBLIC_DIR_RE
from debpython.tools import sitedir, fix_shebang, shebang2pyver, clean_egg_name
from debpython.option import Option

# initialize script
logging.basicConfig(format='%(levelname).1s: %(module)s:%(lineno)d: '
                           '%(message)s')
log = logging.getLogger(__name__)
os.umask(0o22)

# Tag that will be added to .so files without one.  Because these values are
# different between versions of Python 3 (e.g. 3.2 has dmu but 3.3 only has
# dm), this maps vrepr()'s to extension templates.
EXTENSION_TAGS = {
    '3.2': 'cpython-%smu',
    '3.3': 'cpython-%sm',
    }
DBG_EXTENSION_TAGS = {
    '3.2': 'cpython-%sdmu',
    '3.3': 'cpython-%sdm',
    }
TAG_RE = re.compile(r'-([0-9]{2})[^-.]*\.so$')

# naming conventions used in the file:
# * version - tuple of integers
# * ver - string representation of version
# * vrange - version range, pair of max and min versions
# * fn - file name (without path)
# * fpath - file path


### FILES ######################################################
def fix_locations(package):
    """Move files to the right location."""
    for version in SUPPORTED:
        ver = vrepr(version)
        to_check = [i % ver for i in (\
                    'usr/local/lib/python%s/site-packages',
                    'usr/local/lib/python%s/dist-packages',
                    'usr/lib/python%s/site-packages',
                    'usr/lib/python%s/dist-packages',
                    'var/lib/python-support/python%s',
                    'usr/lib/pymodules/python%s')]
        dstdir = sitedir(version, package)

        for location in to_check:
            srcdir = "debian/%s/%s" % (package, location)
            if isdir(srcdir):
                log.warn('Python %s should install files in %s. '
                         'Did you forget "--install-layout=deb"?',
                         ver, sitedir(version))
                # TODO: what about relative symlinks?
                log.debug('moving files from %s to %s', srcdir, dstdir)
                share_files(srcdir, dstdir, package.endswith('-dbg'))
                parent_dir = '/'.join(srcdir.split('/')[:-1])
                if exists(parent_dir) and not os.listdir(parent_dir):
                    os.rmdir(parent_dir)

        # do the same with debug locations
        dbg_to_check = ['usr/lib/debug/%s' % i for i in to_check]
        dbg_to_check.append("usr/lib/debug/usr/lib/pyshared/python%s" % ver)
        dstdir = sitedir(version, package, gdb=True)

        for location in dbg_to_check:
            srcdir = "debian/%s/%s" % (package, location)
            if isdir(srcdir):
                log.debug('moving files from %s to %s', srcdir, dstdir)
                share_files(srcdir, dstdir, package.endswith('-dbg'))
                parent_dir = '/'.join(srcdir.split('/')[:-1])
                if exists(parent_dir) and not os.listdir(parent_dir):
                    os.rmdir(parent_dir)


def share_files(srcdir, dstdir, dbg_package=False):
    """Try to move as many files from srcdir to dstdir as possible."""
    for i in os.listdir(srcdir):
        fpath1 = join(srcdir, i)
        if i.rsplit('.', 1)[-1] == 'so' and 'cpython' not in i:
            # try to rename extension here as well (in :meth:`scan` info about
            # Python version is gone)
            public_dir = PUBLIC_DIR_RE.match(srcdir)
            fpath1_orig = fpath1
            i = tagged_extname(i, public_dir.groups()[0], dbg_package)
            fpath1 = join(srcdir, i)
            log.warn('renaming %s to %s', fpath1_orig, fpath1)
            os.rename(fpath1_orig, fpath1)
        fpath2 = join(dstdir, i)
        if not exists(fpath2):
            os.renames(fpath1, fpath2)
            continue
        if isdir(fpath1):
            share_files(fpath1, fpath2, dbg_package)
        elif cmpfile(fpath1, fpath2, shallow=False):
            os.remove(fpath1)
        # XXX: check symlinks

    if exists(srcdir) and not os.listdir(srcdir):
        os.rmdir(srcdir)


### PACKAGE DETAILS ############################################
def tagged_extname(fname, version, dbg_package=False):
    """Return tagged extension name for given file & version."""
    extkey = vrepr(version)  # make sure it's a string
    vers = extkey.replace('.', '')
    if vers == '3':
        # Assume that .so files without tags in /usr/lib/python3 are built for
        # the default Python 3 version.
        extkey = '.'.join(str(digit) for digit in DEFAULT)
        vers = extkey.replace('.', '')
    try:
        ext_tag = EXTENSION_TAGS[extkey]
        dbg_tag = DBG_EXTENSION_TAGS[extkey]
    except KeyError:
        raise ValueError('No known tag for Python version {}'.format(extkey))
    if fname.endswith('_d.so'):
        fname = "%s.%s.so" % (fname[:-5], dbg_tag % vers)
    elif dbg_package:
        fname = "%s.%s.so" % (fname[:-3], dbg_tag % vers)
    else:
        fname = "%s.%s.so" % (fname[:-3], ext_tag % vers)
    return fname


def scan(package, dname=None, options=None):
    """Gather statistics about Python files in given package."""
    r = {'requires.txt': set(),
         'shebangs': set(),
         'private_dirs': {},
         'compile': False,
         'ext': set()}

    dbg_package = package.endswith('-dbg')

    if not dname:
        proot = "debian/%s" % package
        if dname is False:
            private_to_check = []
        else:
            private_to_check = [i % package for i in
                                ('usr/lib/%s', 'usr/lib/games/%s',
                                'usr/share/%s', 'usr/share/games/%s')]
    else:
        # scan private directory *only*
        proot = join('debian', package, dname.strip('/'))
        private_to_check = [dname[1:]]

    for root, dirs, file_names in os.walk(proot):
        # ignore Python 2.X locations
        if '/usr/lib/python2' in root or\
           '/usr/local/lib/python2' in root or\
           '/usr/share/pyshared/' in root or\
           '/usr/lib/pyshared/' in root:
            # warn only once
            tmp = root.replace('/local', '').split('/')
            if len(tmp) == 5:  # debian/package/usr/foo/bar
                log.warning('Python 2.x location detected, '
                            'please use dh_python2: %s', root)
            continue

        bin_dir = private_dir = None
        public_dir = PUBLIC_DIR_RE.match(root)
        if not public_dir:
            for i in private_to_check:
                if root.startswith(join('debian', package, i)):
                    private_dir = '/' + i
                    break
            else:  # i.e. not public_dir and not private_dir
                if len(root.split('/', 6)) < 6 and (\
                   root.endswith('/sbin') or root.endswith('/bin') or\
                   root.endswith('/usr/games')):
                   # /(s)bin or /usr/(s)bin or /usr/games
                    bin_dir = root

        for name in dirs:
            if name == '__pycache__':
                rmtree(join(root, name))
                dirs.remove(name)
                continue
            # handle EGG related data (.egg-info dirs)
            if name.endswith('.egg-info'):
                if dbg_package and options.clean_dbg_pkg:
                    rmtree(join(root, name))
                    dirs.remove(name)
                    continue
                clean_name = clean_egg_name(name)
                if clean_name != name:
                    log.info('renaming %s to %s', name, clean_name)
                    os.rename(join(root, name), join(root, clean_name))
                    dirs[dirs.index(name)] = clean_name
        if root.endswith('.egg-info'):
            if 'requires.txt' in file_names:
                r['requires.txt'].add(join(root, 'requires.txt'))
            continue

        # check files
        for fn in sorted(file_names):
            # sorted() to make sure .so files are handled before .so.foo
            fpath = join(root, fn)
            if not exists(fpath):
                # could be removed while handling .so symlinks
                if islink(fpath) and '.so.' in split(fpath)[-1]:
                    # dangling symlink to (now removed/renamed) .so file
                    # which wasn't removed yet (see test3's quux.so.0)
                    log.info('removing symlink: %s', fpath)
                    os.remove(fpath)
                continue
            fext = fn.rsplit('.', 1)[-1]
            if fext in ('pyc', 'pyo'):
                os.remove(fpath)
                continue
            if public_dir:
                if fext == 'so' and islink(fpath):
                    dstfpath = fpath
                    links = set()
                    while islink(dstfpath):
                        links.add(dstfpath)
                        dstfpath = join(root, os.readlink(dstfpath))
                    if exists(dstfpath) and '.so.' in split(dstfpath)[-1]:
                        # rename .so.$FOO symlinks, remove other ones
                        for lpath in links:
                            log.info('removing symlink: %s', lpath)
                            os.remove(lpath)
                        log.info('renaming %s to %s', dstfpath, fn)
                        os.rename(dstfpath, fpath)
                if dbg_package and options.clean_dbg_pkg and \
                   fext not in ('so', 'h'):
                    os.remove(join(root, fn))
                    continue
                # assume all extensions were build for cPython
                if fext == 'so' and 'cpython' not in fn:
                    old_fn = fn
                    fn = tagged_extname(fn, public_dir.groups()[0], dbg_package)
                    log.warn('renaming %s to %s', old_fn, fn)
                    os.rename(join(root, old_fn), join(root, fn))

            elif private_dir:
                if exists(fpath):
                    mode = os.stat(fpath)[ST_MODE]
                    if mode & S_IXUSR or mode & S_IXGRP or mode & S_IXOTH:
                        if (options.no_shebang_rewrite or \
                            fix_shebang(fpath, options.shebang)) and \
                           not options.ignore_shebangs:
                            res = shebang2pyver(fpath)
                            if res:
                                r['private_dirs'].setdefault(private_dir, {})\
                                    .setdefault('shebangs', set()).add(res)

            if public_dir or private_dir:
                if fext == 'so':
                    tagver = TAG_RE.search(fn)
                    if tagver is None:
                        # yeah, python3.1 is not covered, but we don't want to
                        # mess with non-Python libraries, don't we?
                        continue
                    tagver = tagver.group(1)
                    tagver = getver("%s.%s" % (tagver[0], tagver[1]))
                    (r if public_dir else
                     r['private_dirs'].setdefault(private_dir, {}))\
                    .setdefault('ext', set()).add(tagver)
                    continue
                elif fext == 'py':
                    (r if public_dir else
                     r['private_dirs'].setdefault(private_dir, {}))\
                    ['compile'] = True
                    continue

            # .egg-info files
            if fn.endswith('.egg-info'):
                clean_name = clean_egg_name(fn)
                if clean_name != fn:
                    log.info('renaming %s to %s', fn, clean_name)
                    os.rename(join(root, fn), join(root, clean_name))
                continue
            # search for scripts in bin dirs
            if bin_dir:
                if (options.no_shebang_rewrite or \
                    fix_shebang(fpath, options.shebang)) and \
                   not options.ignore_shebangs:
                     res = shebang2pyver(fpath)
                     if res:
                         r['shebangs'].add(res)

    if dbg_package and options.clean_dbg_pkg:
        # remove empty directories in -dbg packages
        proot = proot + '/usr/lib'
        for root, dirs, file_names in os.walk(proot, topdown=False):
            if '-packages/' in root and not file_names:
                try:
                    os.rmdir(root)
                except Exception:
                    pass

    log.debug("package %s details = %s", package, r)
    return r


################################################################
def main():
    usage = '%prog -p PACKAGE [-V [X.Y][-][A.B]] DIR [-X REGEXPR]\n'
    parser = OptionParser(usage, version='%prog 3.1',
                                   option_class=Option)
    parser.add_option('--no-guessing-deps', action='store_false',
        dest='guess_deps', default=True, help='disable guessing dependencies')
    parser.add_option('--skip-private', action='store_true', default=False,
        help='don\'t check private directories')
    parser.add_option('-v', '--verbose', action='store_true', default=False,
        help='turn verbose mode on')
    # arch=False->arch:all only, arch=True->arch:any only, None->all of them
    parser.add_option('-i', '--indep', action='store_false',
        dest='arch', default=None,
        help='act on architecture independent packages')
    parser.add_option('-a', '-s', '--arch', action='store_true',
        dest='arch', help='act on architecture dependent packages')
    parser.add_option('-q', '--quiet', action='store_false', dest='verbose',
        help='be quiet')
    parser.add_option('-p', '--package', action='append',
        help='act on the package named PACKAGE')
    parser.add_option('-N', '--no-package', action='append',
        help='do not act on the specified package')
    parser.add_option('--compile-all', action='store_true', default=False,
        help='compile all files from given private directory in postinst, '
             'not just the ones provided by the package')
    parser.add_option('-V', type='version_range', dest='vrange',
        help='specify list of supported Python versions. ' +\
             'See py3compile(1) for examples')
    parser.add_option('-X', '--exclude', action='append', dest='regexpr',
        help='exclude items that match given REGEXPR. You may use this option '
             'multiple times to build up a list of things to exclude.')
    parser.add_option('--depends', action='append',
        help='translate given requirements into Debian dependencies '
             'and add them to ${python:Depends}. '
             'Use it for missing items in requires.txt.')
    parser.add_option('--recommends', action='append',
        help='translate given requirements into Debian '
             'dependencies and add them to ${python:Recommends}')
    parser.add_option('--suggests', action='append',
        help='translate given requirements into Debian '
             'dependencies and add them to ${python:Suggests}')
    parser.add_option('--shebang',
        help='use given command as shebang in scripts')
    parser.add_option('--ignore-shebangs', action='store_true', default=False,
        help='do not translate shebangs into Debian dependencies')
    parser.add_option('--no-dbg-cleaning', action='store_false',
            dest='clean_dbg_pkg', default=True,
        help='do not remove files from debug packages')
    parser.add_option('--no-shebang-rewrite', action='store_true',
            default=False, help='do not rewrite shebangs')
    # ignore some debhelper options:
    parser.add_option('-O', help=SUPPRESS_HELP)

    options, args = parser.parse_args(sys.argv[1:] + \
                    os.environ.get('DH_OPTIONS', '').split())
    # regexpr option type is not used so lets check patterns here
    for pattern in options.regexpr or []:
        # fail now rather than at runtime
        try:
            pattern = re.compile(pattern)
        except Exception:
            log.error('regular expression is not valid: %s', pattern)
            exit(1)

    if not args:
        private_dir = None
    else:
        private_dir = args[0]
        if not private_dir.startswith('/'):
            # handle usr/share/foo dirs (without leading slash)
            private_dir = '/' + private_dir
    # TODO: support more than one private dir at the same time (see :meth:scan)
    if options.skip_private:
        private_dir = False

    if options.verbose or os.environ.get('DH_VERBOSE') == '1':
        log.setLevel(logging.DEBUG)
        log.debug('argv: %s', sys.argv)
        log.debug('options: %s', options)
        log.debug('args: %s', args)

    try:
        dh = DebHelper(options)
    except Exception as e:
        log.error('cannot initialize DebHelper: %s', e)
        exit(2)
    if not options.vrange and dh.python_version:
        options.vrange = parse_pycentral_vrange(dh.python_version)

    for package, pdetails in dh.packages.items():
        if options.arch is False and pdetails['arch'] != 'all' or \
           options.arch is True and pdetails['arch'] == 'all':
            continue
        log.debug('processing package %s...', package)
        if not private_dir:
            fix_locations(package)
        stats = scan(package, private_dir, options)

        dependencies = Dependencies(package)
        dependencies.parse(stats, options)

        if stats['ext']:
            dh.addsubstvar(package, 'python3:Versions', \
                           ', '.join(sorted(vrepr(stats['ext']))))
            ps = package.split('-', 1)
            if len(ps) > 1 and ps[0] == 'python3':
                dh.addsubstvar(package, 'python3:Provides', \
                           ', '.join("python%s-%s" % (i, ps[1])\
                           for i in sorted(vrepr(stats['ext']))))

        pyclean_added = False  # invoke pyclean only once in maintainer script
        if stats['compile']:
            dh.autoscript(package, 'postinst', 'postinst-py3compile', '')
            dh.autoscript(package, 'prerm', 'prerm-py3clean', '')
            pyclean_added = True
        for pdir, details in stats['private_dirs'].items():
            if not details.get('compile'):
                continue
            if not pyclean_added:
                dh.autoscript(package, 'prerm', 'prerm-py3clean', '')
                pyclean_added = True

            args = pdir

            ext_for = details.get('ext')
            if ext_for is None:  # no extension
                shebangs = list(v for i, v in details.get('shebangs', []) if v)
                if not options.ignore_shebangs and len(shebangs) == 1:
                    # only one version from shebang
                    args += " -V %s" % vrepr(shebangs[0])
                elif options.vrange and options.vrange != (None, None):
                    args += " -V %s" % vrange_str(options.vrange)
            elif False in ext_for:
                # at least one extension's version not detected
                if options.vrange and '-' not in vrange_str(options.vrange):
                    ver = vrange_str(options.vrange)
                else:  # try shebang or default Python version
                    ver = (list(v for i, v in details.get('shebangs', [])
                           if v) or [None])[0] or DEFAULT
                    ver = vrepr(ver)
                dependencies.depend("python%s" % ver)
                args += " -V %s" % vrepr(ver)
            else:
                extensions = sorted(ext_for)
                vr = (extensions[0], extensions[-1])
                args += " -V %s" % vrange_str(vr)

            for pattern in options.regexpr or []:
                args += " -X '%s'" % pattern.replace("'", r"'\''")

            dh.autoscript(package, 'postinst', 'postinst-py3compile', args)

        dependencies.export_to(dh)

        pydist_file = join('debian', "%s.pydist" % package)
        if exists(pydist_file):
            if not validate_pydist(pydist_file):
                log.warning("%s.pydist file is invalid", package)
            else:
                dstdir = join('debian', package, 'usr/share/python3/dist/')
                if not exists(dstdir):
                    os.makedirs(dstdir)
                fcopy(pydist_file, join(dstdir, package))

    dh.save()

if __name__ == '__main__':
    main()