/usr/bin/lintian

#!/usr/bin/perl -w
# {{{ Legal stuff
# Lintian -- Debian package checker
#
# Copyright (C) 1998 Christian Schwarz and Richard Braakman
#
# This program is free software.  It is distributed under the terms of
# the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, you can find it on the World Wide
# Web at http://www.gnu.org/copyleft/gpl.html, or write to the Free
# Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
# MA 02110-1301, USA.
# }}}

# {{{ libraries and such
use strict;
use warnings;

use Getopt::Long;

# }}}

# {{{ Application Variables

# Variables from %opt (defined below) we must export to %ENV
my @MUST_EXPORT = (qw(
    LINTIAN_LAB
    LINTIAN_ROOT
));
# LINTIAN_DEBUG, but that is handled separately
# TMPDIR, handled separatedly

# Environment variables Lintian cares about - the list contains
# the ones that can also be set via the config file
#
# %opt (defined below) will be updated with values of the env
# after parsing cmd-line options.  A given value in %opt is
# updated to use the ENV variable if the one in %opt is undef
# and ENV has a value.
#
# NB: Variables listed here are not always exported - use @MUST_EXPORT
# for that.
my @ENV_VARS = (
# LINTIAN_CFG  - handled manually
# LINTIAN_ROOT - handled manually
qw(
    LINTIAN_PROFILE
    LINTIAN_LAB
    TMPDIR
));


### "Normal" application variables

# Version number - Is replaced during build with sed, see d/rules
my $LINTIAN_VERSION = "2.5.10.4";      #External Version number
if ($LINTIAN_VERSION eq '<VERSION>') {
    # For some reason the version above has not be substituted.
    # Most likely this means we are a git clone or an unpacked
    # source package.  If so, we will use a version that best
    # describes our situation...
    my $guess = _guess_version (__FILE__);
    $LINTIAN_VERSION = $guess if $guess;
}
my $BANNER = "Lintian v$LINTIAN_VERSION"; #Version Banner - text form

# Variables used to record commandline options
# Commented out variables have "defined" checks somewhere to determine if
# they were set via commandline or environment variables
my $pkg_mode = 'auto';          # auto -- automatically search for
                                # binary and source pkgs
my $debug = 0;
my $check_everything = 0;       #flag for -a|--all switch
my $lintian_info = 0;           #flag for -i|--info switch
my $ftpmaster_tags = 0;         #flag for -F|--ftp-master-rejects switch
my $allow_root = 0;             #flag for --allow-root switch
my $keep_lab = 0;               #flag for --keep-lab switch
my $packages_file = 0;          #string for the -p option

my $no_conf = 0;                #flag for --no-cfg
my %opt;                        #hash of some flags from cmd or cfg
my %conf_opt;                   #names of options set in the cfg file

my %group_cache = ();           # Cache to store groups in case of group
                                # queries

# The profile search path except LINTIAN_ROOT/profiles
#  which will be added later (we dont know LINTIAN_ROOT
#  at this point)
my @prof_inc;
# In some (rare) cases, $ENV{HOME} will not be available.
# - Handle that gracefully by not emitting "Uninitialized ...".
push @prof_inc, "$ENV{HOME}/.lintian/profiles" if defined $ENV{HOME};
push @prof_inc, '/etc/lintian/profiles';


my $experimental_output_opts = undef;

my @certainties = qw(wild-guess possible certain);
my @display_level;
my %display_source = ();
my %suppress_tags = ();

my $pool;

my $action;
my $checks;
my $check_tags;
my $dont_check;
my @unpack_info;
my $cwd;
my $exit_code = 0;
my $LAB;

my %keep_coll;
my %collection_info;
my %check_abbrev;
my %unpack_infos;
my %requested_unpack;

# Timer handling (by default to nothing)
my $start_timer = sub { 0; };
my $finish_timer =  sub { ''; };

# }}}

# {{{ Setup Code

#turn off file buffering
$| = 1;

# Globally ignore SIGPIPE.  We'd rather deal with error returns from write
# than randomly delivered signals.
$SIG{PIPE} = 'IGNORE';

# reset locale definition (necessary for tar)
$ENV{'LC_ALL'} = 'C';
# reset timezone definition (also for tar)
$ENV{'TZ'}     = '';

# When run in some automated ways, Lintian may not have a PATH, but we assume
# we can call standard utilities without their full path.  If PATH is
# completely unset, add something basic.
$ENV{PATH} = '/bin:/usr/bin' unless $ENV{PATH};

# }}}

# {{{ Process Command Line

#######################################
# Subroutines called by various options
# in the options hash below.  These are
# invoked to process the commandline
# options
#######################################
# Display Command Syntax
# Options: -h|--help
sub syntax {
    print "$BANNER\n";
    print <<"EOT-EOT-EOT";
Syntax: lintian [action] [options] [--] [packages] ...
Actions:
    -c, --check               check packages (default action)
    -C X, --check-part X      check only certain aspects
    -F, --ftp-master-rejects  only check for automatic reject tags
    -r, --remove              remove package from the lab
    -R, --remove-lab          remove static lab
    -S, --setup-lab           set up static lab
    -T X, --tags X            only run checks needed for requested tags
    --tags-from-file X        like --tags, but read list from file
    -u, --unpack              only unpack packages in the lab
    -X X, --dont-check-part X don\'t check certain aspects
General options:
    -d, --debug               turn Lintian\'s debug messages ON
    -h, --help                display short help text
    --print-version           print unadorned version number and exit
    -q, --quiet               suppress all informational messages
    -v, --verbose             verbose messages
    -V, --version             display Lintian version and exit
Behaviour options:
    --allow-root              suppress lintian\'s warning when run as root
    --color never/always/auto disable, enable, or enable color for TTY
    --display-source X        restrict displayed tags by source
    -E, --display-experimental display "X:" tags (normally suppressed)
    --fail-on-warnings        return a non-zero exit status if warnings found
    -i, --info                give detailed info about tags
    -I, --display-info        display "I:" tags (normally suppressed)
    --keep-lab                keep lab after run, even if temporary
    -L, --display-level       display tags with the specified level
    -o, --no-override         ignore overrides
    --pedantic                display "P:" tags (normally suppressed)
    --profile X               Use the profile X or use vendor X checks
    --show-overrides          output tags that have been overriden
    --suppress-tags T,...     don\'t show the specified tags
    --suppress-tags-from-file X don\'t show the tags listed in file X
    -U X, --unpack-info X     specify which info should be collected
Configuration options:
    --arch ARCH               scan packages with architecture ARCH
    --area AREA               scan packages in this archive area (e.g. main)
    --archivedir ARCHIVEDIR   location of Debian archive to scan for packages
    --cfg CONFIGFILE          read CONFIGFILE for configuration
    --no-cfg                  do not read any config files
    --dist DIST               scan packages in this distribution (e.g. sid)
    -j X, --jobs X            limit the number of parallel unpacking jobs to X
    --lab LABDIR              use LABDIR as permanent laboratory
    --root ROOTDIR            use ROOTDIR instead of /usr/share/lintian
Package selection options:
    -a, --all                 process all packages in distribution
    -b, --binary              process only binary packages
    -p X, --packages-file X   process all files in file (special syntax!)
    --packages-from-file  X   process the packages in a file (if "-" use stdin)
    -s, --source              process only source packages
    --udeb                    process only udeb packages
EOT-EOT-EOT

    exit 0;
}

# Display Version Banner
# Options: -V|--version, --print-version
sub banner {
    if ($_[0] eq 'print-version') {
        print "$LINTIAN_VERSION\n";
    } else {
        print "$BANNER\n";
    }
    exit 0;
}

# Record action requested
# Options: -S, -R, -c, -u, -r
sub record_action {
    if ($action) {
        die("too many actions specified: $_[0]");
    }
    $action = "$_[0]";
}

# Record Parts requested for checking
# Options: -C|--check-part
sub record_check_part {
    if (defined $action and $action eq 'check' and $checks) {
        die('multiple -C or --check-part options not allowed');
    }
    if ($dont_check) {
        die('both -C or --check-part and -X or --dont-check-part options not allowed');
    }
    if ($action) {
        die("too many actions specified: $_[0]");
    }
    $action = 'check';
    $checks = "$_[1]";
}

# Record Parts requested for checking
# Options: -T|--tags
sub record_check_tags {
    if (defined $action and $action eq 'check' and $check_tags) {
        die('multiple -T or --tags options not allowed');
    }
    if ($checks) {
        die('both -T or --tags and -C or --check-part options not allowed');
    }
    if ($dont_check) {
        die('both -T or --tags and -X or --dont-check-part options not allowed');
    }
    if ($action) {
        die("too many actions specified: $_[0]");
    }
    $action = 'check';
    $check_tags = "$_[1]";
}

# Record Parts requested for checking
# Options: --tags-from-file
sub record_check_tags_from_file {
    my ($option, $name) = @_;
    open(my $file, '<', $name)
        or die("failed to open $name: $!");
    my @tags;
    for my $line (<$file>) {
        $line =~ s/^\s+//;
        $line =~ s/\s+$//;
        next unless $line;
        next if $line =~ /^\#/;
        push(@tags, split(/\s*,\s*/, $line));
    }
    close $file;
    record_check_tags($option, join(',', @tags));
}

# Record tags that should be suppressed.
# Options: --suppress-tags
sub record_suppress_tags {
    my ($option, $tags) = @_;
    for my $tag (split(/\s*,\s*/, $tags)) {
        $suppress_tags{$tag} = 1;
    }
}

# Record tags that should be suppressed from a file.
# Options: --suppress-tags-from-file
sub record_suppress_tags_from_file {
    my ($option, $name) = @_;
    open(my $file, '<', $name)
        or die("failed to open $name: $!");
    for my $line (<$file>) {
        chomp $line;
        $line =~ s/^\s+//;
        $line =~ s/(\#.*+|\s+)$//; #Remove trailing white-space/comments
        next unless $line;
        record_suppress_tags($option, $line);
    }
    close $file;
}

# Record Parts requested not to check
# Options: -X|--dont-check-part X
sub record_dont_check_part {
    if (defined $action and $action eq 'check' and $dont_check) {
        die('multiple -X or --dont-check-part options not allowed');
    }
    if ($checks) {
        die('both -C or --check-part and -X or --dont-check-part options not allowed');
    }
    if ($action) {
        die("too many actions specified: $_[0]");
    }
    $action = 'check';
    $dont_check = "$_[1]";
}


# Process for -U|--unpack-info flag
sub record_unpack_info {
    push @unpack_info, "$_[1]";
}

# Record what type of data is specified
# Options: -b|--binary, -s|--source, --udeb
sub record_pkgmode {
    $pkg_mode = 'binary' if $_[0] eq 'binary';
    $pkg_mode = 'source' if $_[0] eq 'source';
    $pkg_mode = 'udeb' if $_[0] eq 'udeb';
}

# Process -L|--display-level flag
sub record_display_level {
    my ($option, $level) = @_;
    my ($op, $rel);
    if ($level =~ s/^([+=-])//) {
        $op = $1;
    }
    if ($level =~ s/^([<>]=?|=)//) {
        $rel = $1;
    }
    my ($severity, $certainty) = split('/', $level);
    $op = '=' unless defined $op;
    $rel = '=' unless defined $rel;
    if (not defined $certainty) {
        if (grep { $severity eq $_ } @certainties) {
            $certainty = $severity;
            undef $severity;
        }
    }
    push(@display_level, [ $op, $rel, $severity, $certainty ]);
}

# Process -I|--display-info flag
sub display_infotags {
    push(@display_level, [ '+', '>=', 'wishlist' ]);
}

# Process --pedantic flag
sub display_pedantictags {
    push(@display_level, [ '+', '=', 'pedantic' ]);
}

# Process --display-source flag
sub record_display_source {
    $display_source{$_[1]} = 1;
}

# Process -q|--quite flag
sub record_quiet {
    $opt{'verbose'} = -1;
}

# Process deprecated flags
sub deprecated{
    print STDERR "warning: $_[0] is deprecated and may be removed\n";
    print STDERR "in a future Lintian release.\n";
}

# Process display-info and display-level options in cfg files
#  - dies if display-info and display-level are used together
#  - adds the relevant display level unless the command-line
#    added something to it.
#  - uses @display_level to track cmd-line appearences of
#    --display-level/--display-info
sub cfg_display_level {
    my ($var, $val) = @_;
    if ($var eq 'display-info' or $var eq 'pedantic'){
        die "$var and display-level may not both appear in the config file.\n"
            if $conf_opt{'display-level'};

        return unless $val; # case "display-info=no" (or "pedantic=no")

        # We are only supposed to modify @display_level if it was not
        # set by a command-line option.  However, both display-info
        # and pedantic comes here so we cannot determine this solely
        # by checking if @display_level is empty.  We use
        # "__conf-display-opts" to determine if @display_level was set
        # by a conf option or not.
        return if @display_level && !$conf_opt{'__conf-display-opts'};

        $conf_opt{'__conf-display-opts'} = 1;
        display_infotags() if $var eq 'display-info';
        display_pedantictags() if $var eq 'pedantic';
    } elsif ($var eq 'display-level'){
        foreach my $other (qw(pedantic display-info)) {
            die "$other and display-level may not both appear in the config file.\n"
                if $conf_opt{$other};
        }

        return if @display_level;
        $val =~ s/^\s++//;
        $val =~ s/\s++$//;
        foreach my $dl (split m/\s++/, $val) {
            record_display_level('display-level', $dl);
        }
    }

}

# Processes quiet and verbose options in cfg files.
# - dies if quiet and verbose are used together
# - sets the verbosity level ($opt{'verbose'}) unless
#   already set.
sub cfg_verbosity {
    my ($var, $val) = @_;
    if (($var eq 'verbose' && exists $conf_opt{'quiet'}) ||
        ($var eq 'quiet' && exists $conf_opt{'verbose'})) {
        die "verbose and quiet may not both appear in the config file.\n";
    }
    # quiet = no or verbose = no => no change
    return unless $val;
    # Do not change the value if set by command line.
    return if defined $opt{'verbose'};
    # quiet = yes => verbosity_level = -1
    #
    # technically this allows you to enable verbose by using "quiet =
    # -1" (etc.), but most people will probably not use this
    # "feature".
    $val = -$val if $var eq 'quiet';
    $opt{'verbose'} = $val;
}

# Process overrides option in the cfg files
sub cfg_override {
    my ($var, $val) = @_;
    return if defined $opt{'no-override'};
    # This option is inverted in the config file
    $opt{'no-override'} = !$val;
}

# Hash used to process commandline options
my %opthash = (                 # ------------------ actions
               'setup-lab|S' => \&record_action,
               'remove-lab|R' => \&record_action,
               'check|c' => \&record_action,
               'check-part|C=s' => \&record_check_part,
               'tags|T=s' => \&record_check_tags,
               'tags-from-file=s' => \&record_check_tags_from_file,
               'ftp-master-rejects|F' => \$ftpmaster_tags,
               'dont-check-part|X=s' => \&record_dont_check_part,
               'unpack|u' => \&record_action,
               'remove|r' => \&record_action,

               # ------------------ general options
               'help|h' => \&syntax,
               'version|V' => \&banner,
               'print-version' => \&banner,

               'verbose|v' => \$opt{'verbose'},
               'debug|d+' => \$debug, # Count the -d flags
               'quiet|q' => \&record_quiet, # sets $opt{'verbose'} to -1

               # ------------------ behaviour options
               'info|i' => \$opt{'info'},
               'display-info|I' => \&display_infotags,
               'display-experimental|E' => \$opt{'display-experimental'},
               'pedantic' => \&display_pedantictags,
               'display-level|L=s' => \&record_display_level,
               'display-source=s' => \&record_display_source,
               'suppress-tags=s' => \&record_suppress_tags,
               'suppress-tags-from-file=s' => \&record_suppress_tags_from_file,
               'no-override|o' => \$opt{'no-override'},
               'show-overrides' => \$opt{'show-overrides'},
               'color=s' => \$opt{'color'},
               'unpack-info|U=s' => \&record_unpack_info,
               'checksums|md5sums|m' => \&deprecated,
               'allow-root' => \$allow_root,
               'fail-on-warnings' => \$opt{'fail-on-warnings'},
               'keep-lab' => \$keep_lab,

               # ------------------ configuration options
               'cfg=s' => \$opt{'LINTIAN_CFG'},
               'no-cfg' => \$no_conf,
               'lab=s' => \$opt{'LINTIAN_LAB'},
               'archivedir=s' => \&deprecated,
               'dist=s' => \&deprecated,
               'area=s' => \&deprecated,
               'section=s' => \&deprecated,
               'arch=s' => \&deprecated,
               'profile=s' => \$opt{'LINTIAN_PROFILE'},
               'root=s' => \$opt{'LINTIAN_ROOT'},

               'jobs|j=i' => \$opt{'jobs'},

               # ------------------ package selection options
               'all|a' => \$check_everything,
               'binary|b' => \&record_pkgmode,
               'source|s' => \&record_pkgmode,
               'udeb' => \&record_pkgmode,
               'packages-file|p=s' => \$packages_file,
               'packages-from-file=s' => \$opt{'packages-from-file'},

               # ------------------ experimental
               'exp-output:s' => \$experimental_output_opts,
              );

# Options that can appear in the config file
my %cfghash = (
               'color'                => \$opt{'color'},
               'display-experimental' => \$opt{'display-experimental'},
               'display-info'         => \&cfg_display_level,
               'display-level'        => \&cfg_display_level,
               'fail-on-warnings'     => \$opt{'fail-on-warnings'},
               'info'                 => \$opt{'info'},
               'jobs'                 => \$opt{'jobs'},
               'pedantic'             => \&cfg_display_level,
               'quiet'                => \&cfg_verbosity,
               'override'             => \&cfg_override,
               'show-overrides'       => \$opt{'show-overrides'},
               'verbose'              => \&cfg_verbosity,
    );

# init commandline parser
Getopt::Long::config('bundling', 'no_getopt_compat', 'no_auto_abbrev');

# process commandline options
GetOptions(%opthash)
    or die("error parsing options\n");

# root permissions?
# check if effective UID is 0
if ($> == 0 and not $allow_root) {
    print STDERR "warning: the authors of lintian do not recommend running it with root privileges!\n";
}

# }}}

# {{{ Read important environment

# determine LINTIAN_ROOT if it was not set with --root.
$opt{'LINTIAN_ROOT'} = $ENV{'LINTIAN_ROOT'} unless (defined($opt{'LINTIAN_ROOT'}));
if (defined $opt{'LINTIAN_ROOT'}) {
    unless ($opt{'LINTIAN_ROOT'} =~ m,^/,) {
        require Cwd;
        my $cwd = Cwd::getcwd();
        $opt{'LINTIAN_ROOT'} = "$cwd/$opt{'LINTIAN_ROOT'}";
    }
} else {
    $opt{'LINTIAN_ROOT'} = '/usr/share/lintian';
}

# environment variables overwrite settings in conf file, so load them now
# assuming they were not set by cmd-line options
foreach my $var (@ENV_VARS) {
    # note $opt{$var} will usually always exists due to the call to GetOptions
    # so we have to use "defined" here
    $opt{$var} = $ENV{$var} if $ENV{$var} && ! defined $opt{$var};
}

# }}}

# {{{ Sanity check command-line options (that are unaffected by the configuration file)

# option --all and packages specified at the same time?
if (($check_everything or $packages_file or $opt{'packages-from-file'}) and $#ARGV+1 > 0) {
    print STDERR "warning: options -a, -p and --packages-from-file cannot be mixed with package parameters!\n";
    print STDERR "(will ignore -a, -p or/and --packages-from-file option)\n";
    undef $check_everything;
    undef $packages_file;
    delete $opt{'packages-from-file'};
}

if ($packages_file && $opt{'packages-from-file'}) {
    die "The options -p and --packages-from-file cannot be used together.\n"
}

# check specified action
$action = 'check' unless $action;

# check for arguments
if ($action =~ /^(?:check|unpack|remove)$/ and $#ARGV == -1 and not $check_everything and not $packages_file and not $opt{'packages-from-file'}) {
    my $ok = 0;
    if ($action eq 'check' or $action eq 'unpack') {
        # If debian/changelog exists, assume an implied "../<source>_<version>_<arch>.changes"
        # (or "../<source>_<version>_source.changes").
        if ( -f 'debian/changelog' ) {
            my $file = _find_changes();
            push @ARGV, $file;
            $ok = 1;
        }
    }
    syntax() unless $ok;
}

die "Cannot use profile together with --ftp-master-rejects.\n" if $opt{'LINTIAN_PROFILE'} and $ftpmaster_tags;
# --ftp-master-rejects is implemented in a profile
$opt{'LINTIAN_PROFILE'} = 'debian/ftp-master-auto-reject' if $ftpmaster_tags;

# }}}

# {{{ Loading lintian's own libraries, parse config file and setup output
unshift @INC, "$opt{'LINTIAN_ROOT'}/lib";

require Lintian::Lab;

require Lintian::Util;
import Lintian::Util qw(fail read_dpkg_control);

require Lintian::Collect;
require Lintian::CollScript;
require Lintian::DepMap::Properties;
require Lintian::Data;
require Lintian::Output;
import Lintian::Output qw(:messages);
require Lintian::Command::Simple;
require Lintian::Command;
import Lintian::Command qw(spawn reap);
require Lintian::Internal::FrontendUtil;
import Lintian::Internal::FrontendUtil;
require Lintian::ProcessablePool;
require Lintian::Profile;
require Lintian::Tags;
import Lintian::Tags qw(tag);

if (defined $experimental_output_opts) {
    my %opts = map { split(/=/) } split( /,/, $experimental_output_opts );
    foreach (keys %opts) {
        if ($_ eq 'format') {
            if ($opts{$_} eq 'colons') {
                require Lintian::Output::ColonSeparated;
                $Lintian::Output::GLOBAL = Lintian::Output::ColonSeparated->new;
            } elsif ($opts{$_} eq 'letterqualifier') {
                require Lintian::Output::LetterQualifier;
                $Lintian::Output::GLOBAL = Lintian::Output::LetterQualifier->new;
            } elsif ($opts{$_} eq 'xml') {
                require Lintian::Output::XML;
                $Lintian::Output::GLOBAL = Lintian::Output::XML->new;
            } elsif ($opts{$_} eq 'fullewi') {
                require Lintian::Output::FullEWI;
                $Lintian::Output::GLOBAL = Lintian::Output::FullEWI->new;
            }
        }
        no strict 'refs';
        ${"Tags::$_"} = $opts{$_};
    }
}

# search for configuration file if it was not set with --cfg
# do not search the default locations if it was set.
unless ($no_conf) {
    if ($opt{'LINTIAN_CFG'}) {
    } elsif (exists $ENV{'LINTIAN_CFG'} &&
             -f ($opt{'LINTIAN_CFG'} = $ENV{'LINTIAN_CFG'})) {
    } elsif (-f ($opt{'LINTIAN_CFG'} = $opt{'LINTIAN_ROOT'} . '/lintianrc')) {
    } elsif (exists $ENV{'HOME'} &&
             -f ($opt{'LINTIAN_CFG'} = $ENV{'HOME'} . '/.lintianrc')) {
    } elsif (-f ($opt{'LINTIAN_CFG'} = '/etc/lintianrc')) {
    } else {
        $opt{'LINTIAN_CFG'} = '';
    }
} else {
    $opt{'LINTIAN_CFG'} = '';
}

# read configuration file
if ($opt{'LINTIAN_CFG'}) {
    open(CFG, '<', $opt{'LINTIAN_CFG'})
        or die("cannot open configuration file $opt{'LINTIAN_CFG'} for reading: $!");
    while (<CFG>) {
        chop;
        s/\#.*$//go;
        s/\"//go;
        next if m/^\s*$/o;

        # substitute some special variables
        s,\$HOME/,$ENV{'HOME'}/,go;
        s,\~/,$ENV{'HOME'}/,go;

        my $found = 0;
        foreach my $var (@ENV_VARS) {
            if (m/^\s*$var\s*=\s*(.*\S)\s*$/i) {
                if (exists $conf_opt{$var}){
                    print STDERR "Configuration variable $var appears more than once\n";
                    print STDERR " in $opt{'LINTIAN_CFG'} (line: $.) - Using the first value!\n";
                    next;
                }
                $opt{$var} = $1 unless defined $opt{$var};
                $conf_opt{$var} = 1;
                $found = 1;
                last;
            }
        }
        if (m/^\s*LINTIAN_(ARCH|ARCHIVEDIR|AREA|DIST)\s*=/o) {
            print STDERR "LINTIAN_$1 is deprecated (seen in $opt{'LINTIAN_CFG'} at line $.)\n";
            $found = 1;
        }
        unless ($found) {
            # check if it is a config option
            if (m/^\s*([-a-z]+)\s*=\s*(.*\S)\s*$/o){
                my ($var, $val) = ($1, $2);
                my $ref = $cfghash{$var};
                die "Unknown configuration variable $var at line: ${.}.\n"
                    unless $ref;
                if (exists $conf_opt{$var}){
                    print STDERR "Configuration variable $var appears more than once\n";
                    print STDERR " in $opt{'LINTIAN_CFG'} (line: $.) - Using the first value!\n";
                    next;
                }
                $conf_opt{$var} = 1;
                $found = 1;
                if ($val =~ m/^y(?:es)?|true$/oi){
                    $val = 1;
                } elsif ($val =~ m/^no?|false$/oi){
                    $val = 0;
                }
                if (ref $ref eq 'SCALAR'){
                    # Check it was already set
                    next if defined $$ref;
                    $$ref = $val;
                } elsif (ref $ref eq 'CODE'){
                    $ref->($var, $val);
                }

            }
        }
        unless ($found) {
            die "syntax error in configuration file: $_\n";
        }
    }
    close(CFG);
}

# check permitted values for --color / color
#  - We set the default to 'never' here; because we cannot do
#    it before the config check.
$opt{'color'} = 'never' unless defined $opt{'color'};
if ($opt{'color'} and $opt{'color'} !~ /^(?:never|always|auto|html)$/) {
    die "The color value must be one of \"never\", \"always\", \"auto\" or \"html\"\n";
}

# export current settings for our helper scripts
foreach my $var (@MUST_EXPORT) {
    if ($opt{$var}) {
        $ENV{$var} = $opt{$var};
    } else {
        $ENV{$var} ='';
        $opt{$var} = ''; # Avoids some undef warnings later
    }
}

# We do this manually since the above would set $ENV{TMPDIR} to ''
# if it was undef and that causes tempdir to give us some "funny"
# (read: broken) paths.
$ENV{'TMPDIR'} = $opt{'TMPDIR'} if defined $opt{'TMPDIR'};

# If we are running the test suite we should ignore
# user/system profiles.
if ($ENV{'LINTIAN_INTERNAL_TESTSUITE'}){
    @prof_inc = ();
}

if ($debug) {
    $opt{'verbose'} = 1;
    $ENV{'LINTIAN_DEBUG'} = $debug;
    if ($debug > 1) {
        eval {
            require Time::HiRes;
            import Time::HiRes qw(gettimeofday tv_interval);

            $start_timer = sub {
                return [gettimeofday()];
            };
            $finish_timer =  sub {
                my ($start) = @_;
                my $diff = tv_interval ($start);
                return sprintf (' (%.3fs)', $diff);
            };
            print "N: Using Time::HiRes to debug running times\n";
        };
        if ($@) {
            print "N: Cannot load Time::HiRes ($@)\n";
            print "N: Running times will not be timed.\n";
        }
    }
} else {
    # Ensure verbose has a defined value
    $opt{'verbose'} = 0 unless defined $opt{'verbose'};
}

# Use our custom-generated locale for programs we call, if it's available.  We
# first look in the Lintian root and then in /var/lib/lintian, which is the
# standard location for the install-time-generated locale.
if (-d "$opt{'LINTIAN_ROOT'}/locale/en_US.UTF-8") {
    $ENV{LOCPATH} = "$opt{'LINTIAN_ROOT'}/locale";
} elsif (-d '/var/lib/lintian/locale/en_US.UTF-8') {
    $ENV{LOCPATH} = '/var/lib/lintian/locale';
}

$Lintian::Output::GLOBAL->verbosity_level ($opt{'verbose'});
$Lintian::Output::GLOBAL->debug ($debug);
$Lintian::Output::GLOBAL->color ($opt{'color'});
$Lintian::Output::GLOBAL->showdescription ($opt{'info'});

# }}}



# {{{ Load profile, setup display setting etc.

# Print Debug banner, now that we're finished determining
# the values and have Lintian::Output available
debug_msg(1,
          $BANNER,
          "Lintian root directory: $opt{'LINTIAN_ROOT'}",
          "Configuration file: $opt{'LINTIAN_CFG'}",
          "Laboratory: $opt{'LINTIAN_LAB'}",
          delimiter(),
    );

my $PROFILE;
our $TAGS = Lintian::Tags->new;
$TAGS->show_experimental($opt{'display-experimental'});
$TAGS->show_overrides($opt{'show-overrides'});
$TAGS->sources(keys %display_source) if %display_source;

unless ($opt{'LINTIAN_PROFILE'}){
    # Time to ask dpkg-vendor for a vendor name
    $opt{'LINTIAN_PROFILE'} = find_default_profile(@prof_inc, "$opt{'LINTIAN_ROOT'}/profiles");
}
$PROFILE = Lintian::Profile->new ($opt{'LINTIAN_PROFILE'}, $opt{'LINTIAN_ROOT'},
                                  [@prof_inc, "$opt{'LINTIAN_ROOT'}/profiles"]);
v_msg('Using profile ' . $PROFILE->name . '.');

if ($dont_check || %suppress_tags || $checks || $check_tags) {
    _update_profile ($PROFILE, $dont_check, \%suppress_tags, $checks, $check_tags);
}

$TAGS->profile ($PROFILE);
Lintian::Data->set_vendor ($PROFILE);

# Initialize display level settings.
for my $level (@display_level) {
    eval { $TAGS->display(@$level) };
    if ($@) {
        my $error = $@;
        $error =~ s/ at .*//;
        die $error, "\n";
    }
}


# }}}

# {{{ Set up clean-up handlers.

$SIG{'INT'} = \&interrupted;
$SIG{'QUIT'} = \&interrupted;

# }}}

# {{{ Create/Maintain Lab and add any specified Debian Archives (*.debs)

$LAB = Lintian::Lab->new( $opt{'LINTIAN_LAB'} );

#######################################
# Process -S option
if ($action eq 'setup-lab') {
    if ($#ARGV+1 > 0) {
        warning('ignoring additional command line arguments');
    }

    $LAB->create
        or fail('There was an error while setting up the static lab.');

    exit 0;

#######################################
# Process -R option
} elsif ($action eq 'remove-lab') {
    if ($#ARGV+1 > 0) {
        warning('ignoring additional command line arguments');
    }

    $LAB->remove
        or fail('There was an error while removing the static lab.');

    exit 0;

#######################################
#  Check for non deb specific actions
} elsif (not (($action eq 'unpack') or ($action eq 'check')
              or ($action eq 'remove'))) {
    fail("bad action $action specified");
}

if (!$LAB->is_temp) {
    # sanity check:
    fail('lintian lab has not been set up correctly (perhaps you forgot to run lintian --setup-lab?)')
        unless $LAB->exists;
} else {
    $LAB->create ( {'keep-lab' => $keep_lab} );
}

$LAB->open;

#  Update the ENV var as well - unlike the original values,
#  $LAB->dir is always absolute
$ENV{'LINTIAN_LAB'} = $opt{'LINTIAN_LAB'} = $LAB->dir;

v_msg ("Setting up lab in $opt{'LINTIAN_LAB'} ...")
    if $LAB->is_temp;

# }}}

# {{{ Compile list of files to process

$pool = Lintian::ProcessablePool->new($LAB);
# process package/file arguments

while (my $arg = shift) {
    # file?
    if (-f $arg) {
        if ($arg =~ m/\.(?:u?deb|dsc|changes)$/o){
            eval {
                $pool->add_file($arg);
            };
            if ($@) {
                print STDERR "Skipping $arg: $@";
                $exit_code = 2;
            }
        } else {
            fail("bad package file name $arg (neither .deb, .udeb, .changes or .dsc file)");
        }
    } else {
        # parameter is a package name--so look it up
        handle_lab_query ($arg);
    }
}

if ($check_everything) {
    my $t = $pkg_mode;
    my $visitor = sub {
        my ($lpkg) = @_;
        $pool->add_proc ($lpkg);
    };
    $t = undef if $pkg_mode eq 'auto';
    $LAB->visit_packages($visitor, $t);
} elsif ($packages_file) {
    # process all packages listed in packages file?
    print STDERR "warning: --packages-file is deprecated and may be removed in a later release.\n";
    print STDERR "  - consider using --packages-from-file (one pkg per line)\n";
    open(my $pkgin, '<', $packages_file) or fail("Reading $packages_file: $!");
    while (my $line = <$pkgin>) {
        chomp($line);
        my ($t, undef, undef, $file) = split(/\s+/, $line, 4);
        unless (defined $file && length $t == 1) {
            print STDERR "Syntax error in packages-file at line $.\n";
            print STDERR " - perhaps you meant to use \"--packages-from-file $packages_file\"\n";
            exit 1;
        }
        $pool->add_file($file);
    }
    close($pkgin);
} elsif ($opt{'packages-from-file'}){
    my $fd;
    if ($opt{'packages-from-file'} eq '-') {
        $fd = \*STDIN;
    } else {
        open $fd, '<', $opt{'packages-from-file'} or die "opening $opt{'packages-from-file'}: $!";
    }
    while (my $file = <$fd>) {
        chomp $file;
        if ($file =~ m/^!query:\s*(\S(?:.*\S)?)/o) {
            my $query = $1;
            handle_lab_query ($query);
        } else {
            $pool->add_file ($file);
        }
    }
    # close unless it is STDIN (else we will see a lot of warnings
    # about STDIN being reopened as "output only")
    close $fd unless $opt{'packages-from-file'} eq '-';
}

# Remove the group cache in case there has been group lab queries.  We
# do not need this cache anymore.
_clear_group_cache();

# }}}

# {{{ Some silent exit
if ($pool->empty()) {
    v_msg('No packages selected.');
    exit $exit_code;
}
# }}}

# {{{ Handle $action eq 'remove'
# We have enough information to handle remove now.

if($action eq 'remove'){
    # Handle remove here - makes the unpack/check loop simpler.
    foreach my $group ($pool->get_groups()){
        foreach my $proc ($group->get_processables()){
            my $pkg_name = $proc->pkg_name();
            my $pkg_ver  = $proc->pkg_version();
            my $pkg_type = $proc->pkg_type();
            my $pkg_path = $proc->pkg_path();
            my $pkg_arch = $proc->pkg_arch();
            my $lpkg = _get_lpkg ($proc);

            if (!defined $lpkg){
                my $err = '.';
                $err = ": $@" if defined $@;
                warning("skipping $action of $pkg_type package ${pkg_name}$err");
                $exit_code = 2;
                next;
            }
            $TAGS->file_start($pkg_path, $pkg_name, $pkg_ver,
                              $pkg_arch, $pkg_type);
            debug_msg(1, 'Removing package in lab ...');
            unless ($lpkg->remove){
                warning("cannot remove entry for $pkg_name: $!");
                $exit_code = 2;
            }
        }
    }
    $TAGS->file_end();
    # Write the lab state to the disk, so it remembers they are gone.
    $LAB->close;
    exit $exit_code;
}
# }}}

# {{{ Load information about collector scripts
load_collections(\%collection_info, "$opt{'LINTIAN_ROOT'}/collection");
# }}}

# {{{ determine which checks have been requested
if ($action eq 'check') {

    # determine which info is needed by the checks
    for my $c (sort $PROFILE->scripts) {
        my $cs = $PROFILE->get_script($c);
        for my $i ($cs->needs_info) {
            $unpack_infos{$i} = 1;
            $requested_unpack{$i} = 1;
        }
    }
}

# }}}

# {{{ determine which info is needed by the collection scripts
if ($action eq 'unpack') {
    # With --unpack we want all of it
    for my $c (keys %collection_info) {
        $unpack_infos{$c} = 1;
        $requested_unpack{$c} = 1;
    }
} else {
    my @needed = keys %unpack_infos;
    my %added = ();
    unless ($opt{'no-override'}) {
        push @needed, 'override-file';
        $requested_unpack{'override-file'} = 1;
    }
    while ( my $c = pop @needed ) {
        next if $added{$c};
        $added{$c} = 1;
        $unpack_infos{$c} = 1;
        push @needed, $collection_info{$c}->needs_info;
    }
}

if (@unpack_info) {
    # Add collections specifically requested by the user (--unpack-info)
    for my $i (map { split m/,/ } @unpack_info) {
        unless ($collection_info{$i}) {
            fail("unknown info specified: $i");
        }
        $unpack_infos{$i} = 1;
        $keep_coll{$i} = 1;
    }
}


# }}}

# {{{ Create the dependency tree and populate it with checks and collections

# All required checks and collections have been calculated at this point.
# We are just adding this information to a map now that will generate the
# execution order.
#
# $map is just here to check that all the needed collections are present.
my $map = Lintian::DepMap::Properties->new();
my @scripts = sort $PROFILE->scripts;
my $collmap = Lintian::DepMap::Properties->new();

for my $c (keys %unpack_infos) {
    # Add the collections with their dependency information
    $map->add('coll-' . $c, {'type' => 'collection', 'name' => $c});
    $collmap->add($c, {'type' => 'collection', 'name' => $c});
    $map->addp('coll-' . $c, 'coll-', $collection_info{$c}->needs_info);
    $collmap->add($c, $collection_info{$c}->needs_info);
}

for my $c (@scripts) {
    # Add the checks with their dependency information
    my $cs = $PROFILE->get_script ($c);
    $map->add('check-' . $c, $cs);
    if ($cs->needs_info) {
        $map->addp('check-' . $c, 'coll-', $cs->needs_info);
    }
}
# }}}

# {{{ Okay, now really processing the packages in one huge loop
$opt{'jobs'} = default_parallel() unless defined $opt{'jobs'};

debug_msg(1,
          "Selected action: $action",
          sprintf('Requested data to collect: %s', join(',',sort keys %unpack_infos)),
          sprintf('Selected checks: %s', join(',',sort $PROFILE->scripts)),
          "Parallization limit: $opt{'jobs'}",
    );


# Make sure the resolver is in a sane state:
scalar($map->missing()) == 0
    or fail('There are missing nodes on the resolver: '.join(', ', $map->missing()));
undef $map;


# Now action is always either "check" or "unpack"
# these two variables are used by process_package
#  and need to persist between invocations.
my %running_jobs;
my %overrides;

foreach my $gname (sort $pool->get_group_names()) {
    my $group = $pool->get_group($gname);
    unpack_group($group);
    if ($action eq 'check'){
        process_group($group);
        clear_group_cache($group);
    }
}

# Write the lab state to the disk, so it remembers the new packages
$LAB->close;
$TAGS->file_end();

if ($action eq 'check' and not $opt{'no-override'} and not $opt{'show-overrides'}) {
    my $errors = $overrides{errors} || 0;
    my $warnings = $overrides{warnings} || 0;
    my $info = $overrides{info} || 0;
    my $total = $errors + $warnings + $info;
    if ($total > 0) {
        my $text = ($total == 1)
            ? "$total tag overridden"
            : "$total tags overridden";
        my @output;
        if ($errors) {
            push (@output, ($errors == 1) ? "$errors error" : "$errors errors");
        }
        if ($warnings) {
            push (@output, ($warnings == 1) ? "$warnings warning" : "$warnings warnings");
        }
        if ($info) {
            push (@output, "$info info");
        }
        msg("$text (". join (', ', @output). ')');
    }
}

my $ign_over = $TAGS->ignored_overrides;
if (keys %$ign_over) {
    msg('Some overrides were ignored, since the tags were marked "non-overridable".');
    if ($opt{'verbose'}) {
        v_msg('The following tags were "non-overridable" and had at least one override');
        foreach my $tag (sort keys %$ign_over) {
            v_msg("  - $tag");
        }
    } else {
        msg('Use --verbose for more information.');
    }
}

# }}}


# Wait for any remaining jobs - %running_jobs will usually be empty here
# unless we had an issue examining the last package.  We patiently wait
# for them here; if the user cannot be bothered to wait, he/she can send
# us a signal and the END handler will kill any remaining jobs.
while (my ($coll, undef) = Lintian::Command::Simple::wait(\%running_jobs)) {
    delete $running_jobs{$coll};
}
%running_jobs = ();

exit $exit_code;

# {{{ Some subroutines

#  Check to make sure there are packages to check.
sub set_value {
    my ($f,$target,$field,$source,$required) = @_;
    if ($required and not defined($source->{$field})) {
        fail("description file $f does not define required tag $field");
    }
    $target->{$field} = $source->{$field};
    delete $source->{$field};
}


# Given a ref to %collection_info and the path to the collection
# directory, this will load all the collection information into
# %collection_info.
sub load_collections{
    my ($cinfo, $dirname) = @_;
    opendir(my $dir, $dirname)
        or fail("cannot read directory $dirname");

    for my $f (readdir($dir)) {
        next if $f =~ /^\./;
        next unless $f =~ /\.desc$/;

        debug_msg(2, "Reading collector description file $f ...");
        my $cs = Lintian::CollScript->new ("$dirname/$f");
        $cinfo->{$cs->name} = $cs;
    }

    closedir($dir);
}

# Removes all collections with "Auto-Remove: yes"; takes a Lab::Package
#  - depends on global variables %collection_info and %keep_coll
#
sub auto_clean_package {
    my ($lpkg) = @_;
    my $pkg_name = $lpkg->pkg_name();
    my $pkg_type = $lpkg->pkg_type();
    my $base = $lpkg->base_dir();
    my $changed = 0;
    for my $coll (keys %collection_info) {
        my $ci = $collection_info{$coll};
        if ($ci->auto_remove and not $keep_coll{$coll}) {
            next unless $lpkg->is_coll_finished ($coll, $ci->version);
            debug_msg(1, "Auto removing: $coll ...");
            $changed = 1;
            unless (system ($ci->script_path, $pkg_name, "remove-${pkg_type}", $base) == 0) {
                warning("removing collect info $coll about package $pkg_name failed",
                        "skipping cleanup of $pkg_type package $pkg_name");
                return -1;
            }
            $lpkg->_clear_coll_status ($coll);
        }
    }
    return $changed;
}

sub post_pkg_process_overrides{
    my ($pkg_path) = @_;
    # report unused overrides
    if (not $opt{'no-override'}) {
        my $overrides = $TAGS->overrides($pkg_path);

        for my $tag (sort keys %$overrides) {
            next if $TAGS->suppressed($tag);

            my $taginfo = $PROFILE->get_tag ($tag);

            for my $extra (sort keys %{$overrides->{$tag}}) {
                next if $overrides->{$tag}{$extra};
                tag( 'unused-override', $tag, $extra );
            }
        }
    }

    # Report override statistics.
    if (not $opt{'no-override'} and not $opt{'show-overrides'}) {
        my $stats = $TAGS->statistics($pkg_path);
        my $errors = $stats->{overrides}{types}{E} || 0;
        my $warnings = $stats->{overrides}{types}{W} || 0;
        my $info = $stats->{overrides}{types}{I} || 0;
        $overrides{errors} += $errors;
        $overrides{warnings} += $warnings;
        $overrides{info} += $info;
    }
}

sub _get_lpkg {
    my ($proc) = @_;
    my $lpkg;
    # If $proc is an lab entry (i.e. we got it from the Lab already)
    # just use it directly.
    if ($proc->isa ('Lintian::Lab::Entry')) {
        $lpkg = $proc;
    } else {
        # Else use the $proc to get the lab entry
        eval{
            $lpkg = $LAB->get_package($proc);
        };
    }
    return $lpkg;
}

sub unpack_group {
    my ($group) = @_;
    my %worklists;
    my %job_data = ();
    my $groupid;
    my $jobs = $opt{'jobs'};

    foreach my $proc ($group->get_processables()){
        my $pkg_name = $proc->pkg_name();
        my $pkg_type = $proc->pkg_type();
        my $pkg_ver  = $proc->pkg_version();
        my $pkg_arch = $proc->pkg_arch();
        my $lpkg = _get_lpkg ($proc);
        my $base;
        my $info;
        my $changed = 0;
        my $cmap = $collmap->clone;

        if (!defined $lpkg) {
            my $err = '.';
            $err = ": $@" if defined $@;
            warning("skipping $action of $pkg_type package ${pkg_name}$err");
            $exit_code = 2;
            $group->remove_processable($proc);
            next;
        }
        # determine base directory
        $base = $lpkg->base_dir();
        debug_msg(1, "Unpacking $pkg_name $pkg_ver [$pkg_arch] ($pkg_type) in $base");

        $cmap->initialise();

        if ($lpkg->exists) {
            # It already exists - only collect what we need.
            # - $collmap has everything we need, but in some cases more than that.
            my %need = ();
            my @check = keys %requested_unpack;
            while (my $cname = pop @check) {
                my $coll = $collection_info{$cname};
                # Skip collections not relevant to us (they will never
                # be finished and we do not want to use their
                # dependencies if they are the only ones using them)
                next unless $coll->is_type ($pkg_type);
                next if $lpkg->is_coll_finished ($cname, $coll->version);
                $need{$cname} = 1;
                push @check, $coll->needs_info;
            }
            while (1) {
                my @s = grep { not $need{$_} } $cmap->selectable;
                last if not @s;
                $cmap->satisfy (@s);
            }
        } elsif (not $lpkg->create){
            # Cannot create the entry? Skip
            warning("could not create the package entry in the lab: $!",
                    "skipping $action of $pkg_type package $pkg_name");
            $exit_code = 2;
            $group->remove_processable($proc);
            next;
        } else {
            # created
            $changed = 1;
        }

        $groupid = $proc->pkg_src . '/' . $proc->pkg_src_version
            unless defined $groupid;
        # We only need this if we are checking the package later
        $proc->lab_pkg($lpkg) unless $proc->isa ('Lintian::Lab::Entry');
        $worklists{$lpkg->identifier} = {
            'collmap' => $cmap,
            'lab-entry' => $lpkg,
            'changed' => $changed
        };
    }

    # Stop here if there is nothing list for us to do
    return unless %worklists;

    # Kill pending jobs, if any
    Lintian::Command::Simple::kill(\%running_jobs);
    %running_jobs = ();


    while (1) {
        my $newjobs = 0;
        my $nohang = 0;
      PROC:
        foreach my $proc ($group->get_processables){
            my $procid = $proc->identifier;
            my $wlist = $worklists{$procid};
            my $cmap = $wlist->{'collmap'};
            my $lpkg = $wlist->{'lab-entry'};
            my $pkg_name = $lpkg->pkg_name;
            my $pkg_type = $lpkg->pkg_type;
            my $pkg_ver  = $lpkg->pkg_version;
            my $pkg_arch = $lpkg->pkg_arch;
            my $base = $lpkg->base_dir;
            foreach my $req ($cmap->selectable) {
                my $ri = $cmap->getProp($req);
                my $coll = $ri->{'name'};
                my $ci = $collection_info{$coll};

                # current type?
                unless ($ci->is_type ($pkg_type)) {
                    $cmap->satisfy ($req);
                    next;
                }

                # check if it has been run previously
                if ($lpkg->is_coll_finished ($coll, $ci->version)) {
                    $cmap->satisfy ($req);
                    next;
                }
                # Not run before (or out of date)
                $lpkg->_clear_coll_status($coll);

                # collect info
                $cmap->select ($req);
                $wlist->{'changed'} = 1;
                debug_msg(1, "Collecting info: $coll for $procid ...");
                my $cmd = Lintian::Command::Simple->new();
                unless ($cmd->background ($ci->script_path, $pkg_name, $pkg_type, $base) > 0) {
                    warning("collect info $coll about package $pkg_name failed",
                            "skipping $action of $pkg_type package $pkg_name");
                    $exit_code = 2;
                    $group->remove_processable ($lpkg);
                    delete $worklists{$proc->identifier};
                    # Lets not start any more jobs for this processable
                    last;
                }
                $running_jobs{$cmd->pid} = $cmd;
                $job_data{$cmd->pid} = [$ci, $cmap, $lpkg, $start_timer->()];
                if ($jobs) {
                    # Have we hit the limit of running jobs?
                    last PROC if scalar keys %running_jobs >= $jobs;
                }
            }
        }
        # wait until a job finishes to run its branches, if any, or skip
        # this package if any of the jobs failed.
        debug_msg(1, "Reaping done jobs ... unpack group $groupid");

        while (my ($key, $cmd) = Lintian::Command::Simple::wait(\%running_jobs, $nohang)) {
            my $jdata = $job_data{$key};
            my (undef, undef, $lpkg) = @$jdata;
            my $res;
            delete $running_jobs{$key};
            delete $job_data{$key};

            # Check if the processable has been dropped due other errors.
            next unless exists $worklists{$lpkg->identifier};

            $res = finished_coll ($key, $cmd, $jdata);
            if ($res < 0) {
                $group->remove_processable ($lpkg);
                delete $worklists{$lpkg->identifier};
                next;
            }
            $newjobs += $res;
            $nohang = 1 if $res;
        }
        debug_msg(1, "Reap done jobs ... unpack group $groupid");

        # Stop when there are no running jobs and no new pending ones.
        unless (%running_jobs or $newjobs) {
            # No more running jobs and no new jobs have become available...
            # It is not quite sufficient, so ensure that all jobs have in
            # fact been run.
            my $done = 1;
            foreach my $wlist (values %worklists) {
                my $cmap = $wlist->{'collmap'};
                if ($cmap->pending) {
                    $done = 0;
                    last;
                }
            }
            last if $done;
        }
    }

    foreach my $proc ($group->get_processables) {
        my $lpkg = _get_lpkg ($proc);
        my $wlist = $worklists{$proc->identifier};
        my $changed = 1;
        $changed = $wlist->{'changed'} if $wlist;
        if ($action ne 'check') {
            # we are done now - start auto-cleaning
            if (!$keep_lab) {
                my $ret = auto_clean_package($lpkg);
                $exit_code = 2 if $ret < 0;
                $changed = 1 if $ret;
            }
        }
        if ($changed) {
            # All successful, make sure to record it so we do not unpack the same package
            # in a later run (mostly for archive-wide checks).
            unless ($lpkg->update_status_file) {
                my $pkg_name = $lpkg->pkg_name;
                warning("could not create status file for package $pkg_name: $!");
            }
        }
    }
    return 1;
}

sub process_group {
    my ($group) = @_;
  PROC:
    foreach my $proc ($group->get_processables()){
        my $pkg_name = $proc->pkg_name();
        my $pkg_ver  = $proc->pkg_version();
        my $pkg_type = $proc->pkg_type();
        my $pkg_path = $proc->pkg_path();
        my $pkg_arch = $proc->pkg_arch();
        my $lpkg = $proc->lab_pkg();
        my $info = $proc->info();
        my $base = $lpkg->base_dir();

        $TAGS->file_start($pkg_path, $pkg_name, $pkg_ver, $pkg_arch, $pkg_type);

        debug_msg(1, "Base directory in lab: $base");

        # chdir to base directory
        unless (chdir($base)) {
            warning("could not chdir into directory $base: $!",
                    "skipping $action of $pkg_type package $pkg_name");
            $exit_code = 2;
            next;
        }

        unless ($opt{'no-override'}) {
            if ($collmap->known('override-file') && -f "$base/override") {
                debug_msg(1, 'Override file collected, loading it ...');
                $TAGS->file_overrides("$base/override");
            }
        }
        foreach my $script (@scripts) {
            my $cs = $PROFILE->get_script ($script);
            my $check = $cs->name;
            my $timer = $start_timer->();

            # The lintian check is done by this frontend and we
            # also skip the check if it is not for this type of
            # package.
            next if (!$cs->is_check_type ($pkg_type) || $check eq 'lintian');

            debug_msg(1, "Running check: $check ...");
            my $returnvalue = _run_check ($cs, $pkg_name, $pkg_type, $info, $proc, $group);
            # Set exit_code correctly if there was not yet an exit code
            $exit_code = $returnvalue unless $exit_code;
            my $tres = $finish_timer->($timer);
            debug_msg(1, "Finished check: $check$tres");

            if ($returnvalue == 2) {
                warning("skipping $action of $pkg_type package $pkg_name");
                $exit_code = 2;
                next PROC;
            }
        }
        # chdir to lintian root directory (to unlock $base so it can be removed below)
        unless (chdir($opt{'LINTIAN_ROOT'})) {
            warning("could not chdir into directory $opt{'LINTIAN_ROOT'}: $!",
                    "skipping $action of $pkg_type package $pkg_name");
            $exit_code = 2;
            next;
        }

        unless ($exit_code) {
            my $stats = $TAGS->statistics($pkg_path);
            if ($stats->{types}{E}) {
                $exit_code = 1;
            } elsif ($opt{'fail-on-warnings'} && $stats->{types}{W}) {
                $exit_code = 1;
            }
        }
        post_pkg_process_overrides($pkg_path);
    } # end foreach my $proc ($group->get_processable())

    if (!$keep_lab) {
        # Invoke auto-clean now that the group has been checked
        foreach my $proc ($group->get_processables()){
            my $lpkg = $proc->lab_pkg();
            my $ret = auto_clean_package($lpkg);
            $exit_code = 2 if $ret < 0;
            if ($ret) {
                # Update the status file as auto_clean_package may have removed some
                # collections
                unless ($lpkg->update_status_file) {
                    my $pkg_name = $proc->pkg_name;
                    warning("could not create status file for package $pkg_name: $!");
                }
            }
        }

    }

    return 1;
}


# cleans the cache of all elements in this group - this avoids
# memory being hogged by packages/groups that have been checked
# and will not be checked again.
sub clear_group_cache {
    my ($group) = @_;
    foreach my $proc ($group->get_processables()){
        $proc->clear_cache;
    }
    return 1;
}

sub handle_lab_query {
    my ($query) = @_;
    my @res;
    my $type = $pkg_mode;
    my ($pkg, $version, $arch);
    my $orig = $query; # Save for the error message later


    # "britney"-like format - note this catches the old style, where only the
    # package name was specified.
    # Check if it starts with a type specifier (i.e. binary:eclipse/3.5.2-1/amd64)
    if ($query =~ m,^([^:]+):(.*),) {
        ($type, $query) = ($1, $2);
    }
    # Split on /
    ($pkg, $version, $arch) = split m,/,o, $query, 3;
    if ($pkg =~ m|^\.{0,2}$| or $pkg =~ m,_, or
          (defined $arch and $arch =~ m,/,) ) {
        # Technically, a string like "../somewhere/else",
        # "somepkg_version_arch.deb", "/somewhere/somepkg.deb" or even
        # "http://ftp.debian.org/pool/l/lintian/lintian_2.5.5_all.deb"
        # could match the above.  Obviously, that is not a lab query.
        # But the frontend sends it here, because the file denoted by
        # that string does not exist.
        warning ("\"$orig\" cannot be processed.");
        warning ("It is not a valid lab query and it is not an existing file.");
        exit 2;
    }

    # if version (or/and arch) is omitted or is the special
    # value "_", let it be wildcard.
    $version = undef if !$version or $version eq '_';
    $arch = undef if !$arch or $arch eq '_';
    debug_msg (2, "$orig => $type, $pkg, " . ($version//'*') . ', ' . ($arch//'*'));

    if ($type eq 'auto' or $type eq 'ALL') {
        # Check for all types
        foreach my $t (qw(binary source udeb changes)) {
            my @pkgs = $LAB->get_package ($pkg, $t, $version, $arch);
            push @res, @pkgs;
        }
    } elsif ($type eq 'GROUP') {
        _build_group_cache() unless %group_cache;
        if (exists $group_cache{$pkg}) {
            if (defined $version) {
                push @res, @{ $group_cache{$pkg}->{$version} };
            } else {
                foreach my $v (keys %{ $group_cache{$pkg} }) {
                    push @res, @{ $group_cache{$pkg}->{$v} };
                }
            }
        }
    } else {
        # specific type requested
        my @pkgs;
        eval {
            @pkgs = $LAB->get_package ($pkg, $type, $version, $arch);
            push @res, @pkgs;
        };
    }

    if (@res) {
        foreach my $p (@res) {
            $pool->add_proc ($p);
        }
    } else {
        my $tuple = join (', ', map { $_//'*'} ($type, $pkg, $version, $arch));
        debug_msg (1, "Did not find a match for $orig (pkg_mode = $pkg_mode)",
                   " - Search tuple: ($tuple)");
        warning ("cannot find binary, udeb or source package $orig in lab (skipping)");
        $exit_code = 2;
    }
}

sub _build_group_cache {
    # Globals %group_cache and $LAB
    $LAB->visit_packages (sub {
        my ($entry) = @_;
        my $src = $entry->pkg_src;
        my $src_version = $entry->pkg_src_version;
        push @{ $group_cache{$src}->{$src_version} }, $entry;
    });
}

sub _clear_group_cache {
    undef %group_cache;
}

sub _find_changes {
    require Parse::DebianChangelog;
    my $dch = Parse::DebianChangelog->init ( { infile => 'debian/changelog', quiet => 1 } );
    my $data = $dch->data;
    my $last = $data ? $data->[0] : undef;
    my ($source, $version, $arch);
    my $changes;
    my @archs;
    my @dirs;
    if (not $last) {
        my @errors = $dch->get_parse_errors;
        if (@errors) {
            print STDERR "Cannot parse debian/changelog due to errors:\n";
            for my $error (@errors) {
                print STDERR "$error->[2] (line $error->[1])\n";
            }
        } else {
            print STDERR "debian/changelog does not have any data?\n";
        }
        exit 2;
    }
    $version = $last->Version;
    $source = $last->Source;
    if (not defined $version or not defined $source) {
        $version//='<N/A>';
        $source//='<N/A>';
        print STDERR "Cannot determine source and version from debian/changelog:\n";
        print STDERR "Source: $source\n";
        print STDERR "Version: $source\n";
        exit 2;
    }
    # remove the epoch
    $version =~ s/^\d+://;
    if (exists $ENV{'DEB_BUILD_ARCH'}) {
        my $barch = $ENV{'DEB_BUILD_ARCH'};
        push @archs, $barch;
        $changes = "../${source}_${version}_${barch}.changes";
        return $changes if -f $changes;
    }
    if (exists $ENV{'DEB_HOST_ARCH'}) {
        $arch = $ENV{'DEB_HOST_ARCH'};
    } else {
        chomp ( $arch = `dpkg --print-architecture` );
    }
    if ($arch ne ($ENV{'DEB_BUILD_ARCH'}//'')) {
        push @archs, $arch;
        $changes = "../${source}_${version}_${arch}.changes";
        return $changes if -f $changes;
    }
    if (system ('dpkg', '--assert-multi-arch') == 0) {
        # Maybe cross-built for something dpkg knows about...
        open my $foreign, '-|', 'dpkg', '--print-foreign-architectures'
            or die "dpkg --print-foreign-architectures: $!\n";
        while ( my $line = <$foreign> ) {
            chomp $line;
            # Skip already attempted architectures (e.g. via DEB_BUILD_ARCH)
            next if grep { $_ eq $line } @archs;
            $changes = "../${source}_${version}_${line}.changes";
            if ( -f $changes ) {
                # Consume the entire input, even if we don't need it.
                1 for <$foreign>;
                close $foreign;
                return $changes;
            }
            push @archs, $line;
        }
        close $foreign;
    }
    foreach my $a (qw(multi all source)) {
        $changes = "../${source}_${version}_${a}.changes";
        return $changes if -f $changes;
        push @archs, $a;
    }
    foreach my $dir ('../build-area', '/var/cache/pbuilder/result') {
        next unless -d $dir;
        foreach my $a (@archs) {
            $changes = "$dir/${source}_${version}_${a}.changes";
            return $changes if -f $changes;
        }
        push @dirs, $dir;
    }
    print STDERR "Cannot find changes file for ${source}/${version}, tried:\n";
    foreach my $a (@archs) {
        print STDERR "  ../${source}_${version}_${a}.changes\n";
    }
    if (@dirs) {
        print STDERR " Also tried the following dirs:\n";
        print STDERR '  ' , join ("\n  ", @dirs), "\n";
    }
    exit 0;
}

sub _guess_version {
    require File::Basename;
    require Cwd;
    my ($frontend) = @_;
    my $guess;
    my $absfront = Cwd::abs_path ($frontend);
    my $rootdir;
    return '' unless $absfront;
    $rootdir = File::Basename::dirname (File::Basename::dirname ($absfront));

    if ( -d "$rootdir/.git" ) {
        # Lets try git
        eval {
            require IPC::Run;
            IPC::Run::run (['git', "--git-dir=$rootdir/.git", 'describe'], \undef, \$guess);
            chomp $guess;
        };
        return $guess if $guess;
    }
    # git was not possible - maybe the changelog is available
    if ( -f "$rootdir/debian/changelog" ) {
        eval {
            my $changelog = Parse::DebianChangelog->init({ infile => "$rootdir/debian/changelog" });
            $guess = $changelog->dpkg()->{'Version'} if $changelog;
        };
        return $guess if $guess;
    }
    # Out of guesses ...
    return;
}

sub _update_profile {
    my ($profile, $sup_check, $sup_tags, $only_check, $only_tags) = @_;
    my %abbrev = ();

    if ($sup_check || $only_check) {
        # Build an abbreviation map
        for my $c ($profile->scripts (1)) {
            my $cs = $profile->get_script ($c, 1);
            next unless $cs->abbrev;
            $abbrev{$cs->abbrev} = $cs;
        }
    }

    # if tags are listed explicitly (--tags) then show them even if
    # they are pedantic/experimental etc.  However, for --check-part
    # people explicitly have to pass the relevant options.
    if ($checks || $check_tags) {
        $profile->disable_tags ($profile->tags);
        if ($check_tags) {
            $TAGS->show_experimental(1);
            # discard whatever is in @display_level and request
            # everything
            @display_level = ();
            display_infotags();
            display_pedantictags();
            $profile->enable_tags (split /,/, $check_tags);
        } else {
            for my $c (split /,/, $checks) {
                my $cs = $profile->get_script ($c, 1) || $abbrev{$c};
                fail ("Unknown check script $c") unless $cs;
                $profile->enable_tags ($cs->tags);
            }
        }
    } elsif ($sup_check) {
        # we are disabling checks
        for my $c (split(/,/, $sup_check)) {
            my $cs = $profile->get_script ($c, 1) || $abbrev{$c};
            fail ("Unknown check script $c") unless $cs;
            $profile->disable_tags ($cs->tags);
        }
    } elsif (%$sup_tags) {
        # we are disabling tags
        $profile->disable_tags (keys %$sup_tags);
    }
}

sub _run_check {
    my ($cs, $pkg_name, @args) = @_;
    my $check = $cs->name;
    my $ret = 0;
    my $cs_pkg = $cs->script_pkg;

    require "$opt{'LINTIAN_ROOT'}/checks/$check";

    {
        # minimal "no strict refs" scope.
        no strict 'refs';
        eval { &{'Lintian::' . $cs_pkg . '::run'}($pkg_name, @args); };
    }

    if ( $@ ) {
        print STDERR $@;
        print STDERR "internal error: cannot run $check check on package $pkg_name\n";
        $ret = 2;
    }
    return $ret;
}

sub finished_coll {
    my ($jobid, $job, $job_data) = @_;
    my ($ci, $cmap, $lpkg, $timer) = @$job_data;
    my $coll = $ci->name;
    if ($job->status() == 0) {
        my $tres = $finish_timer->($timer);
        my $procid = $lpkg->identifier;

        $lpkg->_mark_coll_finished($coll, $ci->version)
            or fail ("cannot mark $coll for $procid as complete: $!");

        debug_msg (1, "Collection script $coll for $procid done$tres");
    } else {
        my $pkg_name  = $lpkg->pkg_name;
        my $pkg_type = $lpkg->pkg_type;
        warning ("collect info $coll about package $pkg_name failed");
        warning ("skipping $action of $pkg_type package $pkg_name");
        $exit_code = 2;
        return -1;
    }

    $cmap->satisfy ($coll);
    return $cmap->selectable;
}

sub default_parallel {
    # check cpuinfo for the number of cores...
    my $cpus;
    chomp ( $cpus = `nproc 2>&1` );
    if ($? == 0 and $cpus =~ m/^\d+$/ ) {
        # Running up to twice the number of cores usually gets the most out
        # of the CPUs and disks but it might be too aggresive to be the
        # default for -j. Only use <cores>+1 then.
        debug_msg (1, "Number of CPUs: $cpus");
        return $cpus + 1;
    }

    # No decent number of jobs? Just use 2 as a default
    return 2;
}

# }}}

# {{{ Exit handler.

sub END {
    # Prevent Lab->close from affecting the exit code.
    local $?;

    $SIG{'INT'} = 'DEFAULT';
    $SIG{'QUIT'} = 'DEFAULT';

    # Kill any remaining jobs.
    if(%running_jobs) {
        Lintian::Command::Simple::kill(\%running_jobs);
        %running_jobs = ();
    }

    $LAB->close if $LAB;
}

sub interrupted {
    $SIG{$_[0]} = 'DEFAULT';
    die "N: Interrupted.\n";
}

# }}}

# Local Variables:
# indent-tabs-mode: nil
# cperl-indent-level: 4
# End:
# vim: syntax=perl sw=4 sts=4 sr et
lintian 2.5.10.4 / usr / bin / lintian
