debdelta 0.59 / usr / share / debdelta / debmirror-marshal-deltas

#!/bin/bash

set -e

# A complex script that creates a repository of deltas, that can be
# used by debdelta-upgrade for upgrading packages.
# See also --help below.

# Copyright (C) 2006-11 Andrea Mennucci.
# License: GNU Library General Public License, version 2 or later

#since the output is automatically parsed, do it in English 
unset LANG
unset LC_ALL
unset LC_NUMERIC

startgpgagent () {
 gpg-agent --homedir "${GNUPGHOME}" --daemon --write-env-file "$GNUPGAGENTINFO" ; 
}


DEBUG=''
VERBOSE=''
RM='rm'
MV='mv'
DO_MIRROR=true
DO_CLEAN_DELTAS=''
DO_CLEAN_DEBS=''
CONF=''
STARTAGENT=''
while [ "$1" ] ; do
 case "$1" in
  -h|--help)
  cat <<EOF
Usage: debmirror-marshal-deltas [options] -c CONFFILE

This script keeps a full mirror of the Debian repository (using
'debmirror'), and it uses it to build a repository of deltas (using
'debdeltas').

Note that this script assumes that the current Debian
distributions are $DISTc .
It will generate deltas for the upgrades in those,
on arches $ARCHc , and sections $SECTIONc .

Options:

-v   verbose   (passed to debmirror and debdeltas)
-d   debug     (passed to debmirror and debdeltas),
               also, do not 'rm' stuff, just tell
-c CONF        configuration file
--no-mirror    do not download debs to the local mirror
--clean-deltas clean useless deltas (see debdeltas man page)
--clean-debs   clean older snapshots in deb mirror
--start-gpg-agent        start gpg agent, feed the
               passwd for the secret gpg key in it, then exit
EOF
  exit ;;
  -v)  VERBOSE="$VERBOSE -v"  ;;
  -d)  DEBUG=--debug ;  RM='echo -- would rm ' ;  MV='echo -- would mv ' ;;
  -c)  shift; CONF="$1" ;;
  --no-mirror) DO_MIRROR='' ;;
  --clean-deltas) DO_CLEAN_DELTAS=true ;;
  --clean-debs)   DO_CLEAN_DEBS=true ;;
  --snapshot) echo $0 does not use its own snapshots any more ; exit ;;
  --start-gpg-agent) STARTAGENT=true  ;;
  *) echo "$0: Unknown option $1 , try --help" ;  exit 1 ;;
 esac
 shift
done

################## read configuration

if test "$CONF" = ""  ; then
 echo "please specify a configuration file"
 exit 1
fi
if test ! -r "$CONF" ; then
 echo "cannot read configuration file: $CONF"
 exit 1
fi

. "$CONF"

#################

if test "$STARTAGENT" ; then
     echo ------  start agent
     startgpgagent
     . "$GNUPGAGENTINFO" 
     export GPG_AGENT_INFO
     #force loading of the private key (I know no better way)
     echo ------ sign dummy file, to load the key
     t=`tempfile`
     echo pippo > $t
     gpg2 --quiet --homedir "${GNUPGHOME}" -o /dev/null --sign $t
     rm $t
     exit 0
fi

#################### set gpg stuff, test it



gpgagentcmd="$0 -c $CONF --start-gpg-agent"

# set gpg-agent variables, test it
if test "$GNUPGAGENTINFO"  ; then 
 if test ! -r "$GNUPGAGENTINFO"  ; then 
  echo ERROR no agent info, please start the agent with
  echo $gpgagentcmd
  exit 1
 else
  . "$GNUPGAGENTINFO" 
  export GPG_AGENT_INFO
  if test ! "${GPG_AGENT_INFO}" -o  ! -e "${GPG_AGENT_INFO/:*/}" -o ! -O "${GPG_AGENT_INFO/:*/}" ; then
     echo ERROR agent info is not OK, please run the command
     echo $gpgagentcmd
     exit 1
  elif ! echo | gpg-connect-agent --homedir ${GNUPGHOME} ; then
     echo ERROR agent is not responding, please run the command
     echo $gpgagentcmd
     exit 1
  fi
 fi
fi


#test that we can sign, possibly loading the password in the agent
if test  "$GNUPGSEC" ; then
 t=`tempfile`
 echo pippo > $t
 if  ! gpg2 --quiet --batch --homedir "${GNUPGHOME}"  -o /dev/null --default-key $GNUPGSEC --sign $t  ;
 then
  echo signature test FAILED
  rm $t
  exit 1
 fi
 rm $t
fi

###################### some useful variables

#the lock used by debmirror
debmirlock=$debmir/Archive-Update-in-Progress-`hostname  -f`

b=`basename $0`

mylockfile=${TMPDIR:-/tmp}/$b.lock
lockfile -r 10  $mylockfile || exit 1
trap "rm $VERBOSE -f $mylockfile" 0

log=/nonexistant

## profile debdeltas
#debdeltas="python -m cProfile -o /tmp/debdelta-profile-$(dirname $p | tr / _ ) $debdeltas"

today=`date +'%F'`
yyyymm=`date +'%Y-%m'`

################## routines



clean_old_deltas () {
 find $deltamir/pool \
   \( -name '*debdelta-fails' -or -name '*debdelta-too-big' \
      -or -name  '*debdelta' \) -mtime +${old_delete_days} -type f |\
       xargs -r  $RM   $VERBOSE || true
 
 find $deltamir/pool/ -type d -empty | xargs -r rmdir $VERBOSE || true
}



run_debmirror () {
 if test -e $debmirlock ; then 
  echo Archive $debmir is locked 
  exit 1
 fi

 echo -n "---start mirroring at " >> $log ; date --utc >> $log

 tm=`tempfile`
 tme=`tempfile`

 $debmirror  $debmir --nosource \
  $debmirror_opt  --arch=$ARCHc \
  --section=$SECTIONc \
  -v $DEBUG -h $host -d $DISTc \
     > $tm 2> $tme ; dme=$? 

 if test "$dme" != 0  ; then
   echo debmirror failed , exit code $dme , stdout  $tm 
   awk '{ print "> " $0 }' $tm 
   echo debmirror failed stderr $tme 
   awk '{ print "> " $0 }' $tme
   $RM -f $debmirlock || true
   #sometimes the temporary directory of debmirror is messed up
   #rm -r $debmir/.temp
   clean_old_deltas
   rm $log
   exit 1
 else
   $RM $VERBOSE $tm $tme
 fi
 echo -n "---end mirroring at " >> $log ; date --utc >> $log
}



findmelog () {
 date=`date +'%F-%H'`
 log=$deltamir/log/$yyyymm/${date}.log
 if test -r $log -o -r $log.gz ; then 
  for i in 0 1 2 3 4 5 6 7 8 9 a b c d e f g h i j k l m n o p q r s t u v w x y z ; do 
   if test -r $log -o -r $log.gz  ; then 
     log=$deltamir/log/$yyyymm/${date}.$i.log
   fi
  done
 fi
}

created_deltas=0

run_debdeltas  () {
 debdeltas_errors=0
 echo -n "--------- running debdeltas ------"  >> $log  ; date --utc  >> $log  ;
 echo  "---- options ---- $debdelta_opt " >> $log
 for dist in $DISTs ; do
   dist_created_deltas=0
   distlog=`tempfile`
   #find latest
   pushd $debmir/dists/$dist/ > /dev/null
   latest=$(echo [0-9] [0-9][0-9] [0-9][0-9][0-9] [0-9][0-9][0-9][0-9] \
            | tr ' ' '\n' |  sort -n | tail -n 1)
   let prev=latest || true
   while test "$prev" -gt 1 && cmp -s Release $prev/Release ; do
     let prev=prev-1 || true
   done
   popd > /dev/null
   echo -------------- $dist ---- >> $distlog
   echo -n "---Release " >> $distlog ; grep ^Date $debmir/dists/$dist/Release >> $distlog
   echo -n "---Start at " >> $distlog ; date --utc >> $distlog
   #lenny contains debdelta 0.27, that does not understand lzma
   if test "$dist" = lenny ; then x="--disable-feature lzma,xz" ;
   #squeeze contains debdelta 0.39trl, that does not understand xz
   elif test "$dist" = squeeze ; then x="--disable-feature xz" ; else x="" ; fi
   for sec in $SECTIONs ; do  for arch in $ARCHs ; do
     sec_arch_dist_log=`tempfile`
     sec_arch_dist_err=`tempfile`
     wo="--old $debmir/dists/$dist/$prev/$sec/binary-$arch/Packages.gz"
     if test "$dist" = squeeze-updates ; then
       wo="$wo --old  $debmir/dists/squeeze/$sec/binary-$arch/Packages.gz" 
     fi
     OV=0 ; while test $OV -lt $prev ; do 
      if test -r   $debmir/dists/$dist/$OV/$sec/binary-$arch/Packages.gz  ; then
        wo="$wo --old  $debmir/dists/$dist/$OV/$sec/binary-$arch/Packages.gz"
      fi
      let OV=OV+10
     done
     echo "------ $sec $arch --  (debdelta options  +=   $x $wo )" >> $sec_arch_dist_log
     if $debdeltas $VERBOSE $debdelta_opt   $x $wo --dir $deltamir// \
        $debmir/dists/$dist/$sec/binary-$arch/Packages.gz >> $sec_arch_dist_log 2>> $sec_arch_dist_err ; then
      #todo, only write if something happened
      cat $sec_arch_dist_log >> $distlog
     else
      debdeltas_errors=$?
      cat $sec_arch_dist_log >> $distlog
      echo "--- debdeltas failed above, ret = ${debdeltas_errors}" >> $distlog
      echo "--- debdeltas failed on $dist $sec $arch"
     fi
     if test -s "$sec_arch_dist_err" ; then
        debdeltas_errors=11111 #make sure we keep this log
        echo "--- debdeltas stderr " >> $distlog
        cat $sec_arch_dist_err >> $distlog
        echo "--- end of debdeltas stderr " >> $distlog
        echo "--- debdeltas produced stderr on $dist $sec $arch"
     fi
     rm  $sec_arch_dist_log   $sec_arch_dist_err
   done ; done
   
   if  grep -v '^ Total running time'  $distlog |  grep -v '^---' | grep  -q '.' ; then
     let created_deltas=created_deltas+1
     dist_created_deltas=1
   fi
   if test  "${debdeltas_errors}" -gt 0 -o "${dist_created_deltas}" -gt 0 ; then
     #only log distributions stanzas when some deltas were generated, or errors occourred
     cat $distlog >> $log
     if test "${publish_deltas}" ; then
       echo -n "--- publishing starts at "  >> $log ; date --utc >> $log
       if ${publish_deltas} ; then
         echo -n "--- publishing ends at "  >> $log ; date --utc >> $log
       else
         echo -n "--- publishing FAILS, at " >> $log  ; date --utc >> $log
         echo "--- publishing FAILS"
       fi
     fi
   #else
   #  echo SKIPPED THIS STANZA ; cat $distlog
   fi
   rm $distlog
 done
 gzip -9 $log
 if test "$debdeltas_errors" -gt 0  ; then
  #some error occurred
  echo "-----ERRORS--- full log is in $log.gz "
 else
  #no error occurred, clean up
  if test "$created_deltas" = 0 ; then
    $RM $log.gz
  fi
 fi
}


run_debdeltas_clean () {
   echo -------------- cleanup delta pool
   $debdeltas --clean-deltas -n 0 \
     --dir $deltamir// \
       $(  for arch in $ARCHs ; do
            for sec in $SECTIONs ; do
              for dist in $DISTs ; do
                echo $debmir/dists/$dist/$sec/binary-$arch/Packages.gz 
           done ; done ; done )
}


debmarshal_trim_snapshots ()
{
 rectcode=1
 for dist in $DISTs ; do 
  pushd $debmir/dists/$dist/ > /dev/null
  #act only if there are at least 3 snapshots
  if test -d 0 -a -d 1 -a -d 3 ; then
   latest=$(echo [0-9] [0-9][0-9] [0-9][0-9][0-9] [0-9][0-9][0-9][0-9] \
            | tr ' ' '\n' |  sort -n | tail -n 1)
   #check sanity
   sane=1 ; t=0 ; f=1 ;
   while test "$f" -le "$latest" ; do
    if test ! -d "$f" ; then
     echo "Warning, ignoring $debmir/dists/$dist , the snapshot $f does not exist"
     sane=0
    elif test $t -nt $f ; then
     echo "Warning, ignoring $debmir/dists/$dist , the snapshot $f is older than the $t "
     sane=0
    fi
    let f++ ; let t++ ;
   done
   # if sane and too many snapshots, move all down a step
   if test "$sane" = 1 -a "$keep_marshal_snapshots"  -a "$latest" -gt "$keep_marshal_snapshots" ; then
    retcode=0
    let d=latest-keep_marshal_snapshots
    t=0
    #remove older
    while test "$t" -le "$d" -a -d "$d" ; do $RM -r $t ; let t++ ; done
    #move down others
    t=0
    let f=d+1
    while test "$f" -le "$latest" ; do
     $MV $f $t
     let t++
     let f++
    done
    let t--
    rm latest
    ln -s $t latest
    echo "------------ deleted revisions from 0 to $d in $dist (latest now points to $t)"
   fi # end of " if sane and too many snapshots"
  fi #end of "if at least 3 snapshots"
  popd > /dev/null
 done
 return $retcode
}

clean_old_debs () {
 if test -e $debmirlock ; then 
  echo Archive $debmir is locked 
  exit 1
 fi
 echo -------------- cleanup mirror
 if debmarshal_trim_snapshots ; then
  z=`tempfile`
  ${debmarshal_list_useless_debs} $debmir > $z
  ! grep -vx '.*\.deb' $z  #(just in case)
  n=$(wc -l < $z)
  if test $n -lt $max_delete ; then
   echo "--- delete $n debs"
   tr '\n' '\000' < $z | xargs -r0 $RM
   rm $z
  else
   echo "--- will not delete $n debs (too many!), the list is in $z"
  fi
 fi
}


########################################### code


if test `stat -f -c "%a" "$deltamir"` -le 512 ; then
 echo "------- emergency delta pool cleanup , very low disk space in delta mirror"
 run_debdeltas_clean
 clean_old_deltas
elif test "${DO_CLEAN_DELTAS}" ; then
 run_debdeltas_clean
 clean_old_deltas
fi

if test `stat -f -c "%a" "$debmir"` -le 512 ; then
 echo "------- emergency debs pool cleanup , very low disk space in debs mirror"
 clean_old_debs
elif test "${DO_CLEAN_DEBS}" ; then
 clean_old_debs
fi


test -d $deltamir/log/$yyyymm ||  mkdir $deltamir/log/$yyyymm

findmelog

if [ "${DO_MIRROR}" ] ; then
 run_debmirror
else
 echo ---------- skipped mirror step
fi

#the mirror was updated, so we check the difference 
run_debdeltas