#!/bin/sh # /usr/bin/makelocalsite - index local debian packages site # - build/ensure an local packages site file tree # - delete duplicate (new and older versions) packages # - link "all" packages into corresponding "${CONF_ARCH}" directories # - generate override files to stop Debian complaining # - generate Packages[.gz] and Sources[.gz] for local package site # author Neil Franklin, last modification 2006.12.19 # copyright ETH Zuerich Physics Departement, # use under either modified/non-advertising BSD or GPL license # this script is intended to be run as normal user, with access to the server # to use this: # - make an http virtualserver or directory in an http server (= base URL) # this will contain an Debian-like package structure # - place your local packages within that base URL: # dists//local//[source|binary-*]//*.deb # (this is done automatically by makesourcepackage -u) # - run this script, possibly with parameter(s) # - binary-all packages are auto-linked from binary-${CONF_ARCH} # - override files are generated from the packages # - Packages, Packages.gz, Sources, Sources.gz are generated, takes while :-) # (this is also done automatically by makesourcepackage -u) # - add the base URL to your clients /etc/apt/sources.list # - run apt-get update, then apt-get install as you wish ### ------ configuration for this site # first CONF_* various site or user dependant user config variables # then DEBUG_* various debugging settings # last SYS_* various system internal values # some of these are overridable by command line options # --- CONF_* various site or subnet dependant user config variables # make sure multiple users (group of them) can install files on package server # if this is set, chgrp directories and files to this group and chmod g+w CONF_GROUP="" # base directory on server where we put out packages file tree # this is base directory for adding dists//local/
/*/*/*.deb # paralleling Debians dists//
/*/*/*.deb on their servers # and so should appear in the http:// DocumentRoot space of your server # makesourcepackage -u uploads everything relative to this # and entire makelocalsite process runs relative to this CONF_PKG_BASE="/not/configured/directory" # distribution directory within our package server # we have added /local subdirectory analog to /non-US or /updates CONF_DISTRIB="sarge/local" # what architecture we use # this is used to link *_all.deb into *_${CONF_BINARCH}.deb # presently we can only handle single architecture package sites CONF_BINARCH="i386" # --- DEBUG_*, various debugging settings # these can be set to "yes" by -D option, followed by name without DEBUG_ # such as like this: dphys2rescue -D PRINT_STEP -D LEAVE_TEMPFILES -g # set this to sleep after displaying each steps header #DEBUG_SLEEP=yes # set this to output debug state info after each step #DEBUG_PRINT_STEP=yes # set this to wait after each debug state info #DEBUG_WAIT_STEP=yes # --- SYS_*, various system internal values # presently none ### ------ actual implementation from here on # no user settings any more below this point set -e # --- get ready to work # sanitise this place, else some commands may fail # must be before any commands are executed, incl config/input processing PATH=/sbin:/bin:/usr/sbin:/usr/bin export PATH # --- tidy up some commands, make systematic # stuff that goes wrong, not expected by user, not in data output, use >&2 # so also with $0 in case this script was called by an other script # something within the system that user does not expect, give up CMD_FATAL="echo $0: FATAL:" # something from users input, user will correct this and continue CMD_ERROR="echo $0: ERROR:" # something we can continue with, but may be wrong, and user may not suspect it CMD_WARNING="echo $0: WARNING:" # something most likely not wrong, but tell user for the odd case it is wrong CMD_NOTE="echo $0: NOTE:" # normal stuff users expect, so to stdout as normal output, no $0, no marking CMD_INFO="echo" # stuff users asked for, so add to stdout as normal output, no $0, but mark it CMD_DEBUG="echo DEBUG:" # other stuff we may want to use CMD_SLEEP="sleep 2" CMD_WAIT="read -p ---DEBUG-wait-after-step--- dummy" # DEBUG_* or option controllable stuff CMD_INFO_PRINT="${CMD_INFO}" CMD_VERBOSE_PRINT=true CMD_DEBUG_SLEEP=true CMD_DEBUG_PRINT=true CMD_DEBUG_WAIT=true # set debug option controllable stuff here if [ "${DEBUG_SLEEP}" = yes ] ; then CMD_DEBUG_SLEEP="${CMD_SLEEP}" fi if [ "${DEBUG_PRINT_STEP}" = yes ] ; then CMD_DEBUG_PRINT="${CMD_DEBUG}" fi if [ "${DEBUG_WAIT_STEP}" = yes ] ; then CMD_DEBUG_WAIT="${CMD_WAIT}" fi # --- config file stuff # what we are NAME=makelocalsite PNAME=dphys-pkgtools # check user config file(s), let user override settings # same config files used in makesourcepackage if [ -e /etc/"${PNAME}" ] ; then . /etc/"${PNAME}" fi if [ -e ~/."${PNAME}" ] ; then . ~/."${PNAME}" fi if [ -f ./"${PNAME}" ] ; then . ./"${PNAME}" fi # --- parse command line # so long next parameter is a set of options (= begins with an - character) while [ "`echo "$1" | cut -c 1`" = - ] ; do # extract options from parameter (cut off the "-") OPTS="`echo "$1" | cut -c 2-`" shift 1 # so long still unprocessed option characters while [ "${OPTS}" != "" ] ; do # first option to process OPT="`echo "${OPTS}" | cut -c 1`" # and rest of options for later OPTS="`echo "${OPTS}" | cut -c 2-`" case "${OPT}" in b) # base: upload generated package files to this base directory CONF_PKG_BASE="$1" shift 1 ;; d) # distrib: upload generated package files to this distribution CONF_DISTRIB="$1" shift 1 ;; q) # quiet: don't inform user what we are doing CMD_INFO_PRINT=true ;; v) # verbose: detailed inform user what we are doing CMD_VERBOSE_PRINT="${CMD_INFO}" ;; D) # Debug: set an debug option to yes eval "DEBUG_$1"=yes if [ "DEBUG_$1" = DEBUG_SLEEP ] ; then CMD_DEBUG_SLEEP="${CMD_SLEEP}" fi if [ "DEBUG_$1" = DEBUG_PRINT_STEP ] ; then CMD_DEBUG_PRINT="${CMD_DEBUG}" fi if [ "DEBUG_$1" = DEBUG_WAIT_STEP ] ; then CMD_DEBUG_WAIT="${CMD_WAIT}" fi shift 1 ;; h) # help: give out help how this script can be used cat << END-HELP-TEXT Usage is: $0 [options] section [section ...] options: -b baseaddr base: upload generated package files to this base directory -d distrib distrib: upload generated package files to this distribution -q quiet: don't give user an running report of what we are doing -v verbose: give user detailed running report -D Debug: activate an debug option, see source for operation -h help: output this text, and then abort operation section: must be section part 1 name (such as main or non-free or contrib) or multiple of them END-HELP-TEXT exit 0 ;; *) # not one of our recognized options ${CMD_ERROR} "unknown option: ${OPT}" >&2 ${CMD_INFO} >&2 # call self with -h to display help "$0" -h >&2 exit 1 ;; esac done done if [ "$#" = 0 ] ; then # user did not give an parameter (section) ${CMD_ERROR} "missing section name(s) to work on" >&2 ${CMD_INFO} >&2 # call self with -h to display help "$0" -h >&2 exit 1 fi ${CMD_DEBUG_PRINT} "options parsed" ${CMD_DEBUG_WAIT} # --- set work environment # ensure we have an work directory, environment is sane if [ ! -d "${CONF_PKG_BASE}" ] ; then ${CMD_FATAL} "work directory ${CONF_PKG_BASE} not found" >&2 exit 1 fi # switch to our packages file tree, so all paths can be shorter relative ones cd "${CONF_PKG_BASE}" # --- check if demanded sections are valid # only process valid distributions, else massive error cascade and junk files if [ "`ls -1d "dists/${CONF_DISTRIB}" 2> /dev/null`" = "" ] ; then ${CMD_FATAL} "there exists no distribution \"${CONF_DISTRIB}\"" \ "to work in among:" >&2 (cd dists; ls -1d * >&2) exit 1 fi # only process valid sections, else massive error cascade and junk files SECTIONS="" while [ "$1" != "" ] ; do # sanitise section names, only valid chars, avoid escaping bugs hitting "for" # other user error junk will get caught by ${SECTION} file name check SECTION=`echo "$1" | tr -c -d 'a-z-'` # this test replaces to original fixed contrib main non-free test # and so allows users to have their own sections, such as backports if [ "`ls -1d "dists/${CONF_DISTRIB}/${SECTION}" 2>/dev/null`" = "" ] ; then ${CMD_FATAL} "there exists no section \"${SECTION}\" to process among:" >&2 (cd "dists/${CONF_DISTRIB}"; ls -1d * >&2) exit 1 fi SECTIONS="${SECTIONS} ${SECTION}" shift 1 done ${CMD_DEBUG_PRINT} "sections: ${SECTIONS}" ${CMD_DEBUG_WAIT} # --- handle each section # no "${SECTIONS}" in for, else intervening spaces end up in SECTION # and that would then really confuse the stuff processing ${SECTION} # content of ${SECTIONS} has already been sanitised up above for SECTION in ${SECTIONS} ; do ${CMD_INFO_PRINT} "indexing section ${SECTION} of ${CONF_DISTRIB}" \ "on ${CONF_PKG_BASE} ..." ${CMD_DEBUG_SLEEP} # --- set up for this section # stuff to work on LOCK="dists/${CONF_DISTRIB}/${SECTION}/.makelocalsite.lock" B_ALL="dists/${CONF_DISTRIB}/${SECTION}/binary-all" B_ARCH="dists/${CONF_DISTRIB}/${SECTION}/binary-${CONF_BINARCH}" SOURCE="dists/${CONF_DISTRIB}/${SECTION}/source" # --- ensure user does not abbort part-indexed, leaving server inconsistant # yes this program needs quite a bit of time, if started unnecessarily # but inconsistency requires full rerun to fix, and running does no harm # if terminal lost, just carry on, senseless to say anything trap "" SIGHUP # if user presses Ctrl-C educate them of the consequences # we allow kill -term in case user really is serious about this # ### bash lossage!!! no way to tell child processes to ignore signal # unless also set to ignore, no action, no echo, in the parent #trap "echo 'do not interrupt (re-)indexing, \ #else the server will be inconsistant!' >&2" SIGINT SIGQUIT trap "" SIGINT SIGQUIT # --- wait if other user is already running makelocalsite on this section if [ -e "${LOCK}" ] ; then # someone else is running an makelocalsite, don't collide ${CMD_NOTE} "waiting for lock from `cat "${LOCK}"` to disapper," \ "the already running indexer to finish ..." >&2 while [ -e "${LOCK}" ] ; do sleep 1 done ${CMD_NOTE} "continuing ..." >&2 fi # strictly we have an race condition here # ignore it, as only small damage, undoable by re-running echo "user `whoami` on `hostname` PID $$" > "${LOCK}" # --- eliminate duplicate/multicate (new and older) versions of packages # tidy up "all" ${CMD_INFO_PRINT} "removing duplicate .deb ${SECTION}/binary-all ..." ${CMD_DEBUG_SLEEP} for DEB in "${B_ALL}"/*/*.deb ; do if [ -e "${DEB}" ]; then PKG="`basename "${DEB}"`" PKGN="`echo "${PKG}" | cut -f 1 -d '_'`" if [ "`ls -1 "${B_ALL}"/*/"${PKGN}"_*.deb | wc -l`" != "1" ] ; then # time sort -tr may fail for links, doesn't matter, will remake anyway NEWEST="`ls -1tr "${B_ALL}"/*/"${PKGN}"_*.deb | tail -1`" mv "${NEWEST}" ".tmp.${PKGN}.deb" rm "${B_ALL}"/*/"${PKGN}"_*.deb mv ".tmp.${PKGN}.deb" "${NEWEST}" fi fi done # tidy up "${CONF_BINARCH}" ${CMD_INFO_PRINT} "removing duplicate .deb Files in" \ "${SECTION}/binary-${CONF_BINARCH} ..." ${CMD_DEBUG_SLEEP} for DEB in "${B_ARCH}"/*/*.deb ; do if [ -e "${DEB}" ]; then PKG="`basename "${DEB}"`" PKGN="`echo "${PKG}" | cut -f 1 -d '_'`" if [ "`ls -1 "${B_ARCH}"/*/"${PKGN}"_*.deb | wc -l`" != "1" ] ; then # time sort -tr may fail for links, doesn't matter, will remake anyway NEWEST="`ls -1tr "${B_ARCH}"/*/"${PKGN}"_*.deb | tail -1`" mv "${NEWEST}" ".tmp.${PKGN}.deb" rm "${B_ARCH}"/*/"${PKGN}"_*.deb mv ".tmp.${PKGN}.deb" "${NEWEST}" fi fi done # tidy up "source" # duplicates/mutliples of these do not generate any error messages # so keep them for archival purpose, delete manually if too little space # --- links from binary-${CONF_BINARCH} to binary-all # remove old links from ${CONF_BINARCH} ${CMD_INFO_PRINT} "deleting old links ${SECTION}/binary-${CONF_BINARCH} ..." ${CMD_DEBUG_SLEEP} for DEB in "${B_ARCH}"/*/*_all.deb ; do if [ -L "${DEB}" ] ; then rm -f "${DEB}" fi done # generate new ones ${CMD_INFO_PRINT} "making new links ${SECTION}/binary-${CONF_BINARCH}" \ "to binary-all ..." ${CMD_DEBUG_SLEEP} for DEB in "${B_ALL}"/*/*.deb ; do if [ -e "${DEB}" ]; then # extract last directory name in path (= section) SEC="`dirname "${DEB}"`" while [ "`echo "${SEC}" | grep '/'`" != "" ] ; do SEC="`echo "${SEC}" | cut -f 2- -d '/'`" done PKG="`basename "${DEB}"`" # ensure there is an ${CONF_BINARCH} directory for this package section if [ ! -d "${B_ARCH}/${SEC}" ] ; then mkdir -p "${B_ARCH}/${SEC}" if [ "${CONF_GROUP}" != "" ] ; then chgrp "${CONF_GROUP}" "${B_ARCH}/${SEC}" chmod 2775 "${B_ARCH}/${SEC}" fi fi ln -s "../../binary-all/${SEC}/${PKG}" "${B_ARCH}/${SEC}" fi done ${CMD_DEBUG_PRINT} "+++ begin links listing +++" if [ "${DEBUG_PRINT_STEP}" = yes ] ; then ls -al "${B_ARCH}"/*/*.deb 2>/dev/null fi ${CMD_DEBUG_PRINT} "+++ end links listing +++" ${CMD_DEBUG_WAIT} # --- override files # ensure there is an directory for then if [ ! -d indices ] ; then mkdir -p indices if [ "${CONF_GROUP}" != "" ] ; then chgrp "${CONF_GROUP}" indices chmod 2775 indices fi fi # stuff to work on DISTRIB_BASE="`echo "${CONF_DISTRIB}" | cut -f 1 -d '/'`" OVRS="indices/override.${DISTRIB_BASE}.${SECTION}.src" OVRB="indices/override.${DISTRIB_BASE}.${SECTION}" # generate source override files ${CMD_INFO_PRINT} "making source override ${SECTION}.src ..." ${CMD_DEBUG_SLEEP} # delete old file and its owner, else chmod complains # may not exist, when new distribution/section first time indexed if [ -f "${OVRS}" ] ; then rm "${OVRS}" fi # start empty file, for loop using >> echo -n > "${OVRS}" # here we need to evaluate .dsc files, as .tar.gz no recognizable content for DSC in "${SOURCE}"/*/*.dsc ; do if [ -e "${DSC}" ]; then DIR="`dirname "${DSC}"`" TAR="`grep '.* [0-9]* .*\.tar\.gz' "${DSC}" | cut -f 4 -d ' '`" if [ "`echo "${TAR}" | grep '\.orig\.'`" = "" ] ; then # simple .tar.gz file, extract control from that tar zxvf "${DIR}/${TAR}" -O "*/debian/control" > .ctrl 2> /dev/null else # combination of .orig.tar.gz + .diff.gz files, reconstruct from .diff DIFF="`grep ' .* [0-9]* .*\.diff\.gz' "${DSC}" | cut -f 4 -d ' '`" zcat "${DIR}/${DIFF}" | \ sed -n -e '/^+++.*\/debian\/control$/,/^---/p' | \ grep '^+' | cut -f 2 -d '+' > .ctrl fi # use head -1 here because multi-package sources have repeat lines PKG="`grep "Source: " .ctrl | head -1 | cut -f 2 -d ' '`" PRI="`grep "Priority: " .ctrl | head -1 | cut -f 2 -d ' '`" SEC="`grep "Section: " .ctrl | head -1 | cut -f 2 -d ' '`" if [ "${PKG}" != "" -a "${PRI}" != "" -a "${SEC}" != "" ]; then printf "%s\t%s\t%s\n" "${PKG}" "${PRI}" "${SEC}" >> "${OVRS}" else ${CMD_WARNING} "${DSC} incomplete control file, ommitted" >&2 ${CMD_DEBUG_PRINT} "+++ begin control file +++" if [ "${DEBUG_PRINT_STEP}" = yes ] ; then cat .ctrl fi ${CMD_DEBUG_PRINT} "+++ end control file +++" ${CMD_DEBUG_WAIT} fi rm .ctrl fi done # eliminate duplicates (2 versions of same package), so no warning sort "${OVRS}" | uniq > "${OVRS}.tmp" mv "${OVRS}.tmp" "${OVRS}" # we have multiple users adding packages, so they need group access here if [ "${CONF_GROUP}" != "" ] ; then chgrp "${CONF_GROUP}" "${OVRS}" chmod 664 "${OVRS}" fi # generate binary override files, similar code as above ${CMD_INFO_PRINT} "making binary override ${SECTION} ..." ${CMD_DEBUG_SLEEP} if [ -f "${OVRB}" ] ; then rm "${OVRB}" fi echo -n > "${OVRB}" # here we need to evaluate .deb files, as they contain everything # only search ${CONF_BINARCH}, as all .all packages have links in there for DEB in "${B_ARCH}"/*/*.deb ; do if [ -e "${DEB}" ]; then PKG="`dpkg-deb --info "${DEB}" | grep "Package: " | cut -f 3 -d ' '`" PRI="`dpkg-deb --info "${DEB}" | grep "Priority: " | cut -f 3 -d ' '`" SEC="`dpkg-deb --info "${DEB}" | grep "Section: " | cut -f 3 -d ' '`" printf "%s\t%s\t%s\n" "${PKG}" "${PRI}" "${SEC}" >> "${OVRB}" fi done sort "${OVRB}" | uniq > "${OVRB}.tmp" mv "${OVRB}.tmp" "${OVRB}" if [ "${CONF_GROUP}" != "" ] ; then chgrp "${CONF_GROUP}" "${OVRB}" chmod 664 "${OVRB}" fi ${CMD_DEBUG_PRINT} "`ls -al "${OVRS}" 2>/dev/null`" ${CMD_DEBUG_PRINT} "`ls -al "${OVRB}" 2>/dev/null`" ${CMD_DEBUG_WAIT} # --- source and package lists # check for stray files, else these are not in override # and dpkg-scanpackages shouts then for DEB in "${SECTION}"/*.tar.gz "${B_ALL}"/*.deb "${B_ARCH}"/*.deb ; do if [ -e "${DEB}" ] ; then ${CMD_WARNING} "missplaced package file: ${DEB}" >&2 fi done # generate source lists ${CMD_INFO_PRINT} "making sources list ${SECTION}/source ..." ${CMD_DEBUG_SLEEP} if [ -d "${SOURCE}" ] ; then fakeroot dpkg-scansources "${SOURCE}" "${OVRS}" > "${SOURCE}/Sources" fakeroot gzip < "${SOURCE}/Sources" > "${SOURCE}/Sources.gz" fi # generate "all" package lists ${CMD_INFO_PRINT} "making packages list ${SECTION}/binary-all ..." ${CMD_DEBUG_SLEEP} if [ -d "${B_ALL}" ] ; then fakeroot dpkg-scanpackages "${B_ALL}" "${OVRB}" > "${B_ALL}/Packages" fakeroot gzip < "${B_ALL}/Packages" > "${B_ALL}/Packages.gz" fi # generate "${CONF_BINARCH}" package lists ${CMD_INFO_PRINT} "making packages list" \ "${SECTION}/binary-${CONF_BINARCH} ..." ${CMD_DEBUG_SLEEP} if [ -d "${B_ARCH}" ] ; then fakeroot dpkg-scanpackages "${B_ARCH}" "${OVRB}" > "${B_ARCH}/Packages" fakeroot gzip < "${B_ARCH}/Packages" > "${B_ARCH}/Packages.gz" fi ${CMD_DEBUG_PRINT} "`ls -al "${SOURCE}/Sources" 2>/dev/null`" ${CMD_DEBUG_PRINT} "`ls -al "${SOURCE}/Sources.gz" 2>/dev/null`" ${CMD_DEBUG_PRINT} "`ls -al "${B_ALL}/Packages" 2>/dev/null`" ${CMD_DEBUG_PRINT} "`ls -al "${B_ALL}/Packages.gz" 2>/dev/null`" ${CMD_DEBUG_PRINT} "`ls -al "${B_ARCH}/Packages" 2>/dev/null`" ${CMD_DEBUG_PRINT} "`ls -al "${B_ARCH}/Packages.gz" 2>/dev/null`" ${CMD_DEBUG_WAIT} # --- undo lock as we are not running this section any more if [ -e "${LOCK}" ] ; then rm "${LOCK}" fi # --- and undo trapping of signals, allow Ctrl-C after each section trap - SIGHUP SIGINT SIGQUIT done exit 0