implement apt-transport-mirror handling

README.md

@@ -187,14 +187,21 @@ Official mirror lists:
 * Debian: http://www.debian.org/mirror/list
 * Ubuntu: https://launchpad.net/ubuntu/+archivemirrors
 
-Then add them to whitespace and comma separated list in config file, e.g.:
+Then add them to space and/or comma separated list in config file, e.g.:
 
 ```sh
-MIRRORS=( 'http://deb.debian.org/debian','http://ftp.debian.org/debian, http://ftp2.de.debian.org/debian, http://ftp.de.debian.org/debian, ftp://ftp.uni-kl.de/debian' )
+MIRRORS=(
+  'http://deb.debian.org/debian','http://ftp.debian.org/debian, http://ftp2.de.debian.org/debian, http://ftp.de.debian.org/debian ftp://ftp.uni-kl.de/debian'
+)
 ```
 
+If you are using multiple distributions (such as Ubuntu PPA on Debian or other 3rd-party repoitories), you can simply add one more list into this array of lists:
+
 ```sh
-MIRRORS=( 'http://archive.ubuntu.com/ubuntu, http://de.archive.ubuntu.com/ubuntu, http://ftp.halifax.rwth-aachen.de/ubuntu, http://ftp.uni-kl.de/pub/linux/ubuntu, http://mirror.informatik.uni-mannheim.de/pub/linux/distributions/ubuntu/' )
+MIRRORS=(
+  'http://deb.debian.org/debian http://ftp.debian.org/debian http://ftp2.de.debian.org/debian http://ftp.de.debian.org/debian ftp://ftp.uni-kl.de/debian'
+  'http://archive.ubuntu.com/ubuntu http://de.archive.ubuntu.com/ubuntu http://ftp.halifax.rwth-aachen.de/ubuntu http://ftp.uni-kl.de/pub/linux/ubuntu http://mirror.informatik.uni-mannheim.de/pub/linux/distributions/ubuntu/'
+)
 ```
 
 *NOTE:* To use any mirrors you may have in sources.list or sources.list.d you will need to add them to the apt-fast.conf mirror list as well!
@@ -303,3 +310,4 @@ Special thanks
  * Sergio Silva - test to see if axel is installed, root detection/sudo autorun, lock file check/creation
  * Waldemar {BOB}{Burnfaker} Wetzel - lockfile improvements, separate config file
  * maclarke - locking improvements
+ * artoria2e5 - apt-file-mirror support

apt-fast

@@ -17,22 +17,32 @@
 # Message types are 'normal', 'hint' or 'warning'. Warnings and messages with a
 # third argument are piped to stderr.
 msg(){
-  msg_options=()
+  local beginColor msg_options=() fd=1 nl='\n'
   case "$2" in
     normal) beginColor="$cGreen";;
     hint) beginColor="$cBlue";;
-    warning) beginColor="$cRed";;
-    question) beginColor="$cRed"; msg_options=(-n);;
+    warning) beginColor="$cRed" fd=2;;
+    question) beginColor="$cRed"; nl='';;
     *) beginColor= ;;
   esac
+  if [ -n "$3" ]; then fd=2; fi
 
-  if [ -z "$3" ] && [ "$2" != "warning" ]; then
-    echo -e "${msg_options[@]}" "${aptfast_prefix}${beginColor}$1${endColor}"
-  else
-    echo -e "${msg_options[@]}" "${aptfast_prefix}${beginColor}$1${endColor}" >&2
-  fi
+  printf "%s${beginColor}%b${endColor}${nl}" "$(aptfast_prefix)" "$1" >&"$fd"
 }
 
+# Early bash version check.
+if (( BASH_VERSINFO[0] < 4 )) ||
+   (( BASH_VERSINFO[0] == 4 && BASH_VERSINFO[0] < 3 ))
+then
+  printf '%s\n' "This script uses features only available in bash 4.3+: "\
+                "associative arrays and namerefs."$'\n'\
+                "Please run it with a never version of GNU bash "\
+                "(you are running ${BASH_VERSION}.)">&2
+  exit 2
+fi
+
+shopt -s extglob
+
 # Search for known options and decide if root privileges are needed.
 root=1  # default value: we need root privileges
 option=
@@ -62,49 +72,43 @@ done
 # we need to preserve all interesting env variables. As this wouldn't be
 # difficult enough we have to preserve complete env vars (especially if value
 # ist set (even empty) or not) when changing context (sudo)...
+sudo_exports=(DEBUG="$DEBUG")
+shadowvars=()
 # Set a 'random' string to all unset variables.
 TMP_RANDOM="13979853562951413"
-TMP_LCK_FILE="${LCK_FILE-${TMP_RANDOM}}"
-TMP_DOWNLOADBEFORE="${DOWNLOADBEFORE-${TMP_RANDOM}}"
-TMP__APTMGR="${_APTMGR-${TMP_RANDOM}}"
-TMP_APTCACHE="${APTCACHE-${TMP_RANDOM}}"
-TMP_DLDIR="${DLDIR-${TMP_RANDOM}}"
-TMP_DLLIST="${DLLIST-${TMP_RANDOM}}"
-TMP_LISTDIR="${LISTDIR-${TMP_RANDOM}}"
-TMP__MAXNUM="${MAXNUM-${TMP_RANDOM}}"
-TMP__MAXCONPERSRV="${MAXCONPERSRV-${TMP_RANDOM}}"
-TMP__SPLITCON="${SPLITCON-${TMP_RANDOM}}"
-TMP__MINSPLITSZ=${MINSPLITSZ-${TMP_RANDOM}}
-TMP__PIECEALGO=${PIECEALGO-${TMP_RANDOM}}
-TMP_aptfast_prefix="${aptfast_prefix-${TMP_RANDOM}}"
-TMP_APT_FAST_TIMEOUT="${APT_FAST_TIMEOUT-${TMP_RANDOM}}"
-TMP_VERBOSE_OUTPUT="${VERBOSE_OUTPUT-${TMP_RANDOM}}"
-TMP_ftp_proxy="${ftp_proxy-${TMP_RANDOM}}"
-TMP_http_proxy="${http_proxy-${TMP_RANDOM}}"
-TMP_https_proxy="${https_proxy-${TMP_RANDOM}}"
+shadowify() {
+  # I heard that people call this "variable shadowing."
+  declare -n this="$1" tmpthis="TMP_$1"
+  tmpthis="${this-${TMP_RANDOM}}"
+  sudo_exports+=("TMP_$1=$tmpthis")
+  shadowvars+=("$1")
+}
+
+shadowify LCK_FILE
+shadowify DOWNLOADBEFORE
+shadowify _APTMGR
+shadowify APTCACHE
+shadowify DLDIR
+shadowify DLLIST
+shadowify MIRRORLIST
+shadowify LISTDIR
+shadowify _MAXNUM
+shadowify _MAXCONPERSRV
+shadowify _SPLITCON
+shadowify _MINSPLITSZ
+shadowify _SPLITCON
+shadowify _MINSPLITSZ
+shadowify _PIECEALGO
+shadowify APT_FAST_TIMEOUT
+shadowify VERBOSE_OUTPUT
+shadowify ftp_proxy
+shadowify http_proxy
+shadowify https_proxy
 
 # Check for proper privileges.
 # Call explicitly with environment variables to get them into root conext.
 if [ "$root" = 1 ] && [ "$UID" != 0 ]; then
-  exec sudo DEBUG="$DEBUG" \
-            LCK_FILE="$TMP_LCK_FILE" \
-            DOWNLOADBEFORE="$TMP_DOWNLOADBEFORE" \
-            _APTMGR="$TMP__APTMGR" \
-            APTCACHE="$TMP_APTCACHE" \
-            DLDIR="$TMP_DLDIR" \
-            DLLIST="$TMP_DLLIST" \
-            LISTDIR="$TMP_LISTDIR" \
-            _MAXNUM="$TMP__MAXNUM" \
-            _MAXCONPERSRV="$TMP__MAXCONPERSRV" \
-            _SPLITCON="$TMP__SPLITCON" \
-            _MINSPLITSZ="$TMP__MINSPLITSZ" \
-            _PIECEALGO="$TMP__PIECEALGO" \
-            aptfast_prefix="$TMP_aptfast_prefix" \
-            APT_FAST_TIMEOUT="$TMP_APT_FAST_TIMEOUT" \
-            VERBOSE_OUTPUT="$TMP_VERBOSE_OUTPUT" \
-            ftp_proxy="$TMP_ftp_proxy" \
-            http_proxy="$TMP_http_proxy" \
-            https_proxy="$TMP_https_proxy" \
+  exec sudo "${sudo_exports[@]}"
             "$0" "$@"
 fi
 
@@ -130,15 +134,15 @@ fi
 # Currently not needed.
 eval "$(apt-config shell LISTDIR Dir::State::lists/d)"
 DLLIST="/tmp/apt-fast.list"
+MIRRORLIST="/tmp/apt-fast-mirror.list"
 _MAXNUM=5
 _MAXCONPERSRV=10
 _SPLITCON=8
 _MINSPLITSZ="1M"
 _PIECEALGO="default"
 
 # Prefix in front of apt-fast output:
-aptfast_prefix=
-# aptfast_prefix="$(date '+%b %_d %T.%N') apt-fast: "
+# aptfast_prefix(){ printf '%s' "$(date '+%b %_d %T.%N') apt-fast: "; }
 
 # Set color variables.
 cGreen='\e[0;32m'
@@ -175,24 +179,15 @@ https_proxy=
 
 # Now overwrite with preserved values if values were set before (compare with
 # 'random' string).
-[ "$TMP_LCK_FILE" = "$TMP_RANDOM" ] || LCK_FILE="$TMP_LCK_FILE"
-[ "$TMP_DOWNLOADBEFORE" = "$TMP_RANDOM" ] || DOWNLOADBEFORE="$TMP_DOWNLOADBEFORE"
-[ "$TMP__APTMGR" = "$TMP_RANDOM" ] || _APTMGR="$TMP__APTMGR"
-[ "$TMP_APTCACHE" = "$TMP_RANDOM" ] || APTCACHE="$TMP_APTCACHE"
-[ "$TMP_DLDIR" = "$TMP_RANDOM" ] || DLDIR="$TMP_DLDIR"
-[ "$TMP_DLLIST" = "$TMP_RANDOM" ] || DLLIST="$TMP_DLLIST"
-[ "$TMP_LISTDIR" = "$TMP_RANDOM" ] || LISTDIR="$TMP_LISTDIR"
-[ "$TMP__MAXNUM" = "$TMP_RANDOM" ] || _MAXNUM="$TMP__MAXNUM"
-[ "$TMP__MAXCONPERSRV" = "$TMP_RANDOM" ] || _MAXCONPERSRV="$TMP__MAXCONPERSRV"
-[ "$TMP__SPLITCON" = "$TMP_RANDOM" ] || _SPLITCON="$TMP__SPLITCON"
-[ "$TMP__MINSPLITSZ" = "$TMP_RANDOM" ] || _MINSPLITSZ="$TMP__MINSPLITSZ"
-[ "$TMP__PIECEALGO" = "$TMP_RANDOM" ] || _PIECEALGO="$TMP__PIECEALGO"
-[ "$TMP_aptfast_prefix" = "$TMP_RANDOM" ] || aptfast_prefix="$TMP_aptfast_prefix"
-[ "$TMP_APT_FAST_TIMEOUT" = "$TMP_RANDOM" ] || APT_FAST_TIMEOUT="$TMP_APT_FAST_TIMEOUT"
-[ "$TMP_VERBOSE_OUTPUT" = "$TMP_RANDOM" ] || VERBOSE_OUTPUT="$TMP_VERBOSE_OUTPUT"
-[ "$TMP_ftp_proxy" = "$TMP_RANDOM" ] || ftp_proxy="$TMP_ftp_proxy"
-[ "$TMP_http_proxy" = "$TMP_RANDOM" ] || http_proxy="$TMP_http_proxy"
-[ "$TMP_https_proxy" = "$TMP_RANDOM" ] || https_proxy="$TMP_https_proxy"
+for i in "${shadowvars[@]}"
+do
+  declare -n thisvar="$i" tempthisvar="TMP_$i"
+  if [[ $tempthisvar != $TMP_RANDOM ]]
+  then
+    thisvar=${tempthisvar}
+  fi
+done
+unset -n thisvar tempthisvar
 
 
 # Disable colors if not executed in terminal.
@@ -201,8 +196,17 @@ if [ ! -t 1 ]; then
   cRed=
   cBlue=
   endColor=
-  #FIXME: Time not updated.
-  [ -z "$aptfast_prefix" ] && aptfast_prefix="[apt-fast $(date +"%T")]"
+fi
+
+# Define the (logging) prefix if it is never defined.
+if ! declare -f aptfast_prefix 2>/dev/null; then
+  if [ -t 1 ]; then
+    # The TTY user has a clock on the wall. No need.
+    aptfast_prefix(){ true; }
+  else
+    # %(%T)T is like $(date +%T), but built into printf.
+    aptfast_prefix(){ printf '[apt-fast %(%T)T] '; }
+  fi
 fi
 
 
@@ -238,26 +242,28 @@ _remove_lock()
 
 # Move download file away so missing permissions won't stop usage.
 CLEANUP_STATE=0
-cleanup_dllist()
+cleanup_list()
 {
-  if [ -f "$DLLIST" ]
+  if [ ! -f "$1" ]
   then
-    if ! mv -- "$DLLIST{,.old}" 2>/dev/null
-    then
-      if ! rm -f -- "$DLLIST" 2>/dev/null
-      then
-        msg "Could not clean up download list file." "warning"
-        CLEANUP_STATE=1
-      fi
-    fi
+    return
+  fi
+
+  if ! {
+    mv -- "$1{,.old}" 2>/dev/null ||
+    rm -f -- "$1" 2>/dev/null
+  }
+  then
+    msg "Could not clean up $2 list file." "warning"
+    CLEANUP_STATE=1
   fi
 }
 
 cleanup_aptfast()
 {
   local last_exit_code=$?
   [ "$CLEANUP_STATE" -eq 0 ] && CLEANUP_STATE=$last_exit_code
-  cleanup_dllist
+  cleanup_list "$DLLIST" download
   _remove_lock
 }
 
@@ -273,8 +279,55 @@ urldecode()
     printf '%b' "${1//%/\\x}"
 }
 
+# Maps a mirror uri to the first entry of that list, so get_mirrors can take it.
+# We definitely don't want to resolve those more than once each run...
+declare -A mirror_uri_map
+
+# Handles a mirror URI, with caching. Parses the given list into MIRRORS.
+handle_mirror()
+{
+  # The most recognizable split point is the "pool" thing. This
+  # will be apt-ftparchive-specific, but I suspect the transport is doing that
+  # itself.
+  local head="${1%%/pool/*}" body="/pool/${1#*/pool/}"
+
+  if ! [[ ${mirror_uri_map[$head]} ]]; then
+    local protocol="${head%%:*}" path="${head#*:}"
+    local mirrorfile
+
+    case $protocol in
+      (mirror+file)
+        mirrorfile=$path;;
+      (*)
+        # Well, gotta fetch...
+        if [[ $protocol == mirror ]]; then protocol=mirror+http; fi
+        aria2c -q -d "${MIRRORLIST%/*}" -o "${MIRRORLIST##*/}" -- "${protocol#mirror+}:${path}"
+        mirrorfile="${MIRRORLIST}";;
+    esac
+
+    local mirrors
+    mapfile -t mirrors < "$mirrorfile"
+    [ -f "MIRRORLIST" ] && rm -f "$MIRRORLIST"
+    # This is a TAB. The examples say the URI ends at the TAB. We extract that.
+    mirrors=("${mirrors[@]%%  *}")
+    # Normalize the slashes. (s@/*$@/@g)
+    mirrors=("${mirrors[@]/%*(\/)/\/}")
+    # get_mirrors says spaces are fine... no local IFS=, needed here.
+    MIRRORS+=("${mirrors[*]}")
+    mirror_uri_map[$head]=${mirrors[0]}
+  fi
+
+  printf '%s\n' "${mirror_uri_map[$head]}$body"
+}
+
 # Check if mirrors are available. And if so add all mirrors to download list.
 get_mirrors(){
+  local mirrors
+  # Handle apt-transport-mirror
+  if [[ $1 == mirror* ]]; then
+    set -- "$(handle_mirror "$1")"
+  fi
+
   # Check all mirror lists.
   for mirrorstr in "${MIRRORS[@]}"; do
     # Build mirrors array from comma separated string.
@@ -415,7 +468,8 @@ get_uris(){
       fi
       echo " out=$filename"
     } >> "$DLLIST"
-  done <<<"$(echo "$uris_full" | grep -E "^'(http(s|)|(s|)ftp)://")"
+  done <<<"$(<<<"$uris_full" grep -E "^'((mirror\+)?((http(s|)|(s|)ftp)|mirror)://|mirror\+file:/)")"
+  #'
 
   #cat "$DLLIST"
   #LCK_RM
@@ -434,7 +488,7 @@ display_downloadfile(){
         pkg="$(echo "$line" | cut -d' ' -f1)"
         ver="$(echo "$line" | cut -d' ' -f2)"
         size="$(echo "$line" | cut -d' ' -f3)"
-        printf '%s%-40s %-20s %10s\n' "$aptfast_prefix" "$pkg" "$ver" "$size"
+        printf '%s%-40s %-20s %10s\n' "$(aptfast_prefix)" "$pkg" "$ver" "$size"
     done <<<"$(echo -e "$DOWNLOAD_DISPLAY" | sort "${DISPLAY_SORT_OPTIONS[@]}")"
   fi
   msg "Download size: $(echo "$DOWNLOAD_SIZE" | numfmt --to=iec-i --suffix=B)" "normal"

apt-fast.conf

@@ -1,3 +1,4 @@
+# -*- mode: shell -*- vi: set ft=sh
 ###################################################################
 # CONFIGURATION OPTIONS
 ###################################################################
@@ -28,7 +29,7 @@
 # Ubuntu: https://launchpad.net/ubuntu/+archivemirrors
 #
 # It is required to add mirrors in the sources.list to this array as well, so
-# apt-fast can destinguish between different distributions.
+# apt-fast can distinguish between different distributions.
 #
 # Examples:
 #
@@ -39,8 +40,10 @@
 # deb http://de.archive.ubuntu.com/ubuntu/ bionic main universe
 #
 # apt-fast.conf:
-# MIRRORS=( 'http://deb.debian.org/debian','http://ftp.debian.org/debian,http://ftp2.de.debian.org/debian,http://ftp.de.debian.org/debian,ftp://ftp.uni-kl.de/debian'
-#           'http://archive.ubuntu.com/ubuntu,http://de.archive.ubuntu.com/ubuntu,http://ftp.halifax.rwth-aachen.de/ubuntu,http://ftp.uni-kl.de/pub/linux/ubuntu,http://mirror.informatik.uni-mannheim.de/pub/linux/distributions/ubuntu/' )
+# MIRRORS=(
+#    'http://deb.debian.org/debian','http://ftp.debian.org/debian,http://ftp2.de.debian.org/debian,http://ftp.de.debian.org/debian,ftp://ftp.uni-kl.de/debian'
+#    'http://archive.ubuntu.com/ubuntu,http://de.archive.ubuntu.com/ubuntu,http://ftp.halifax.rwth-aachen.de/ubuntu,http://ftp.uni-kl.de/pub/linux/ubuntu,http://mirror.informatik.uni-mannheim.de/pub/linux/distributions/ubuntu/'
+# )
 #
 #
 # Single distribution:
@@ -101,6 +104,14 @@
 #DLLIST=/tmp/apt-fast.list
 
 
+# Mirror listfile
+# This determines where temporary resolutions for the mirror:// url is stored.
+#
+# Default: /tmp/apt-file-mirror.list
+#
+#MIRRORLIST=/tmp/apt-fast-mirror.list
+
+
 # Download command to use. Temporary download list is designed for aria2. But
 # you can choose another download command or download manager. It has to
 # support following input file syntax (\t is tab character):