diff --git a/README.md b/README.md index ff9855c..56f1ab9 100644 --- a/README.md +++ b/README.md @@ -187,14 +187,21 @@ Official mirror lists: * Debian: http://www.debian.org/mirror/list * Ubuntu: https://launchpad.net/ubuntu/+archivemirrors -Then add them to whitespace and comma separated list in config file, e.g.: +Then add them to space and/or comma separated list in config file, e.g.: ```sh -MIRRORS=( 'http://deb.debian.org/debian','http://ftp.debian.org/debian, http://ftp2.de.debian.org/debian, http://ftp.de.debian.org/debian, ftp://ftp.uni-kl.de/debian' ) +MIRRORS=( + 'http://deb.debian.org/debian','http://ftp.debian.org/debian, http://ftp2.de.debian.org/debian, http://ftp.de.debian.org/debian ftp://ftp.uni-kl.de/debian' +) ``` +If you are using multiple distributions (such as Ubuntu PPA on Debian or other 3rd-party repoitories), you can simply add one more list into this array of lists: + ```sh -MIRRORS=( 'http://archive.ubuntu.com/ubuntu, http://de.archive.ubuntu.com/ubuntu, http://ftp.halifax.rwth-aachen.de/ubuntu, http://ftp.uni-kl.de/pub/linux/ubuntu, http://mirror.informatik.uni-mannheim.de/pub/linux/distributions/ubuntu/' ) +MIRRORS=( + 'http://deb.debian.org/debian http://ftp.debian.org/debian http://ftp2.de.debian.org/debian http://ftp.de.debian.org/debian ftp://ftp.uni-kl.de/debian' + 'http://archive.ubuntu.com/ubuntu http://de.archive.ubuntu.com/ubuntu http://ftp.halifax.rwth-aachen.de/ubuntu http://ftp.uni-kl.de/pub/linux/ubuntu http://mirror.informatik.uni-mannheim.de/pub/linux/distributions/ubuntu/' +) ``` *NOTE:* To use any mirrors you may have in sources.list or sources.list.d you will need to add them to the apt-fast.conf mirror list as well! @@ -303,3 +310,4 @@ Special thanks * Sergio Silva - test to see if axel is installed, root detection/sudo autorun, lock file check/creation * Waldemar {BOB}{Burnfaker} Wetzel - lockfile improvements, separate config file * maclarke - locking improvements + * artoria2e5 - apt-file-mirror support diff --git a/apt-fast b/apt-fast index 83cdc78..ccae793 100755 --- a/apt-fast +++ b/apt-fast @@ -17,22 +17,32 @@ # Message types are 'normal', 'hint' or 'warning'. Warnings and messages with a # third argument are piped to stderr. msg(){ - msg_options=() + local beginColor msg_options=() fd=1 nl='\n' case "$2" in normal) beginColor="$cGreen";; hint) beginColor="$cBlue";; - warning) beginColor="$cRed";; - question) beginColor="$cRed"; msg_options=(-n);; + warning) beginColor="$cRed" fd=2;; + question) beginColor="$cRed"; nl='';; *) beginColor= ;; esac + if [ -n "$3" ]; then fd=2; fi - if [ -z "$3" ] && [ "$2" != "warning" ]; then - echo -e "${msg_options[@]}" "${aptfast_prefix}${beginColor}$1${endColor}" - else - echo -e "${msg_options[@]}" "${aptfast_prefix}${beginColor}$1${endColor}" >&2 - fi + printf "%s${beginColor}%b${endColor}${nl}" "$(aptfast_prefix)" "$1" >&"$fd" } +# Early bash version check. +if (( BASH_VERSINFO[0] < 4 )) || + (( BASH_VERSINFO[0] == 4 && BASH_VERSINFO[0] < 3 )) +then + printf '%s\n' "This script uses features only available in bash 4.3+: "\ + "associative arrays and namerefs."$'\n'\ + "Please run it with a never version of GNU bash "\ + "(you are running ${BASH_VERSION}.)">&2 + exit 2 +fi + +shopt -s extglob + # Search for known options and decide if root privileges are needed. root=1 # default value: we need root privileges option= @@ -62,49 +72,43 @@ done # we need to preserve all interesting env variables. As this wouldn't be # difficult enough we have to preserve complete env vars (especially if value # ist set (even empty) or not) when changing context (sudo)... +sudo_exports=(DEBUG="$DEBUG") +shadowvars=() # Set a 'random' string to all unset variables. TMP_RANDOM="13979853562951413" -TMP_LCK_FILE="${LCK_FILE-${TMP_RANDOM}}" -TMP_DOWNLOADBEFORE="${DOWNLOADBEFORE-${TMP_RANDOM}}" -TMP__APTMGR="${_APTMGR-${TMP_RANDOM}}" -TMP_APTCACHE="${APTCACHE-${TMP_RANDOM}}" -TMP_DLDIR="${DLDIR-${TMP_RANDOM}}" -TMP_DLLIST="${DLLIST-${TMP_RANDOM}}" -TMP_LISTDIR="${LISTDIR-${TMP_RANDOM}}" -TMP__MAXNUM="${MAXNUM-${TMP_RANDOM}}" -TMP__MAXCONPERSRV="${MAXCONPERSRV-${TMP_RANDOM}}" -TMP__SPLITCON="${SPLITCON-${TMP_RANDOM}}" -TMP__MINSPLITSZ=${MINSPLITSZ-${TMP_RANDOM}} -TMP__PIECEALGO=${PIECEALGO-${TMP_RANDOM}} -TMP_aptfast_prefix="${aptfast_prefix-${TMP_RANDOM}}" -TMP_APT_FAST_TIMEOUT="${APT_FAST_TIMEOUT-${TMP_RANDOM}}" -TMP_VERBOSE_OUTPUT="${VERBOSE_OUTPUT-${TMP_RANDOM}}" -TMP_ftp_proxy="${ftp_proxy-${TMP_RANDOM}}" -TMP_http_proxy="${http_proxy-${TMP_RANDOM}}" -TMP_https_proxy="${https_proxy-${TMP_RANDOM}}" +shadowify() { + # I heard that people call this "variable shadowing." + declare -n this="$1" tmpthis="TMP_$1" + tmpthis="${this-${TMP_RANDOM}}" + sudo_exports+=("TMP_$1=$tmpthis") + shadowvars+=("$1") +} + +shadowify LCK_FILE +shadowify DOWNLOADBEFORE +shadowify _APTMGR +shadowify APTCACHE +shadowify DLDIR +shadowify DLLIST +shadowify MIRRORLIST +shadowify LISTDIR +shadowify _MAXNUM +shadowify _MAXCONPERSRV +shadowify _SPLITCON +shadowify _MINSPLITSZ +shadowify _SPLITCON +shadowify _MINSPLITSZ +shadowify _PIECEALGO +shadowify APT_FAST_TIMEOUT +shadowify VERBOSE_OUTPUT +shadowify ftp_proxy +shadowify http_proxy +shadowify https_proxy # Check for proper privileges. # Call explicitly with environment variables to get them into root conext. if [ "$root" = 1 ] && [ "$UID" != 0 ]; then - exec sudo DEBUG="$DEBUG" \ - LCK_FILE="$TMP_LCK_FILE" \ - DOWNLOADBEFORE="$TMP_DOWNLOADBEFORE" \ - _APTMGR="$TMP__APTMGR" \ - APTCACHE="$TMP_APTCACHE" \ - DLDIR="$TMP_DLDIR" \ - DLLIST="$TMP_DLLIST" \ - LISTDIR="$TMP_LISTDIR" \ - _MAXNUM="$TMP__MAXNUM" \ - _MAXCONPERSRV="$TMP__MAXCONPERSRV" \ - _SPLITCON="$TMP__SPLITCON" \ - _MINSPLITSZ="$TMP__MINSPLITSZ" \ - _PIECEALGO="$TMP__PIECEALGO" \ - aptfast_prefix="$TMP_aptfast_prefix" \ - APT_FAST_TIMEOUT="$TMP_APT_FAST_TIMEOUT" \ - VERBOSE_OUTPUT="$TMP_VERBOSE_OUTPUT" \ - ftp_proxy="$TMP_ftp_proxy" \ - http_proxy="$TMP_http_proxy" \ - https_proxy="$TMP_https_proxy" \ + exec sudo "${sudo_exports[@]}" "$0" "$@" fi @@ -130,6 +134,7 @@ fi # Currently not needed. eval "$(apt-config shell LISTDIR Dir::State::lists/d)" DLLIST="/tmp/apt-fast.list" +MIRRORLIST="/tmp/apt-fast-mirror.list" _MAXNUM=5 _MAXCONPERSRV=10 _SPLITCON=8 @@ -137,8 +142,7 @@ _MINSPLITSZ="1M" _PIECEALGO="default" # Prefix in front of apt-fast output: -aptfast_prefix= -# aptfast_prefix="$(date '+%b %_d %T.%N') apt-fast: " +# aptfast_prefix(){ printf '%s' "$(date '+%b %_d %T.%N') apt-fast: "; } # Set color variables. cGreen='\e[0;32m' @@ -175,24 +179,15 @@ https_proxy= # Now overwrite with preserved values if values were set before (compare with # 'random' string). -[ "$TMP_LCK_FILE" = "$TMP_RANDOM" ] || LCK_FILE="$TMP_LCK_FILE" -[ "$TMP_DOWNLOADBEFORE" = "$TMP_RANDOM" ] || DOWNLOADBEFORE="$TMP_DOWNLOADBEFORE" -[ "$TMP__APTMGR" = "$TMP_RANDOM" ] || _APTMGR="$TMP__APTMGR" -[ "$TMP_APTCACHE" = "$TMP_RANDOM" ] || APTCACHE="$TMP_APTCACHE" -[ "$TMP_DLDIR" = "$TMP_RANDOM" ] || DLDIR="$TMP_DLDIR" -[ "$TMP_DLLIST" = "$TMP_RANDOM" ] || DLLIST="$TMP_DLLIST" -[ "$TMP_LISTDIR" = "$TMP_RANDOM" ] || LISTDIR="$TMP_LISTDIR" -[ "$TMP__MAXNUM" = "$TMP_RANDOM" ] || _MAXNUM="$TMP__MAXNUM" -[ "$TMP__MAXCONPERSRV" = "$TMP_RANDOM" ] || _MAXCONPERSRV="$TMP__MAXCONPERSRV" -[ "$TMP__SPLITCON" = "$TMP_RANDOM" ] || _SPLITCON="$TMP__SPLITCON" -[ "$TMP__MINSPLITSZ" = "$TMP_RANDOM" ] || _MINSPLITSZ="$TMP__MINSPLITSZ" -[ "$TMP__PIECEALGO" = "$TMP_RANDOM" ] || _PIECEALGO="$TMP__PIECEALGO" -[ "$TMP_aptfast_prefix" = "$TMP_RANDOM" ] || aptfast_prefix="$TMP_aptfast_prefix" -[ "$TMP_APT_FAST_TIMEOUT" = "$TMP_RANDOM" ] || APT_FAST_TIMEOUT="$TMP_APT_FAST_TIMEOUT" -[ "$TMP_VERBOSE_OUTPUT" = "$TMP_RANDOM" ] || VERBOSE_OUTPUT="$TMP_VERBOSE_OUTPUT" -[ "$TMP_ftp_proxy" = "$TMP_RANDOM" ] || ftp_proxy="$TMP_ftp_proxy" -[ "$TMP_http_proxy" = "$TMP_RANDOM" ] || http_proxy="$TMP_http_proxy" -[ "$TMP_https_proxy" = "$TMP_RANDOM" ] || https_proxy="$TMP_https_proxy" +for i in "${shadowvars[@]}" +do + declare -n thisvar="$i" tempthisvar="TMP_$i" + if [[ $tempthisvar != $TMP_RANDOM ]] + then + thisvar=${tempthisvar} + fi +done +unset -n thisvar tempthisvar # Disable colors if not executed in terminal. @@ -201,8 +196,17 @@ if [ ! -t 1 ]; then cRed= cBlue= endColor= - #FIXME: Time not updated. - [ -z "$aptfast_prefix" ] && aptfast_prefix="[apt-fast $(date +"%T")]" +fi + +# Define the (logging) prefix if it is never defined. +if ! declare -f aptfast_prefix 2>/dev/null; then + if [ -t 1 ]; then + # The TTY user has a clock on the wall. No need. + aptfast_prefix(){ true; } + else + # %(%T)T is like $(date +%T), but built into printf. + aptfast_prefix(){ printf '[apt-fast %(%T)T] '; } + fi fi @@ -238,18 +242,20 @@ _remove_lock() # Move download file away so missing permissions won't stop usage. CLEANUP_STATE=0 -cleanup_dllist() +cleanup_list() { - if [ -f "$DLLIST" ] + if [ ! -f "$1" ] then - if ! mv -- "$DLLIST{,.old}" 2>/dev/null - then - if ! rm -f -- "$DLLIST" 2>/dev/null - then - msg "Could not clean up download list file." "warning" - CLEANUP_STATE=1 - fi - fi + return + fi + + if ! { + mv -- "$1{,.old}" 2>/dev/null || + rm -f -- "$1" 2>/dev/null + } + then + msg "Could not clean up $2 list file." "warning" + CLEANUP_STATE=1 fi } @@ -257,7 +263,7 @@ cleanup_aptfast() { local last_exit_code=$? [ "$CLEANUP_STATE" -eq 0 ] && CLEANUP_STATE=$last_exit_code - cleanup_dllist + cleanup_list "$DLLIST" download _remove_lock } @@ -273,8 +279,55 @@ urldecode() printf '%b' "${1//%/\\x}" } +# Maps a mirror uri to the first entry of that list, so get_mirrors can take it. +# We definitely don't want to resolve those more than once each run... +declare -A mirror_uri_map + +# Handles a mirror URI, with caching. Parses the given list into MIRRORS. +handle_mirror() +{ + # The most recognizable split point is the "pool" thing. This + # will be apt-ftparchive-specific, but I suspect the transport is doing that + # itself. + local head="${1%%/pool/*}" body="/pool/${1#*/pool/}" + + if ! [[ ${mirror_uri_map[$head]} ]]; then + local protocol="${head%%:*}" path="${head#*:}" + local mirrorfile + + case $protocol in + (mirror+file) + mirrorfile=$path;; + (*) + # Well, gotta fetch... + if [[ $protocol == mirror ]]; then protocol=mirror+http; fi + aria2c -q -d "${MIRRORLIST%/*}" -o "${MIRRORLIST##*/}" -- "${protocol#mirror+}:${path}" + mirrorfile="${MIRRORLIST}";; + esac + + local mirrors + mapfile -t mirrors < "$mirrorfile" + [ -f "MIRRORLIST" ] && rm -f "$MIRRORLIST" + # This is a TAB. The examples say the URI ends at the TAB. We extract that. + mirrors=("${mirrors[@]%% *}") + # Normalize the slashes. (s@/*$@/@g) + mirrors=("${mirrors[@]/%*(\/)/\/}") + # get_mirrors says spaces are fine... no local IFS=, needed here. + MIRRORS+=("${mirrors[*]}") + mirror_uri_map[$head]=${mirrors[0]} + fi + + printf '%s\n' "${mirror_uri_map[$head]}$body" +} + # Check if mirrors are available. And if so add all mirrors to download list. get_mirrors(){ + local mirrors + # Handle apt-transport-mirror + if [[ $1 == mirror* ]]; then + set -- "$(handle_mirror "$1")" + fi + # Check all mirror lists. for mirrorstr in "${MIRRORS[@]}"; do # Build mirrors array from comma separated string. @@ -415,7 +468,8 @@ get_uris(){ fi echo " out=$filename" } >> "$DLLIST" - done <<<"$(echo "$uris_full" | grep -E "^'(http(s|)|(s|)ftp)://")" + done <<<"$(<<<"$uris_full" grep -E "^'((mirror\+)?((http(s|)|(s|)ftp)|mirror)://|mirror\+file:/)")" + #' #cat "$DLLIST" #LCK_RM @@ -434,7 +488,7 @@ display_downloadfile(){ pkg="$(echo "$line" | cut -d' ' -f1)" ver="$(echo "$line" | cut -d' ' -f2)" size="$(echo "$line" | cut -d' ' -f3)" - printf '%s%-40s %-20s %10s\n' "$aptfast_prefix" "$pkg" "$ver" "$size" + printf '%s%-40s %-20s %10s\n' "$(aptfast_prefix)" "$pkg" "$ver" "$size" done <<<"$(echo -e "$DOWNLOAD_DISPLAY" | sort "${DISPLAY_SORT_OPTIONS[@]}")" fi msg "Download size: $(echo "$DOWNLOAD_SIZE" | numfmt --to=iec-i --suffix=B)" "normal" diff --git a/apt-fast.conf b/apt-fast.conf index f38bbe3..1bdcf1f 100644 --- a/apt-fast.conf +++ b/apt-fast.conf @@ -1,3 +1,4 @@ +# -*- mode: shell -*- vi: set ft=sh ################################################################### # CONFIGURATION OPTIONS ################################################################### @@ -28,7 +29,7 @@ # Ubuntu: https://launchpad.net/ubuntu/+archivemirrors # # It is required to add mirrors in the sources.list to this array as well, so -# apt-fast can destinguish between different distributions. +# apt-fast can distinguish between different distributions. # # Examples: # @@ -39,8 +40,10 @@ # deb http://de.archive.ubuntu.com/ubuntu/ bionic main universe # # apt-fast.conf: -# MIRRORS=( 'http://deb.debian.org/debian','http://ftp.debian.org/debian,http://ftp2.de.debian.org/debian,http://ftp.de.debian.org/debian,ftp://ftp.uni-kl.de/debian' -# 'http://archive.ubuntu.com/ubuntu,http://de.archive.ubuntu.com/ubuntu,http://ftp.halifax.rwth-aachen.de/ubuntu,http://ftp.uni-kl.de/pub/linux/ubuntu,http://mirror.informatik.uni-mannheim.de/pub/linux/distributions/ubuntu/' ) +# MIRRORS=( +# 'http://deb.debian.org/debian','http://ftp.debian.org/debian,http://ftp2.de.debian.org/debian,http://ftp.de.debian.org/debian,ftp://ftp.uni-kl.de/debian' +# 'http://archive.ubuntu.com/ubuntu,http://de.archive.ubuntu.com/ubuntu,http://ftp.halifax.rwth-aachen.de/ubuntu,http://ftp.uni-kl.de/pub/linux/ubuntu,http://mirror.informatik.uni-mannheim.de/pub/linux/distributions/ubuntu/' +# ) # # # Single distribution: @@ -101,6 +104,14 @@ #DLLIST=/tmp/apt-fast.list +# Mirror listfile +# This determines where temporary resolutions for the mirror:// url is stored. +# +# Default: /tmp/apt-file-mirror.list +# +#MIRRORLIST=/tmp/apt-fast-mirror.list + + # Download command to use. Temporary download list is designed for aria2. But # you can choose another download command or download manager. It has to # support following input file syntax (\t is tab character):