Repository for all the scripts syncing my mirrors
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

78 lines
2.4KB

  1. #!/usr/bin/env bash
  2. # This is a simple mirroring script. To save bandwidth it first checks a
  3. # timestamp via HTTP and only runs rsync when the timestamp differs from the
  4. # local copy. As of 2016, a single rsync run without changes transfers roughly
  5. # 6MiB of data which adds up to roughly 250GiB of traffic per month when rsync
  6. # is run every minute. Performing a simple check via HTTP first can thus save a
  7. # lot of traffic.
  8. # Directory where the repo is stored locally. Example: /srv/repo
  9. target="/data/mirrors/arch"
  10. # Directory where files are downloaded to before being moved in place.
  11. # This should be on the same filesystem as $target, but not a subdirectory of $target.
  12. # Example: /srv/tmp
  13. tmp="/data/tmp"
  14. # Lockfile path
  15. lock="/data/tmp/arch-mirror.lock"
  16. # If you want to limit the bandwidth used by rsync set this.
  17. # Use 0 to disable the limit.
  18. # The default unit is KiB (see man rsync /--bwlimit for more)
  19. bwlimit=0
  20. # The source URL of the mirror you want to sync from.
  21. # If you are a tier 1 mirror use rsync.archlinux.org, for example like this:
  22. # rsync://rsync.archlinux.org/ftp_tier1
  23. # Otherwise chose a tier 1 mirror from this list and use its rsync URL:
  24. # https://www.archlinux.org/mirrors/
  25. source_url='rsync://mirror.23media.de/archlinux/'
  26. # An HTTP(S) URL pointing to the 'lastupdate' file on your chosen mirror.
  27. # If you are a tier 1 mirror use: http://rsync.archlinux.org/lastupdate
  28. # Otherwise use the HTTP(S) URL from your chosen mirror.
  29. lastupdate_url='http://mirror.23media.de/archlinux/lastupdate'
  30. #### END CONFIG
  31. [ ! -d "${target}" ] && mkdir -p "${target}"
  32. [ ! -d "${tmp}" ] && mkdir -p "${tmp}"
  33. exec 9>"${lock}"
  34. flock -n 9 || exit
  35. rsync_cmd() {
  36. local -a cmd=(rsync -rtlH --safe-links --delete-after --verbose "--timeout=600" "--contimeout=60" -p \
  37. --delay-updates --no-motd "--temp-dir=${tmp}")
  38. # if stty &>/dev/null; then
  39. # cmd+=(-h -v --progress)
  40. # else
  41. # cmd+=(--quiet)
  42. # fi
  43. if ((bwlimit>0)); then
  44. cmd+=("--bwlimit=$bwlimit")
  45. fi
  46. "${cmd[@]}" "$@"
  47. }
  48. # if we are called without a tty (cronjob) only run when there are changes
  49. if ! tty -s && [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then
  50. # keep lastsync file in sync for statistics generated by the Arch Linux website
  51. rsync_cmd "$source_url/lastsync" "$target/lastsync"
  52. exit 0
  53. fi
  54. rsync_cmd \
  55. --exclude='*.links.tar.gz*' \
  56. --exclude='/other' \
  57. --exclude='/sources' \
  58. "${source_url}" \
  59. "${target}"