Browse Source

Initial commit

master
Johannes Findeisen 7 months ago
commit
b01babd7ca
8 changed files with 161 additions and 0 deletions
  1. +1
    -0
      README.md
  2. +28
    -0
      alpine.sh
  3. +77
    -0
      arch.sh
  4. +7
    -0
      artix.sh
  5. +4
    -0
      blackarch.sh
  6. +4
    -0
      fdroid.sh
  7. +36
    -0
      mirror.sh
  8. +4
    -0
      openwrt.sh

+ 1
- 0
README.md View File

@@ -0,0 +1 @@
Stuff needed for syncing my mirrors to my mirror host

+ 28
- 0
alpine.sh View File

@@ -0,0 +1,28 @@
#!/usr/bin/env sh

# make sure we never run 2 rsync at the same time
lockfile="/tmp/alpine-mirror.lock"
if [ -z "$flock" ] ; then
exec env flock=1 flock -n $lockfile "$0" "$@"
fi

src=rsync://rsync.alpinelinux.org/alpine/
dest=/data/alpine/

# uncomment this to exclude old v2.x branches
exclude="--exclude v2.* --exclude v3.0 --exclude v3.1 --exclude v3.2 --exclude v3.2 --exclude v3.3 --exclude v3.4 --exclude v3.5 --exclude v3.6 --exclude v3.7 --exclude v3.8 --exclude v3.9 --exclude aarch64/ --exclude armhf/ --exclude armv7/ --exclude ppc64le/ --exclude s390x/ --exclude x86/"

mkdir -p "$dest"
/usr/bin/rsync \
--archive \
--update \
--hard-links \
--delete \
--delete-after \
--delay-updates \
--timeout=600 \
--delete-excluded \
--verbose \
$exclude \
"$src" "$dest"


+ 77
- 0
arch.sh View File

@@ -0,0 +1,77 @@
#!/usr/bin/env bash

# This is a simple mirroring script. To save bandwidth it first checks a
# timestamp via HTTP and only runs rsync when the timestamp differs from the
# local copy. As of 2016, a single rsync run without changes transfers roughly
# 6MiB of data which adds up to roughly 250GiB of traffic per month when rsync
# is run every minute. Performing a simple check via HTTP first can thus save a
# lot of traffic.

# Directory where the repo is stored locally. Example: /srv/repo
target="/data/arch"

# Directory where files are downloaded to before being moved in place.
# This should be on the same filesystem as $target, but not a subdirectory of $target.
# Example: /srv/tmp
tmp="/data/tmp"

# Lockfile path
lock="/tmp/syncrepo.lck"

# If you want to limit the bandwidth used by rsync set this.
# Use 0 to disable the limit.
# The default unit is KiB (see man rsync /--bwlimit for more)
bwlimit=0

# The source URL of the mirror you want to sync from.
# If you are a tier 1 mirror use rsync.archlinux.org, for example like this:
# rsync://rsync.archlinux.org/ftp_tier1
# Otherwise chose a tier 1 mirror from this list and use its rsync URL:
# https://www.archlinux.org/mirrors/
source_url='rsync://mirror.23media.de/archlinux/'

# An HTTP(S) URL pointing to the 'lastupdate' file on your chosen mirror.
# If you are a tier 1 mirror use: http://rsync.archlinux.org/lastupdate
# Otherwise use the HTTP(S) URL from your chosen mirror.
lastupdate_url='http://mirror.23media.de/archlinux/lastupdate'

#### END CONFIG

[ ! -d "${target}" ] && mkdir -p "${target}"
[ ! -d "${tmp}" ] && mkdir -p "${tmp}"

exec 9>"${lock}"
flock -n 9 || exit

rsync_cmd() {
local -a cmd=(rsync -rtlH --safe-links --delete-after --verbose "--timeout=600" "--contimeout=60" -p \
--delay-updates --no-motd "--temp-dir=${tmp}")

# if stty &>/dev/null; then
# cmd+=(-h -v --progress)
# else
# cmd+=(--quiet)
# fi
if ((bwlimit>0)); then
cmd+=("--bwlimit=$bwlimit")
fi

"${cmd[@]}" "$@"
}


# if we are called without a tty (cronjob) only run when there are changes
if ! tty -s && [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then
# keep lastsync file in sync for statistics generated by the Arch Linux website
rsync_cmd "$source_url/lastsync" "$target/lastsync"
exit 0
fi

rsync_cmd \
--exclude='*.links.tar.gz*' \
--exclude='/other' \
--exclude='/sources' \
"${source_url}" \
"${target}"


+ 7
- 0
artix.sh View File

@@ -0,0 +1,7 @@
#!/bin/bash

rsync -auvPH --delete-after "rsync://mirrors.dotsrc.org/artix-linux/" /data/artix/

#rsync -auvPH --delete-after "rsync://orion.artixlinux.org:64873/" /data/artix/repos/
#rsync -auvPH --delete-after "rsync://download.artixlinux.org/iso/" /data/artix/iso/


+ 4
- 0
blackarch.sh View File

@@ -0,0 +1,4 @@
#!/bin/sh

rsync -auvPH --delete-after "rsync://ftp.halifax.rwth-aachen.de/blackarch/" /data/blackarch/


+ 4
- 0
fdroid.sh View File

@@ -0,0 +1,4 @@
#!/bin/sh

rsync -auvPH --delete-after "rsync://rsync.cyberbits.eu/fdroid/" /data/fdroid/fdroid/


+ 36
- 0
mirror.sh View File

@@ -0,0 +1,36 @@
#!/bin/bash

echo "############################"
echo "starting mirror sync"
date
echo "############################"

echo "syncing alpine mirror!"
date
/data/bin/alpine.sh > /data/log/alpine.log
echo "############################"

echo "syncing arch mirror!"
date
/data/bin/arch.sh > /data/log/arch.log
echo "############################"

echo "syncing artix mirror!"
date
/data/bin/artix.sh > /data/log/artix.log
echo "############################"

echo "syncing blackarch mirror!"
date
/data/bin/blackarch.sh > /data/log/blackarch.log
echo "############################"

echo "syncing fdroid mirror!"
date
/data/bin/fdroid.sh > /data/log/fdroid.log
echo "############################"

echo "syncing mirrors finished"
date
echo "############################"


+ 4
- 0
openwrt.sh View File

@@ -0,0 +1,4 @@
#!/bin/sh

rsync -auvPH --delete-after "rsync://downloads.openwrt.org/downloads" /data/openwrt/


Loading…
Cancel
Save