[Mageia-dev] Distrib-coffee big failure of the century

[Barry Jackson]

On 30/10/12 21:49, Barry Jackson wrote:
> On 30/10/12 21:29, Johnny A. Solbu wrote:
>> On Tuesday 30 October 2012 22:21, Barry Jackson wrote:
>>> I will set --max-delete=500 locally in future.
>>>
>>> Any better ideas?
>>
>> I always run rsync manually twice. The first run I use «-n» to make
>> sure my mirror isn't wiped, and then run rsync normally.
>>
>
> I run it from a cron job via a script, so I could parse the output of a
> dry run and count deletes for example, then let it decide whether to run
> or not.
> Thanks - worth investigating :)
>
I wrote the attached script. It checks several mirrors until it finds 
one that has the normal number of files and does not fail for any other 
reason.

Currently I have distrib-coffee at the top of the possible mirror list, 
but it is skipped because it only has a fraction of the files yet.

It creates a couple of logs and runs as a cron job (or manually).

If you feel inclined to use it please feel free to suggest any improvements.

I certainly feel safer now.

Barry
-------------- next part --------------
#!/bin/bash
# ~/cronsync
# Run with e.g. crontab:- 30 * * * *	$HOME/cronsync
# Set using crontab -e
###################
# List of mirrors in preference order to use:-
mymirrors=( \
"rsync://distrib-coffee.ipsl.jussieu.fr:/pub/linux/Mageia/distrib" \
"rsync://ftp.LinuxCabal.org/Mageia/distrib" \
"rsync://ftp.acc.umu.se/mirror/mageia/distrib" \
"rsync://mirrors.kernel.org:/mirrors/mageia/distrib" \
"rsync://mageia.c3sl.ufpr.br/mageia/distrib" \
)
# Set limit for loss of # of files on server before it is skipped
allowdropfiles=100
###################
# Personal config:-
myexcludes="--exclude=debug --exclude=SRPMS --exclude=barjac"
mydestination="/zmrepo/pub/linux/Mageia/"
###################

# Get minimum file count for mirror (set at allowdropfiles files less than the last actual file count)
[[ -f .cronsync ]] || echo "140000" > $HOME/.cronsync
minfiles=$(cat .cronsync)
((actualfiles = $minfiles + $allowdropfiles))
# Check a mirror
chk_mirror()
{
echo "Checking $1 ..."
chkout=$(rsync -rlptgoDhHSn \
--stats \
--delete-after \
--delete \
--delete-excluded \
--protect-args \
$myexcludes \
"$1" \
"$mydestination" | grep "Number of files:" )
status1="$?"
files=$(echo $chkout | cut -d' ' -f4)
[[ $files -lt $minfiles ]] && echo "$(date +%d/%m/%Y-%H:%M): Only $files/$actualfiles files in $1 !" | tee -a $HOME/cronsync_error.log
[[ $status1 > 0 ]] && echo "$(date +%d/%m/%Y-%H:%M): Error: $status1 while checking $1" | tee -a $HOME/cronsync_error.log
([[ $status1 = 0 ]] && [[ $files -gt $minfiles ]]) || return 1
((chkcount = $files - $allowdropfiles))
echo $chkcount > $HOME/.cronsync
return 0
}

# Loop through mirror list checking until one succeeds
find_good_mirror()
{
for mirr in ${mymirrors[@]}; do
chk_mirror $mirr
[[ $? = 0 ]] && { echo "Found good mirror - $mirr"; return 0; }
done
return 1
}

#=======================Main Program starts here=====================

# Check if rsync is already running (maybe last cron sync taking for ever?)
ps aux | grep -q [r]sync
if [[ $? > 0 ]]; then

# Find a good mirror from the list (with files in it)
find_good_mirror || { echo "$(date +%d/%m/%Y-%H:%M) Can't find a good mirror - aborting" | tee -a $HOME/cronsync_error.log; exit 1; }

echo "Syncing from $mirr ..."

# Live run with --max-delete set to 1000 just in case ;)
rsync -rlptgoDhHS \
--progress \
--stats \
--delete-after \
--delete \
--max-delete=1000 \
--delete-excluded \
--protect-args \
$myexcludes \
"$mirr" \
"$mydestination" | tee $HOME/cronsync_last.log

status=$?
[[ $status = 0 ]] && { echo "Update complete"; exit 0; }
[[ $status > 0 ]] && { echo "Live run error - $(date +%d/%m/%Y-%H:%M) = $status" >> $HOME/cronsync_error.log; exit 1; }
else
echo "rsync already running, skipping update - $(date +%d/%m/%Y-%H:%M)" >> $HOME/cronsync_error.log
exit 1
fi