#!/bin/sh
# this file:
# http://www.math.fu-berlin.de/~guckes/vim/script/webmirror.wget.txt
# copy:                 http://www.vim.org/script/webmirror.wget.txt

# --------------------------------------------------------------------
# Purpose of this script:
# This script mirrors the webpages
# about the editor "Vim" ("Vi IMproved").
# The original website is for the pages is
# http://www.math.fu-berlin.de/~guckes/vim/
# but you can adjust this script easily to
# mirror yet another mirror of the pages.  :-)
# This script is also designed to send
# the resulting log info to some address.
# --------------------------------------------------------------------

# --------------------------------------------------------------------
# Script History:
#
# [2001-04-13] Felipe Contreras fcont(at)matem.unam.mx
# Added support for fetching pages with "wget".
#
# [2000-03-26] Stefan 'Sec' Zehl sec(at)vim.org
# Moved to the new Webserver.
#
# [1998-12-08] Stefan 'Sec' Zehl sec(at)vim.org
# Included suggestions from Sergei Laskavy.
#
# [1998-06-25] Stefan 'Sec' Zehl sec(at)vim.org
# First version.
# --------------------------------------------------------------------

# --------------------------------------------------------------------
# See Also:
# wget mirroring tool: http://sunsite.dk/wget/
# --------------------------------------------------------------------

###################################################
# Instructions:  Verify and modify the following: #
###################################################

# Default locations for various things:

# Chose between use of "wget" or "webcopy"
# by uncommenting one of the two lines below:
# The location of the "wget" command:
 [ -z "$WGET" ]    && WGET=/usr/local/etc/httpd/adm/bin/wget
# The location of the "webcopy" command:
#[ -z "$WEBCOPY" ] && WEBCOPY=/tmp/webcopy

# The location of the "mail" command:
[ -z "$MAILX" ]   && MAILX=/usr/sbin/Mail
# NOTE: This program MUST understand the option "-s"!

#if you use webcopy also care for the following:
# The location of the "perl" command:
[ -z "$PERL" ]    && PERL=/usr/sbin/perl
# perl version 5 recommended!

# Your (maintainer's) e-mail address:
[ -z "$ADDRESS" ] && ADDRESS=fhca

# wget's log file
[ -z "$LOG" ] && LOG=/usr/local/etc/httpd/adm/out/mirror.vim.out

# WWW source: the web address of the web source files
# ("where do you get the files FROM?")
[ -z "$WWWsource" ] && WWWsource=http://www.math.fu-berlin.de/~guckes/vim/

# WWW destination: where do you want the files stored LOCALLY?
[ -z "$WWWdest" ] && WWWdest=/usr/local/etc/httpd/htdocs/vim

# FTP source: analogous for ftp
[ -z "$FTPsource" ] && FTPsource=ftp://vim.ftp.fu-berlin.de/

# FTP destination (assuming you've set up an ftp server (see "man ftpd")
[ -z "$FTPdest" ] && FTPdest=/home/ftp/pub/vim

# Hint: /usr/local/etc/httpd/htdocs/ contains (for me) the server docs
# but $WWWdest contains Vim! virtual host homepage.
#
# To define a "virtual host" so you get the following to be "homepage" for
# www.xx.vim.org insert the following lines in {/etc/httpd/conf/}httpd.conf
# and restart Apache web server...
#   <VirtualHost www.xx.vim.org>
#   ServerAdmin  you@your.domain
#   DocumentRoot /usr/local/etc/httpd/htdocs/vim  #same as WWWdest
#   ServerName        www.xx.vim.org
#   ErrorLog     logs/www.xx.vim.org-error_log
#   TransferLog  logs/www.xx.vim.org-access_log
#   </VirtualHost>

########################
# End of modifications #
########################

# Mirroring the whole tree  (from the source URL)
# via wget/webcopy.  wget flags used:
# -nH <-- no subdir named after host
# -nv <-- no verbose (simplified output only)
# -m  <-- mirror mode (recursive, timestamped, etc...)
# -np <-- don't mirror outside indicated subdir!  Damn! this should be default!
# --cut-dirs=2  <-- do not create URL subdirs locally (eg ~guckes/vim/)
# -P ... <-- storage area
# the URL
# -o ... <-- log file
WGFLAGS="-nH -nv -m -np"        #some more?

# Do things...
# If WEBCOPY was defined and destiny dir exists do the following...
[ "$WEBCOPY" -a -d $WWWdest ] && (cd $WWWdest;$WEBCOPY $@ -r $WWWsource >$LOG)

# Same for WGET
[ "$WGET" -a -d $WWWdest ] && $WGET $WGFLAGS --cut-dirs=2 $WWWsource -P $WWWdest -o $LOG

# Same for FTP
[ "$WGET" -a -d $FTPdest ] && $WGET $WGFLAGS $FTPsource -P $FTPdest -a $LOG
[ "$WEBCOPY" -a -d $FTPdest ] && (cd $FTPdest;$WEBCOPY $@ -r $FTPsource >$LOG)

# How much data was it this time ?
[ "$WEBCOPY" ] && kbytes=`(echo 0;cat $LOG |awk '$2 == "bytes"{print $1 "+"}';echo '1k 1024 / p')| dc`
[ "$WGET" ] && kbytes=`egrep "Downloaded.*bytes" $LOG`

# Some statistics (for webcopy), and the logfile. Needs perl.
[ "$WEBCOPY" ] && (
echo "The vim-Webmirror transfered $kbytes Kbytes this time."
echo ""
echo "Webserver responses:"
echo ""
<$LOG $PERL -ne 'chop;if ($_ =~ s/^Status: //) { $hash{$_}++;} END { foreach $var (sort {$hash{$a} <=> $hash{$b}} keys %hash) {printf "%3d: $var\n",$hash{$var}}}'
echo ""
echo "Log:"
cat $LOG )|$MAILX -s "webcopy log" $ADDRESS

# This is for wget, guess it can improve
[ "$WGET" ] && (echo $kbytes; cat $LOG)|$MAILX -s "wget log" $ADDRESS

# EOF