This script comes with the standard Python distribution. I found this incredibly useful. Only (silly) modification I've added is to mirror from an NT IIS ftp server.
#! /usr/bin/python
"""Mirror a remote ftp subtree into a local directory tree.
usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat] [-t]
[-l username [-p passwd [-a account]]]
hostname [remotedir [localdir]]
-v: verbose
-q: quiet
-i: interactive mode
-m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o')
-t: NT/DOS server (implies a different kind of listing)
-n: don't log in
-r: remove local files/directories no longer pertinent
-l username [-p passwd [-a account]]: login info (default anonymous ftp)
-s pat: skip files matching pattern
hostname: remote host
remotedir: remote directory (default initial)
localdir: local directory (default current)
"""
import os
import sys
import time
import getopt
import string
import ftplib
from fnmatch import fnmatch
# Print usage message and exit
def usage(*args):
sys.stdout = sys.stderr
for msg in args: print msg
print __doc__
sys.exit(2)
verbose = 1 # 0 for -q, 2 for -v
interactive = 0
mac = 0
rmok = 0
nologin = 0
ntdos = 0
skippats = ['.', '..', '.mir']
# Main program: parse command line and start processing
def main():
global verbose, interactive, mac, rmok, nologin, ntdos
try:
opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrts:v')
except getopt.error, msg:
usage(msg)
login = ''
passwd = ''
account = ''
for o, a in opts:
if o == '-l': login = a
if o == '-p': passwd = a
if o == '-a': account = a
if o == '-v': verbose = verbose + 1
if o == '-q': verbose = 0
if o == '-i': interactive = 1
if o == '-m': mac = 1; nologin = 1; skippats.append('*.o')
if o == '-n': nologin = 1
if o == '-r': rmok = 1
if o == '-t': ntdos = 1
if o == '-s': skippats.append(a)
if not args: usage('hostname missing')
host = args[0]
remotedir = ''
localdir = ''
if args[1:]:
remotedir = args[1]
if args[2:]:
localdir = args[2]
if args[3:]: usage('too many arguments')
#
f = ftplib.FTP()
if verbose: print 'Connecting to %s...' % `host`
f.connect(host)
if not nologin:
if verbose:
print 'Logging in as %s...' % `login or 'anonymous'`
f.login(login, passwd, account)
if verbose: print 'OK.'
pwd = f.pwd()
if verbose > 1: print 'PWD =', `pwd`
if remotedir:
if verbose > 1: print 'cwd(%s)' % `remotedir`
f.cwd(remotedir)
if verbose > 1: print 'OK.'
pwd = f.pwd()
if verbose > 1: print 'PWD =', `pwd`
#
mirrorsubdir(f, localdir)
# Core logic: mirror one subdirectory (recursively)
def mirrorsubdir(f, localdir):
pwd = f.pwd()
if localdir and not os.path.isdir(localdir):
if verbose: print 'Creating local directory', `localdir`
try:
makedir(localdir)
except os.error, msg:
print "Failed to establish local directory", `localdir`
return
infofilename = os.path.join(localdir, '.mir')
try:
text = open(infofilename, 'r').read()
except IOError, msg:
text = '{}'
try:
info = eval(text)
except (SyntaxError, NameError):
print 'Bad mirror info in %s' % `infofilename`
info = {}
subdirs = []
listing = []
if verbose: print 'Listing remote directory %s...' % `pwd`
f.retrlines('LIST', listing.append)
filesfound = []
for line in listing:
if verbose > 1: print '-->', `line`
if mac:
# Mac listing has just filenames;
# trailing / means subdirectory
filename = string.strip(line)
mode = '-'
if filename[-1:] == '/':
filename = filename[:-1]
mode = 'd'
infostuff = ''
elif ntdos:
#NT IIS server has only 4 fields
words = string.split(line, None, 6)
if len(words) < 4:
if verbose > 1: print 'Skipping short line'
continue
filename = string.lstrip(words[-1])
infostuff = words[-5:-1]
mode = words[2]
if mode == '<DIR>': mode = 'drwx'
else:
# Parse, assuming a UNIX listing
words = string.split(line, None, 8)
if len(words) < 6:
if verbose > 1: print 'Skipping short line'
continue
filename = string.lstrip(words[-1])
i = string.find(filename, " -> ")
if i >= 0:
# words[0] had better start with 'l'...
if verbose > 1:
print 'Found symbolic link %s' % `filename`
linkto = filename[i+4:]
filename = filename[:i]
infostuff = words[-5:-1]
mode = words[0]
skip = 0
for pat in skippats:
if fnmatch(filename, pat):
if verbose > 1:
print 'Skip pattern', `pat`,
print 'matches', `filename`
skip = 1
break
if skip:
continue
if mode[0] == 'd':
if verbose > 1:
print 'Remembering subdirectory', `filename`
subdirs.append(filename)
continue
filesfound.append(filename)
if info.has_key(filename) and info[filename] == infostuff:
if verbose > 1:
print 'Already have this version of',`filename`
continue
fullname = os.path.join(localdir, filename)
tempname = os.path.join(localdir, '@'+filename)
if interactive:
doit = askabout('file', filename, pwd)
if not doit:
if not info.has_key(filename):
info[filename] = 'Not retrieved'
continue
try:
os.unlink(tempname)
except os.error:
pass
if mode[0] == 'l':
if verbose:
print "Creating symlink %s -> %s" % (
`filename`, `linkto`)
try:
os.symlink(linkto, tempname)
except IOError, msg:
print "Can't create %s: %s" % (
`tempname`, str(msg))
continue
else:
try:
fp = open(tempname, 'wb')
except IOError, msg:
print "Can't create %s: %s" % (
`tempname`, str(msg))
continue
if verbose:
print 'Retrieving %s from %s as %s...' % \
(`filename`, `pwd`, `fullname`)
if verbose:
fp1 = LoggingFile(fp, 1024, sys.stdout)
else:
fp1 = fp
t0 = time.time()
try:
f.retrbinary('RETR ' + filename,
fp1.write, 8*1024)
except ftplib.error_perm, msg:
print msg
t1 = time.time()
bytes = fp.tell()
fp.close()
if fp1 != fp:
fp1.close()
try:
os.unlink(fullname)
except os.error:
pass # Ignore the error
try:
os.rename(tempname, fullname)
except os.error, msg:
print "Can't rename %s to %s: %s" % (`tempname`,
`fullname`,
str(msg))
continue
info[filename] = infostuff
writedict(info, infofilename)
if verbose and mode[0] != 'l':
dt = t1 - t0
kbytes = bytes / 1024.0
print int(round(kbytes)),
print 'Kbytes in',
print int(round(dt)),
print 'seconds',
if t1 > t0:
print '(~%d Kbytes/sec)' % \
int(round(kbytes/dt),)
print
#
# Remove files from info that are no longer remote
deletions = 0
for filename in info.keys():
if filename not in filesfound:
if verbose:
print "Removing obsolete info entry for",
print `filename`, "in", `localdir or "."`
del info[filename]
deletions = deletions + 1
if deletions:
writedict(info, infofilename)
#
# Remove local files that are no longer in the remote directory
try:
if not localdir: names = os.listdir(os.curdir)
else: names = os.listdir(localdir)
except os.error:
names = []
for name in names:
if name[0] == '.' or info.has_key(name) or name in subdirs:
continue
skip = 0
for pat in skippats:
if fnmatch(name, pat):
if verbose > 1:
print 'Skip pattern', `pat`,
print 'matches', `name`
skip = 1
break
if skip:
continue
fullname = os.path.join(localdir, name)
if not rmok:
if verbose:
print 'Local file', `fullname`,
print 'is no longer pertinent'
continue
if verbose: print 'Removing local file/dir', `fullname`
remove(fullname)
#
# Recursively mirror subdirectories
for subdir in subdirs:
if interactive:
doit = askabout('subdirectory', subdir, pwd)
if not doit: continue
if verbose: print 'Processing subdirectory', `subdir`
localsubdir = os.path.join(localdir, subdir)
pwd = f.pwd()
if verbose > 1:
print 'Remote directory now:', `pwd`
print 'Remote cwd', `subdir`
try:
f.cwd(subdir)
except ftplib.error_perm, msg:
print "Can't chdir to", `subdir`, ":", `msg`
else:
if verbose: print 'Mirroring as', `localsubdir`
mirrorsubdir(f, localsubdir)
if verbose > 1: print 'Remote cwd ..'
f.cwd('..')
newpwd = f.pwd()
if newpwd != pwd:
print 'Ended up in wrong directory after cd + cd ..'
print 'Giving up now.'
break
else:
if verbose > 1: print 'OK.'
# Helper to remove a file or directory tree
def remove(fullname):
if os.path.isdir(fullname) and not os.path.islink(fullname):
try:
names = os.listdir(fullname)
except os.error:
names = []
ok = 1
for name in names:
if not remove(os.path.join(fullname, name)):
ok = 0
if not ok:
return 0
try:
os.rmdir(fullname)
except os.error, msg:
print "Can't remove local directory %s: %s" % \
(`fullname`, str(msg))
return 0
else:
try:
os.unlink(fullname)
except os.error, msg:
print "Can't remove local file %s: %s" % \
(`fullname`, str(msg))
return 0
return 1
# Wrapper around a file for writing to write a hash sign every block.
class LoggingFile:
def __init__(self, fp, blocksize, outfp):
self.fp = fp
self.bytes = 0
self.hashes = 0
self.blocksize = blocksize
self.outfp = outfp
def write(self, data):
self.bytes = self.bytes + len(data)
hashes = int(self.bytes) / self.blocksize
while hashes > self.hashes:
self.outfp.write('#')
self.outfp.flush()
self.hashes = self.hashes + 1
self.fp.write(data)
def close(self):
self.outfp.write('\n')
# Ask permission to download a file.
def askabout(filetype, filename, pwd):
prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd)
while 1:
reply = string.lower(string.strip(raw_input(prompt)))
if reply in ['y', 'ye', 'yes']:
return 1
if reply in ['', 'n', 'no', 'nop', 'nope']:
return 0
print 'Please answer yes or no.'
# Create a directory if it doesn't exist. Recursively create the
# parent directory as well if needed.
def makedir(pathname):
if os.path.isdir(pathname):
return
dirname = os.path.dirname(pathname)
if dirname: makedir(dirname)
os.mkdir(pathname, 0777)
# Write a dictionary to a file in a way that can be read back using
# rval() but is still somewhat readable (i.e. not a single long line).
# Also creates a backup file.
def writedict(dict, filename):
dir, file = os.path.split(filename)
tempname = os.path.join(dir, '@' + file)
backup = os.path.join(dir, file + '~')
try:
os.unlink(backup)
except os.error:
pass
fp = open(tempname, 'w')
fp.write('{\n')
for key, value in dict.items():
fp.write('%s: %s,\n' % (`key`, `value`))
fp.write('}\n')
fp.close()
try:
os.rename(filename, backup)
except os.error:
pass
os.rename(tempname, filename)
main()
Since you are seeing this, it means that your browser does not support cascading style sheets. Please download and use one of the many browsers that support web standards.