This is a modification of ftpmirror.py script that comes with the Python distribution. It needs a Zope DTML method (code given at the end) which produces result like FTP listing of a folder. Using that listing, it makes a local mirror of a Zope folder recursively, over HTTP. I use it for making a static version of my Zope site. I did not find wget or the Zope product Zmirror satisfying all my needs.
If you are serious about using this code, please download the plaint text version instead of trying to copy-paste from this.
# # vsbabu_at_hotmail_dot_com # 03/30/2001 # # this is derived from FTP mirror script available with Python distribution # mirrors a zope site to local folder # see towards the end, the code for the dtml method # # 10/18/2001 - uses 00ignore.txt to ignore a subfolder # this is useful for preventing a folder from going live when you # are working on it. # # 10/29/2001 - added a check to see if by any chance a file was removed in # the local directory (without that being updated in the database files), # force a re-sync # """Mirror a remote Zope subtree into a local directory tree. usage: zopemir [-v] [-q] [-i] [-r] [-s pat] hostname [remotedir [localdir]] -v: verbose -q: quiet -i: interactive mode -r: remove local files/directories no longer pertinent -s pat: skip files matching pattern hostname: remote host remotedir: remote directory (default initial) localdir: local directory (default current) Example: python zopemir http://myzopesite:8080 /mysite/ mylocalfolder/stagingarea/ IMPORTANT NOTE: The hostname does not have / at the end. Instead, the remote folder starts with a / """ # to use, set the top variables here. # then go to the function update_filedata() to set your own # regex search/replace for each file you download import os import sys import time import getopt import string import urllib import urlparse import re from fnmatch import fnmatch # Print usage message and exit def usage(*args): sys.stdout = sys.stderr for msg in args: print msg print __doc__ sys.exit(2) verbose = 1 # 0 for -q, 2 for -v interactive = 0 rmok = 0 host = '' # local mirror information. Useful for speeding up things local_lister = '.zmir' # dtml method in the Zope server for serving down FTP style listing # for a folder. See bottom for code remote_lister = '0_folderfile_listing' #if the listing has this file name, that folder tree is ignored and not downloaded #add this as a DTML Document - content of this is not used at all remote_ignore = '00ignore.txt' # skip current and parent directories and the file where local mirror info is stored skippats = ['.', '..', local_lister] default_filename = 'index.html' # these files will be downloaded regardless of whether they are changed or not force_files = ['index.html'] # the files with the following extensions will be passed through # the function update_filedata() to remove references to wwwstage.tnc:8080 update_files_ext = ['.txt','.htm','.html','.inc'] # Main program: parse command line and start processing def main(): global verbose, interactive, rmok try: opts, args = getopt.getopt(sys.argv[1:], 'i:qrs:v') except getopt.error, msg: usage(msg) for o, a in opts: if o == '-v': verbose = verbose + 1 if o == '-q': verbose = 0 if o == '-i': interactive = 1 if o == '-r': rmok = 1 if o == '-s': skippats.append(a) if not args: usage('hostname missing') host = args[0] remotedir = '' localdir = '' if args[1:]: remotedir = args[1] if args[2:]: localdir = args[2] if args[3:]: usage('too many arguments') # if verbose: print 'Retrieving remote files from %s...' % `host` mirrorsubdir(host+remotedir, localdir) if verbose: print 'OK.' # Core logic: mirror one subdirectory (recursively) def mirrorsubdir(remotedir, localdir): if localdir and not os.path.isdir(localdir): if verbose: print 'Creating local directory', `localdir` try: makedir(localdir) except os.error, msg: print "Failed to establish local directory", `localdir` return infofilename = os.path.join(localdir, local_lister) try: text = open(infofilename, 'r').read() except IOError, msg: text = '{}' try: info = eval(text) except (SyntaxError, NameError): print 'Bad mirror info in %s' % `infofilename` info = {} subdirs = [] listing = [] if verbose: print 'Listing remote directory %s...' % `remotedir` for line in string.split(readURL(host+remotedir+'/'+remote_lister),'\n'): if len(line) > 0: listing.append(line) filesfound = [] #if remote_ignore is a file, return without processing folder if string.find(string.join(listing,' '),remote_ignore) >= 0: if verbose > 1: print 'Ignore folder' return for line in listing: if verbose > 1: print '-->', `line` # Parse, assuming a UNIX listing - your DTML method should be ship shape for this! words = string.split(line, None, 8) if len(words) < 6: if verbose > 1: print 'Skipping short line' continue filename = string.lstrip(words[-1]) infostuff = words[-5:-1] mode = words[0] skip = 0 #if filename == remote_ignore: # if verbose > 1: print 'Ignore folder' # break for pat in skippats: if fnmatch(filename, pat): if verbose > 1: print 'Skip pattern', `pat`, print 'matches', `filename` skip = 1 break if skip: continue # if a directory, remember it and continue if mode[0] == 'd': if verbose > 1: print 'Remembering subdirectory', `filename` subdirs.append(filename) continue # if a file, process it filesfound.append(filename) if info.has_key(filename) and info[filename] == infostuff and filename not in force_files: if not os.path.isfile(os.path.join(localdir,filename)): if verbose > 1: print 'Mismatch between file system and database', `filename` else: if verbose > 1: print 'Already have this version of',`filename` continue fullname = os.path.join(localdir, filename) tempname = os.path.join(localdir, '@'+filename) if interactive: doit = askabout('file', filename, pwd) if not doit: if not info.has_key(filename): info[filename] = 'Not retrieved' continue try: os.unlink(tempname) except os.error: pass try: fp = open(tempname, 'wb') except IOError, msg: print "Can't create %s: %s" % ( `tempname`, str(msg)) continue if verbose: print 'Retrieving %s from %s as %s...' % (`filename`, `remotedir`, `fullname`) if verbose: fp1 = LoggingFile(fp, 1024, sys.stdout) else: fp1 = fp t0 = time.time() file_data = readURL(host+remotedir+'/'+filename) try: if os.path.splitext(filename)[1] in update_files_ext: file_data = update_filedata(file_data) except: pass try: fp1.write(file_data) except: print 'Could not write file data' continue t1 = time.time() bytes = fp.tell() fp.close() if fp1 != fp: fp1.close() try: os.unlink(fullname) except os.error: pass # Ignore the error try: os.rename(tempname, fullname) except os.error, msg: print "Can't rename %s to %s: %s" % (`tempname`, `fullname`, str(msg)) continue info[filename] = infostuff writedict(info, infofilename) if verbose: dt = t1 - t0 kbytes = bytes / 1024.0 print int(round(kbytes)), print 'Kbytes in', print int(round(dt)), print 'seconds', if t1 > t0: print '(~%d Kbytes/sec)' % \ int(round(kbytes/dt),) print # # Remove files from info that are no longer remote deletions = 0 for filename in info.keys(): if filename not in filesfound: if verbose: print "Removing obsolete info entry for", print `filename`, "in", `localdir or "."` del info[filename] deletions = deletions + 1 if deletions: writedict(info, infofilename) # # Remove local files that are no longer in the remote directory try: if not localdir: names = os.listdir(os.curdir) else: names = os.listdir(localdir) except os.error: names = [] for name in names: if name[0] == '.' or info.has_key(name) or name in subdirs: continue skip = 0 for pat in skippats: if fnmatch(name, pat): if verbose > 1: print 'Skip pattern', `pat`, print 'matches', `name` skip = 1 break if skip: continue fullname = os.path.join(localdir, name) if not rmok: if verbose: print 'Local file', `fullname`, print 'is no longer pertinent' continue if verbose: print 'Removing local file/dir', `fullname` remove(fullname) # # Recursively mirror subdirectories for subdir in subdirs: if interactive: doit = askabout('subdirectory', subdir, remotedir) if not doit: continue if verbose: print 'Processing subdirectory', `subdir` localsubdir = os.path.join(localdir, subdir) if verbose > 1: print 'Remote directory now:', `remotedir` print 'Remote cwd', `subdir` print 'Mirroring as', `localsubdir` mirrorsubdir(remotedir+'/'+subdir, localsubdir) if verbose > 1: print 'Remote cwd ..' newpwd = remotedir # Helper to remove a file or directory tree def remove(fullname): if os.path.isdir(fullname) and not os.path.islink(fullname): try: names = os.listdir(fullname) except os.error: names = [] ok = 1 for name in names: if not remove(os.path.join(fullname, name)): ok = 0 if not ok: return 0 try: os.rmdir(fullname) except os.error, msg: print "Can't remove local directory %s: %s" % (`fullname`, str(msg)) return 0 else: try: os.unlink(fullname) except os.error, msg: print "Can't remove local file %s: %s" % (`fullname`, str(msg)) return 0 return 1 # Wrapper around a file for writing to write a hash sign every block. class LoggingFile: def __init__(self, fp, blocksize, outfp): self.fp = fp self.bytes = 0 self.hashes = 0 self.blocksize = blocksize self.outfp = outfp def write(self, data): self.bytes = self.bytes + len(data) hashes = int(self.bytes) / self.blocksize while hashes > self.hashes: self.outfp.write('#') self.outfp.flush() self.hashes = self.hashes + 1 self.fp.write(data) def close(self): self.outfp.write('\n') # Ask permission to download a file. def askabout(filetype, filename, pwd): prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd) while 1: reply = string.lower(string.strip(raw_input(prompt))) if reply in ['y', 'ye', 'yes']: return 1 if reply in ['', 'n', 'no', 'nop', 'nope']: return 0 print 'Please answer yes or no.' # Create a directory if it doesn't exist. Recursively create the # parent directory as well if needed. def makedir(pathname): if os.path.isdir(pathname): return dirname = os.path.dirname(pathname) if dirname: makedir(dirname) os.mkdir(pathname, 0777) # Write a dictionary to a file in a way that can be read back using # rval() but is still somewhat readable (i.e. not a single long line). # Also creates a backup file. def writedict(dict, filename): dir, file = os.path.split(filename) tempname = os.path.join(dir, '@' + file) backup = os.path.join(dir, file + '~') try: os.unlink(backup) except os.error: pass fp = open(tempname, 'w') fp.write('{\n') for key, value in dict.items(): fp.write('%s: %s,\n' % (`key`, `value`)) fp.write('}\n') fp.close() try: os.rename(filename, backup) except os.error: pass os.rename(tempname, filename) # helper function to read the URL data def readURL(url): """Returns the file found in the URL """ f = urllib.urlopen(url) data = f.read() f.close() return data # helper function to do the processing of HTML files def update_filedata(data): """Modifies the hard coding in the data so that it is appropriate do the server change processing ideally this should process the file content to make the links relative. We will just make it relative to root URL (/) """ ##data = re.sub('http://ctx','',data) #data = re.sub('(?<!\.)\','',data) data = re.sub('<base href=.*?>','',data) return data if __name__ == '__main__': main() # 0_folderfile_listing - File Listing ala FTP - used for Slurping ##<dtml-in "objectItems(['Folder','DTML Document','Image','File','Knowledge Document'])"> ##<dtml-if "meta_type=='Folder'">d---------<dtml-else>-rw-rw-rw-</dtml-if> 1 Zope Zope<dtml-try><dtml-var getSize fmt="%13s"><dtml-except> 0</dtml-try> <dtml-var "bobobase_modification_time()" fmt="%b %d %H:%M"> <dtml-var id> ##</dtml-in>
Since you are seeing this, it means that your browser does not support cascading style sheets. Please download and use one of the many browsers that support web standards.