#
# vsbabu_at_hotmail_dot_com
# 03/30/2001
#
# this is derived from FTP mirror script available with Python distribution
# mirrors a zope site to local folder
# see towards the end, the code for the dtml method
#


# 10/18/2001 - uses 00ignore.txt to ignore a subfolder
# this is useful for preventing a folder from going live when you
# are working on it.
#
# 10/29/2001 - added a check to see if by any chance a file was removed in
# the local directory (without that being updated in the database files),
# force a re-sync
#
"""Mirror a remote Zope subtree into a local directory tree.
usage: zopemir [-v] [-q] [-i] [-r] [-s pat] hostname [remotedir [localdir]]

-v: verbose
-q: quiet
-i: interactive mode
-r: remove local files/directories no longer pertinent
-s pat: skip files matching pattern
hostname: remote host
remotedir: remote directory (default initial)
localdir: local directory (default current) 

Example:
python zopemir http://myzopesite:8080 /mysite/ mylocalfolder/stagingarea/
IMPORTANT NOTE: The hostname does not have / at the end. Instead, the remote folder starts with a /
"""

# to use, set the top variables here. 
# then go to the function update_filedata() to set your own
# regex search/replace for each file you download

import os
import sys
import time
import getopt
import string
import urllib
import urlparse
import re
from fnmatch import fnmatch

# Print usage message and exit
def usage(*args):
	sys.stdout = sys.stderr
	for msg in args: print msg
	print __doc__
	sys.exit(2)

verbose = 1 # 0 for -q, 2 for -v
interactive = 0
rmok = 0
host = ''
# local mirror information. Useful for speeding up things
local_lister = '.zmir'
# dtml method in the Zope server for serving down FTP style listing
# for a folder. See bottom for code
remote_lister = '0_folderfile_listing'
#if the listing has this file name, that folder tree is ignored and not downloaded
#add this as a DTML Document - content of this is not used at all
remote_ignore = '00ignore.txt'
# skip current and parent directories and the file where local mirror info is stored
skippats = ['.', '..', local_lister]
default_filename = 'index.html'
# these files will be downloaded regardless of whether they are changed or not
force_files = ['index.html']
# the files with the following extensions will be passed through
# the function update_filedata() to remove references to wwwstage.tnc:8080
update_files_ext = ['.txt','.htm','.html','.inc']

# Main program: parse command line and start processing
def main():
	global verbose, interactive, rmok
	try:
		opts, args = getopt.getopt(sys.argv[1:], 'i:qrs:v')
	except getopt.error, msg:
		usage(msg)
	for o, a in opts:
		if o == '-v': verbose = verbose + 1
		if o == '-q': verbose = 0
		if o == '-i': interactive = 1
		if o == '-r': rmok = 1
		if o == '-s': skippats.append(a)
	if not args: usage('hostname missing')
	host = args[0]
	remotedir = ''
	localdir = ''
	if args[1:]:
		remotedir = args[1]
		if args[2:]:
			localdir = args[2]
			if args[3:]: usage('too many arguments')
	# 
	if verbose: print 'Retrieving remote files from %s...' % `host`
	mirrorsubdir(host+remotedir, localdir)
	if verbose: print 'OK.'

# Core logic: mirror one subdirectory (recursively)
def mirrorsubdir(remotedir, localdir):
	if localdir and not os.path.isdir(localdir):
		if verbose: print 'Creating local directory', `localdir`
		try:
		    makedir(localdir)
		except os.error, msg:
		    print "Failed to establish local directory", `localdir`
		    return
	infofilename = os.path.join(localdir, local_lister)
	try:
		text = open(infofilename, 'r').read()
	except IOError, msg:
		text = '{}'
	try:
		info = eval(text)
	except (SyntaxError, NameError):
		print 'Bad mirror info in %s' % `infofilename`
		info = {}
	subdirs = []
	listing = []
	if verbose: print 'Listing remote directory %s...' % `remotedir`
	for line in string.split(readURL(host+remotedir+'/'+remote_lister),'\n'):
		if len(line) > 0: listing.append(line)
	filesfound = []
	#if remote_ignore is a file, return without processing folder
	if string.find(string.join(listing,' '),remote_ignore) >= 0:
		if verbose > 1: print 'Ignore folder'
		return
	for line in listing:
		if verbose > 1: print '-->', `line`
		# Parse, assuming a UNIX listing - your DTML method should be ship shape for this!
		words = string.split(line, None, 8)
		if len(words) < 6:
			if verbose > 1: print 'Skipping short line'
			continue
		filename = string.lstrip(words[-1])
		infostuff = words[-5:-1]
		mode = words[0]
		skip = 0
		#if filename == remote_ignore:
		#	if verbose > 1: print 'Ignore folder'
		#	break
		for pat in skippats:
			if fnmatch(filename, pat):
				if verbose > 1:
					print 'Skip pattern', `pat`,
					print 'matches', `filename`
				skip = 1
				break
		if skip:
			continue
		# if a directory, remember it and continue
		if mode[0] == 'd':
			if verbose > 1:
				print 'Remembering subdirectory', `filename`
			subdirs.append(filename)
			continue
		# if a file, process it
		filesfound.append(filename)
		if info.has_key(filename) and info[filename] == infostuff  and filename not in force_files:
                        if not os.path.isfile(os.path.join(localdir,filename)):
                            if verbose > 1:
                                print 'Mismatch between file system and database', `filename`
                        else:
                            if verbose > 1:
                                    print 'Already have this version of',`filename`
                            continue
		fullname = os.path.join(localdir, filename)
		tempname = os.path.join(localdir, '@'+filename)
		if interactive:
			doit = askabout('file', filename, pwd)
			if not doit:
				if not info.has_key(filename):
					info[filename] = 'Not retrieved'
				continue
		try:
			os.unlink(tempname)
		except os.error:
			pass
		try:
			fp = open(tempname, 'wb')
		except IOError, msg:
			print "Can't create %s: %s" % ( `tempname`, str(msg))
			continue
		if verbose:
			print 'Retrieving %s from %s as %s...' %  (`filename`, `remotedir`, `fullname`)
		if verbose:
			fp1 = LoggingFile(fp, 1024, sys.stdout)
		else:
			fp1 = fp
		t0 = time.time()
		file_data = readURL(host+remotedir+'/'+filename)
		try:
			if os.path.splitext(filename)[1] in update_files_ext:
				file_data = update_filedata(file_data)
		except:
			pass
		try:
			 fp1.write(file_data)
		except:
			print 'Could not write file data'
			continue
		t1 = time.time()
		bytes = fp.tell()
		fp.close()
		if fp1 != fp:
			fp1.close()
		try:
			os.unlink(fullname)
		except os.error:
			pass		# Ignore the error
		try:
			os.rename(tempname, fullname)
		except os.error, msg:
			print "Can't rename %s to %s: %s" % (`tempname`, `fullname`, str(msg))
			continue
		info[filename] = infostuff
		writedict(info, infofilename)
		if verbose:
			dt = t1 - t0
			kbytes = bytes / 1024.0
			print int(round(kbytes)),
			print 'Kbytes in',
			print int(round(dt)),
			print 'seconds',
			if t1 > t0:
				print '(~%d Kbytes/sec)' % \
					  int(round(kbytes/dt),)
			print
	#
	# Remove files from info that are no longer remote
	deletions = 0
	for filename in info.keys():
		if filename not in filesfound:
			if verbose:
				print "Removing obsolete info entry for",
				print `filename`, "in", `localdir or "."`
			del info[filename]
			deletions = deletions + 1
	if deletions:
		writedict(info, infofilename)
	#
	# Remove local files that are no longer in the remote directory
	try:
	    if not localdir: names = os.listdir(os.curdir)
	    else: names = os.listdir(localdir)
	except os.error:
	    names = []
	for name in names:
		if name[0] == '.' or info.has_key(name) or name in subdirs:
			continue
		skip = 0
		for pat in skippats:
			if fnmatch(name, pat):
				if verbose > 1:
					print 'Skip pattern', `pat`,
					print 'matches', `name`
				skip = 1
				break
		if skip:
			continue
		fullname = os.path.join(localdir, name)
		if not rmok:
			if verbose:
				print 'Local file', `fullname`,
				print 'is no longer pertinent'
			continue
		if verbose: print 'Removing local file/dir', `fullname`
		remove(fullname)
	#
	# Recursively mirror subdirectories
	for subdir in subdirs:
		if interactive:
			doit = askabout('subdirectory', subdir, remotedir)
			if not doit: continue
		if verbose: print 'Processing subdirectory', `subdir`
		localsubdir = os.path.join(localdir, subdir)
		if verbose > 1:
			print 'Remote directory now:', `remotedir`
			print 'Remote cwd', `subdir`
			print 'Mirroring as', `localsubdir`
		mirrorsubdir(remotedir+'/'+subdir, localsubdir)
		if verbose > 1: print 'Remote cwd ..'
		newpwd = remotedir

# Helper to remove a file or directory tree
def remove(fullname):
	if os.path.isdir(fullname) and not os.path.islink(fullname):
		try:
			names = os.listdir(fullname)
		except os.error:
			names = []
		ok = 1
		for name in names:
			if not remove(os.path.join(fullname, name)):
				ok = 0
		if not ok:
			return 0
		try:
			os.rmdir(fullname)
		except os.error, msg:
			print "Can't remove local directory %s: %s" % (`fullname`, str(msg))
			return 0
	else:
		try:
			os.unlink(fullname)
		except os.error, msg:
			print "Can't remove local file %s: %s" %  (`fullname`, str(msg))
			return 0
	return 1

# Wrapper around a file for writing to write a hash sign every block.
class LoggingFile:
	def __init__(self, fp, blocksize, outfp):
		self.fp = fp
		self.bytes = 0
		self.hashes = 0
		self.blocksize = blocksize
		self.outfp = outfp
	def write(self, data):
		self.bytes = self.bytes + len(data)
		hashes = int(self.bytes) / self.blocksize
		while hashes > self.hashes:
			self.outfp.write('#')
			self.outfp.flush()
			self.hashes = self.hashes + 1
		self.fp.write(data)
	def close(self):
		self.outfp.write('\n')

# Ask permission to download a file.
def askabout(filetype, filename, pwd):
	prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd)
	while 1:
		reply = string.lower(string.strip(raw_input(prompt)))
		if reply in ['y', 'ye', 'yes']:
			return 1
		if reply in ['', 'n', 'no', 'nop', 'nope']:
			return 0
		print 'Please answer yes or no.'

# Create a directory if it doesn't exist.  Recursively create the
# parent directory as well if needed.
def makedir(pathname):
	if os.path.isdir(pathname):
		return
	dirname = os.path.dirname(pathname)
	if dirname: makedir(dirname)
	os.mkdir(pathname, 0777)

# Write a dictionary to a file in a way that can be read back using
# rval() but is still somewhat readable (i.e. not a single long line).
# Also creates a backup file.
def writedict(dict, filename):
	dir, file = os.path.split(filename)
	tempname = os.path.join(dir, '@' + file)
	backup = os.path.join(dir, file + '~')
	try:
		os.unlink(backup)
	except os.error:
		pass
	fp = open(tempname, 'w')
	fp.write('{\n')
	for key, value in dict.items():
		fp.write('%s: %s,\n' % (`key`, `value`))
	fp.write('}\n')
	fp.close()
	try:
		os.rename(filename, backup)
	except os.error:
		pass
	os.rename(tempname, filename)


# helper function to read the URL data
def readURL(url):
	"""Returns the file found in the URL
	"""
	f = urllib.urlopen(url)
	data = f.read()
	f.close()
	return data

# helper function to do the processing of HTML files
def update_filedata(data):
	"""Modifies the hard coding in the data so that it is appropriate
	do the server change processing
	ideally this should process the file content
	to make the links relative. We will just make it
	relative to root URL (/)
	"""
        ##data = re.sub('','',data)
	#data = re.sub('(?<!\.)\','',data)
	data = re.sub('<base href=.*?>','',data)
	return data
	

if __name__ == '__main__':
	main()

# 0_folderfile_listing - File Listing ala FTP - used for Slurping
##<dtml-in "objectItems(['Folder','DTML Document','Image','File','Knowledge Document'])">
##<dtml-if "meta_type=='Folder'">d---------<dtml-else>-rw-rw-rw-</dtml-if>   1 Zope     Zope<dtml-try><dtml-var getSize fmt="%13s"><dtml-except>            0</dtml-try> <dtml-var "bobobase_modification_time()" fmt="%b %d %H:%M"> <dtml-var id>
##</dtml-in>