1#!/usr/local/bin/python
2# ex:ts=4
3#-*- mode: Fundamental; tab-width: 4; -*-
4#
5# patchtool.py - a tool to automate common operation with patchfiles in the
6# FreeBSD Ports Collection.
7#
8# ----------------------------------------------------------------------------
9# "THE BEER-WARE LICENSE" (Revision 42, (c) Poul-Henning Kamp):
10# Maxim Sobolev <sobomax@FreeBSD.org> wrote this file.  As long as you retain
11# this notice you can do whatever you want with this stuff. If we meet some
12# day, and you think this stuff is worth it, you can buy me a beer in return.
13#
14# Maxim Sobolev
15# ----------------------------------------------------------------------------
16#
17#
18# MAINTAINER= sobomax@FreeBSD.org <- any unapproved commits to this file are
19#				     highly discouraged!!!
20#
21
22import os, os.path, subprocess, sys, getopt, glob, errno, types
23
24# Some global variables used as constants
25True = 1
26False = 0
27
28
29# Tweakable global variables. User is able to override any of these by setting
30# appropriate environment variable prefixed by `PT_', eg:
31# $ export PT_CVS_ID="FooOS"
32# $ export PT_DIFF_CMD="/usr/local/bin/mydiff"
33# will force script to use "FooOS" as a CVS_ID instead of "FreeBSD" and
34# "/usr/local/bin/mydiff" as a command to generate diffs.
35class Vars:
36	CVS_ID = 'MidnightBSD'
37
38	DIFF_ARGS = '-du'
39	DIFF_SUFX = '.orig'
40	PATCH_PREFIX = 'patch-'
41	PATCH_IGN_SUFX = ('.orig', '.rej')
42	RCSDIFF_SUFX = ',v'
43
44	CD_CMD = 'cd'
45	DIFF_CMD = '/usr/bin/diff'
46	MAKE_CMD = '/usr/bin/make'
47	PRINTF_CMD = '/usr/bin/printf'
48	RCSDIFF_CMD = '/usr/bin/rcsdiff'
49
50	DEFAULT_MAKEFILE = 'Makefile'
51	DEV_NULL = '/dev/null'
52	ETC_MAKE_CONF = '/etc/make.conf'
53
54	SLASH_REPL_SYMBOL = '_'	# The symbol to replace '/' when auto-generating
55							# patchnames
56
57
58#
59# Check if the supplied patch refers to a port's directory.
60#
61def isportdir(path, soft = False):
62	REQ_FILES = ('Makefile', 'pkg-descr', 'distinfo')
63	if not os.path.isdir(path) and soft != True:
64		raise IOError(errno.ENOENT, path)
65		# Not reached #
66
67	try:
68		content = os.listdir(path)
69	except OSError:
70		return False
71
72	for file in REQ_FILES:
73		if file not in content:
74			return False
75	return True
76
77
78#
79# Traverse directory tree up from the path pointed by argument and return if
80# root directory of a port is found.
81#
82def locateportdir(path, wrkdirprefix= '', strict = False):
83	# Flag to relax error checking in isportdir() function. It required when
84	# WRKDIRPREFIX is defined.
85	softisport = False
86
87	path = os.path.abspath(path)
88
89	if wrkdirprefix != '':
90		wrkdirprefix= os.path.abspath(wrkdirprefix)
91		commonprefix = os.path.commonprefix((path, wrkdirprefix))
92		if commonprefix != wrkdirprefix:
93			return ''
94		path = path[len(wrkdirprefix):]
95		softisport = True
96
97	while path != '/':
98		if isportdir(path, softisport) == True:
99			return path
100		path = os.path.abspath(os.path.join(path, '..'))
101
102	if strict == True:
103		raise LocatePDirError(path)
104		# Not reached #
105	else:
106		return ''
107
108
109#
110# Get value of a make(1) variable called varname. Optionally maintain a cache
111# for resolved varname:makepath pairs to speed-up operation if the same variable
112# from the exactly same file is requested repeatedly (invocation of make(1) is
113# very expensive operation...)
114#
115def querymakevar(varname, path = 'Makefile', strict = False, cache = {}):
116	path = os.path.abspath(path)
117
118	if cache.has_key((varname, path)) == 1:
119		return cache[(varname, path)]
120
121	origpath = path
122	if os.path.isdir(path):
123		path = os.path.join(path, Vars.DEFAULT_MAKEFILE)
124	if not os.path.isfile(path):
125		raise IOError(errno.ENOENT, path)
126		# Not reached #
127
128	dir = os.path.dirname(path)
129	CMDLINE = '%s %s && %s -f %s -V %s' % (Vars.CD_CMD, dir, Vars.MAKE_CMD, \
130      path, varname)
131	devnull = open('/dev/null', 'a')
132	pipe = subprocess.Popen(CMDLINE, shell = True, stdin = subprocess.PIPE, \
133            stdout = subprocess.PIPE, stderr = devnull, close_fds = True)
134	retval = ''
135	for line in pipe.stdout.readlines():
136		retval = retval + line.strip() + ' '
137	retval = retval[:-1]
138	if strict == True and retval.strip() == '':
139		raise MakeVarError(path, varname)
140		# Not reached #
141
142	cache[(varname, origpath)] = retval
143	return retval
144
145
146#
147# Get a path of `path'  relatively to wrksrc. For example:
148# path:		/foo/bar
149# wrksrc:	/foo/bar/baz/somefile.c
150# getrelpath:	baz/somefile.c
151# Most of the code here is to handle cases when ../ operation is required to
152# reach wrksrc from path, for example:
153# path:		/foo/bar
154# wrksrc:	/foo/baz/somefile.c
155# getrelpath:	../baz/somefile.c
156#
157def getrelpath(path, wrksrc):
158	path = os.path.abspath(path)
159	wrksrc = os.path.abspath(wrksrc) + '/'
160	commonpart = os.path.commonprefix((path, wrksrc))
161	while commonpart[-1:] != '/':
162		commonpart = commonpart[:-1]
163	path = path[len(commonpart):]
164	wrksrc = wrksrc[len(commonpart):]
165	adjust = ''
166	while os.path.normpath(os.path.join(wrksrc, adjust)) != '.':
167		adjust = os.path.join(adjust, '..')
168	relpath = os.path.join(adjust, path)
169	return relpath
170
171
172#
173# Generare a diff between saved and current versions of the file pointed by the
174# wrksrc+path. Apply heuristics to locate saved version of the file in question
175# and if it fails assume that file is new, so /dev/null is to be used as
176# original file. Optionally save generated patch into `outfile' instead of
177# dumping it to stdout. Generated patches automatically being tagged with
178# "FreeBSD" cvs id.
179#
180def gendiff(path, wrksrc, outfile = ''):
181	fullpath = os.path.join(wrksrc, path)
182	if not os.path.isfile(fullpath):
183		raise IOError(errno.ENOENT, fullpath)
184		# Not reached #
185
186	cmdline = ''
187	if os.path.isfile(fullpath + Vars.DIFF_SUFX):		# Normal diff
188		path_orig = path + Vars.DIFF_SUFX
189		cmdline = '%s %s %s %s' % (Vars.DIFF_CMD, Vars.DIFF_ARGS, path_orig, path)
190	elif os.path.isfile(fullpath + Vars.RCSDIFF_SUFX):	# RCS diff
191		path_orig = path
192		cmdline = '%s %s %s' % (Vars.RCSDIFF_CMD, Vars.DIFF_ARGS, path)
193	else:							# New file
194		path_orig = Vars.DEV_NULL
195		cmdline = '%s %s %s %s' % (Vars.DIFF_CMD, Vars.DIFF_ARGS, path_orig, path)
196
197	savedir = os.getcwd()
198	os.chdir(wrksrc)
199	devnull = open('/dev/null', 'a')
200	pipe = subprocess.Popen(cmdline, shell = True, stdin = subprocess.PIPE, \
201            stdout = subprocess.PIPE, stderr = devnull, close_fds = True)
202	outbuf = pipe.stdout.readlines()
203	exitval = pipe.wait()
204	if exitval == 0:    # No differences were found
205		retval = False
206		retmsg = 'no differencies found between original and current ' \
207			  'version of "%s"' % fullpath
208	elif exitval == 1:  # Some differences  were  found
209		if (outfile != ''):
210			outbuf[0] = '--- %s\n' % path_orig
211			outbuf[1] = '+++ %s\n' % path
212			outbuf.insert(0, '\n')
213			outbuf.insert(0, '$%s$\n' % Vars.CVS_ID)
214			outbuf.insert(0, '\n')
215			open(outfile, 'w').writelines(outbuf)
216		else:
217			sys.stdout.writelines(outbuf)
218		retval = True
219		retmsg = ''
220	else:               # Error occured
221		raise ECmdError('"%s"' % cmdline, \
222		  'external command returned non-zero error code')
223		# Not reached #
224
225	os.chdir(savedir)
226	return (retval, retmsg)
227
228
229#
230# Automatically generate a name for a patch based on its path relative to
231# wrksrc. Use simple scheme to ensute 1-to-1 mapping between path and
232# patchname - replace all '_' with '__' and all '/' with '_'.
233#
234def makepatchname(path, patchdir = ''):
235	SRS = Vars.SLASH_REPL_SYMBOL
236	retval = Vars.PATCH_PREFIX + \
237	  path.replace(SRS, SRS + SRS).replace('/', SRS)
238	retval = os.path.join(patchdir, retval)
239	return retval
240
241
242#
243# Write a specified message to stderr.
244#
245def write_msg(message):
246	if type(message) == types.StringType:
247		message = message,
248	sys.stderr.writelines(message)
249
250
251#
252# Print specified message to stdout and ask user [y/N]?. Optionally allow
253# specify default answer, i.e. return value if user typed only <cr>
254#
255def query_yn(message, default = False):
256	while True:
257		if default == True:
258			yn = 'Y/n'
259		elif default == False:
260			yn = 'y/N'
261		else:
262			yn = 'Y/N'
263
264		reply = raw_input('%s [%s]: ' % (message, yn))
265
266		if reply == 'y' or reply == 'Y':
267			return True
268		elif reply == 'n' or reply == 'N':
269			return False
270		elif reply == '' and default in (True, False):
271			return default
272		print 'Wrong answer "%s", please try again' % reply
273	return default
274
275
276#
277# Print optional message and usage information and exit with specified exit
278# code.
279#
280def usage(code, msg = ''):
281	myname = os.path.basename(sys.argv[0])
282	write_msg((str(msg), """
283Usage: %s [-afi] file ...
284       %s -u [-i] [patchfile|patchdir ...]
285""" % (myname, myname)))
286	sys.exit(code)
287
288
289#
290# Simple custom exception
291#
292class MyError(Exception):
293	msg = 'error'
294
295	def __init__(self, file, msg=''):
296		self.file = file
297		if msg != '':
298			self.msg = msg
299
300	def __str__(self):
301			return '%s: %s' % (self.file, self.msg)
302
303
304#
305# Error parsing patchfile
306#
307class PatchError(MyError):
308	msg = 'corrupt patchfile, or not patchfile at all'
309
310
311#
312# Error executing external command
313#
314class ECmdError(MyError):
315	pass
316
317
318#
319# Error getting value of makefile variable
320#
321class MakeVarError(MyError):
322	def __init__(self, file, makevar, msg=''):
323		self.file = file
324		if msg != '':
325			self.msg = msg
326		else:
327			self.msg = 'can\'t get %s value' % makevar
328
329
330#
331# Error locating portdir
332#
333class LocatePDirError(MyError):
334	msg = 'can\'t locate portdir'
335
336
337class Patch:
338	fullpath = ''
339	minus3file = ''
340	plus3file = ''
341	wrksrc = ''
342	patchmtime = 0
343	targetmtime = 0
344
345	def __init__(self, path, wrksrc):
346		MINUS3_DELIM = '--- '
347		PLUS3_DELIM = '+++ '
348
349		path = os.path.abspath(path)
350		if not os.path.isfile(path):
351			raise IOError(errno.ENOENT, path)
352			# Not reached #
353
354		self.fullpath = path
355		filedes = open(path)
356
357		for line in filedes.readlines():
358			if self.minus3file == '':
359				if line[:len(MINUS3_DELIM)] == MINUS3_DELIM:
360					lineparts = line.split()
361					try:
362						self.minus3file = lineparts[1]
363					except IndexError:
364						raise PatchError(path)
365						# Not reached #
366					continue
367			elif line[:len(PLUS3_DELIM)] == PLUS3_DELIM:
368				lineparts = line.split()
369				try:
370					self.plus3file = lineparts[1]
371				except IndexError:
372					raise PatchError(path)
373					# Not reached #
374				break
375
376		filedes.close()
377
378		if self.minus3file == '' or self.plus3file == '':
379			raise PatchError(path)
380			# Not reached #
381
382		self.wrksrc = os.path.abspath(wrksrc)
383		self.patchmtime = os.path.getmtime(self.fullpath)
384		plus3file = os.path.join(self.wrksrc, self.plus3file)
385		if os.path.isfile(plus3file):
386			self.targetmtime = os.path.getmtime(plus3file)
387		else:
388			self.targetmtime = 0
389
390	def update(self, patch_cookiemtime = 0, ignoremtime = False):
391		targetfile = os.path.join(self.wrksrc, self.plus3file)
392		if not os.path.isfile(targetfile):
393			raise IOError(errno.ENOENT, targetfile)
394			# Not reached #
395
396		patchdir = os.path.dirname(self.fullpath)
397		if not os.path.isdir(patchdir):
398			os.mkdir(patchdir)
399
400		if ignoremtime == True or self.patchmtime == 0 or \
401		  self.targetmtime == 0 or \
402		  (self.patchmtime < self.targetmtime and \
403		  patch_cookiemtime < self.targetmtime):
404			retval = gendiff(self.plus3file, self.wrksrc, self.fullpath)
405			if retval[0] == True:
406				self.patchmtime = os.path.getmtime(self.fullpath)
407		else:
408			retval = (False, 'patch is already up to date')
409		return retval
410
411
412class NewPatch(Patch):
413	def __init__(self, patchdir, wrksrc, relpath):
414		self.fullpath = makepatchname(relpath, os.path.abspath(patchdir))
415		self.wrksrc = os.path.abspath(wrksrc)
416		self.plus3file = relpath
417		self.minus3file = relpath
418		self.patchmtime = 0
419		plus3file = os.path.join(self.wrksrc, self.plus3file)
420		if os.path.isfile(plus3file):
421			self.targetmtime = os.path.getmtime(plus3file)
422		else:
423			self.targetmtime = 0
424
425
426class PatchesCollection:
427	patches = {}
428
429	def __init__(self):
430		self.patches = {}
431		pass
432
433	def adddir(self, patchdir, wrksrc):
434		if not os.path.isdir(patchdir):
435			raise IOError(errno.ENOENT, patchdir)
436			# Not reached #
437
438		for filename in glob.glob(os.path.join(patchdir, Vars.PATCH_PREFIX + '*')):
439			for sufx in Vars.PATCH_IGN_SUFX:
440				if filename[-len(sufx):] == sufx:
441					write_msg('WARNING: patchfile "%s" ignored\n' % filename)
442					break
443			else:
444				self.addpatchfile(filename, wrksrc)
445
446	def addpatchfile(self, path, wrksrc):
447		path = os.path.abspath(path)
448		if not self.patches.has_key(path):
449			self.addpatchobj(Patch(path, wrksrc))
450
451	def addpatchobj(self, patchobj):
452		self.patches[patchobj.fullpath] = patchobj
453
454	def lookupbyname(self, path):
455		path = os.path.abspath(path)
456		if self.patches.has_key(path):
457			return self.patches[path]
458		return None
459
460	def lookupbytarget(self, wrksrc, relpath):
461		wrksrc = os.path.abspath(wrksrc)
462		for patch in self.patches.values():
463			if wrksrc == patch.wrksrc and relpath == patch.plus3file:
464				return patch
465		return None
466
467	def getpatchobjs(self):
468		return self.patches.values()
469
470
471#
472# Resolve all symbolic links in the given path to a file
473#
474def truepath(path):
475	if not os.path.isfile(path):
476		raise IOError(errno.ENOENT, path)
477
478	result = ''
479	while len(path) > 0:
480		path, lastcomp = os.path.split(path)
481		if len(lastcomp) == 0:
482			lastcomp = path
483			path = ''
484		result = os.path.join(lastcomp, result)
485		if len(path) == 0:
486			break
487		if os.path.islink(path):
488			linkto = os.path.normpath(os.readlink(path))
489			if linkto[0] != '/':
490				path = os.path.join(path, linkto)
491			else:
492				path = linkto
493	return result[:-1]
494
495
496def main():
497	try:
498		opts, args = getopt.getopt(sys.argv[1:], 'afui')
499	except getopt.GetoptError, msg:
500		usage(2, msg)
501
502	automatic = False
503	force = False
504	mode = generate
505	ignoremtime = False
506
507	for o, a in opts:
508		if o == '-a':
509			automatic = True
510		elif o == '-f':
511			force = True
512		elif o == '-u':
513			mode = update
514		elif o == '-i':
515			ignoremtime = True
516		else:
517			usage(2)
518
519	# Allow user to override internal constants
520	for varname in dir(Vars):
521		if varname[:2] == '__' and varname[-2:] == '__':
522			continue
523		try:
524			value = os.environ['PT_' + varname]
525			setattr(Vars, varname, value)
526		except KeyError:
527			pass
528
529	mode(args, automatic, force, ignoremtime)
530
531	sys.exit(0)
532
533
534#
535# Display a diff or generate patchfile for the files pointed out by args.
536#
537def generate(args, automatic, force, ignoremtime):
538	if len(args) == 0:
539		usage(2, "ERROR: no input files specified")
540
541	patches = PatchesCollection()
542
543	for filepath in args:
544		for suf in Vars.RCSDIFF_SUFX, Vars.DIFF_SUFX:
545			if filepath.endswith(suf):
546				filepath = filepath[:-len(suf)]
547				break
548		if not os.path.isfile(filepath):
549			raise IOError(errno.ENOENT, filepath)
550			# Not reached #
551
552		filepath = truepath(filepath)
553
554		wrkdirprefix = querymakevar('WRKDIRPREFIX', Vars.ETC_MAKE_CONF, False)
555		portdir = locateportdir(os.path.dirname(filepath), wrkdirprefix, True)
556		wrksrc = querymakevar('WRKSRC', portdir, True)
557
558		relpath = getrelpath(filepath, wrksrc)
559
560		if automatic:
561			patchdir = querymakevar('PATCHDIR', portdir, True)
562
563			if os.path.isdir(patchdir):
564				patches.adddir(patchdir, wrksrc)
565
566			extra_patches = querymakevar('EXTRA_PATCHES', portdir, False)
567			for extra_patch in extra_patches.split():
568				if os.path.isfile(extra_patch):
569					patches.addpatchfile(extra_patch, wrksrc)
570
571			patchobj = patches.lookupbytarget(wrksrc, relpath)
572			if patchobj == None:
573				patchobj = NewPatch(patchdir, wrksrc, relpath)
574				patches.addpatchobj(patchobj)
575
576			if not force and os.path.exists(patchobj.fullpath) and \
577			  os.path.getsize(patchobj.fullpath) > 0:
578				try:
579					retval = query_yn('Target patchfile "%s" already ' \
580					  'exists, do you want to  replace it?' % \
581					  os.path.basename(patchobj.fullpath))
582				except KeyboardInterrupt:
583					sys.exit('\nAction aborted')
584					# Not reached #
585				if retval == False:
586					continue
587
588			write_msg('Generating patchfile: %s...' % \
589			  os.path.basename(patchobj.fullpath))
590
591			try:
592				retval = None
593				retval = patchobj.update(ignoremtime = ignoremtime)
594			finally:
595				# Following tricky magic intended to let us append \n even if
596				# we are going to die due to unhandled exception
597				if retval == None:
598					write_msg('OUCH!\n')
599
600			if retval[0] == False:
601				write_msg('skipped (%s)\n' % retval[1])
602			else:
603				write_msg('ok\n')
604
605		else:	# automatic != True
606			retval = gendiff(relpath, wrksrc)
607			if retval[0] == False:
608				write_msg('WARNING: %s\n' % retval[1])
609
610
611#
612# Atomatically update all patches pointed by args (may be individual
613# patchfiles, patchdirs or any directories in a portdirs). If directory argument
614# is encountered, all patches that belong to the port are updated. If no
615# arguments are supplied - current directory is assumed.
616#
617# The procedure homours last modification times of the patchfile, file from
618# which diff to be generated and `EXTRACT_COOKIE' file (usually
619# ${WRKDIR}/.extract_cookie) to update only those patches that are really need
620# to be updated.
621#
622def update(args, automatic, force, ignoremtime):
623	if len(args) == 0:
624		args = './',
625
626	for path in args:
627		if not os.path.exists(path):
628			raise IOError(errno.ENOENT, path)
629			# Not reached #
630
631		patches = PatchesCollection()
632
633		if os.path.isdir(path):
634			for wrkdirprefix in (querymakevar('WRKDIRPREFIX', \
635              Vars.ETC_MAKE_CONF, False), ''):
636				portdir = locateportdir(path, wrkdirprefix, False)
637				if portdir != '':
638					break
639			if portdir == '':
640				raise LocatePDirError(os.path.abspath(path))
641				# Not reached #
642
643			wrksrc = querymakevar('WRKSRC', portdir, True)
644			patchdir = querymakevar('PATCHDIR', portdir, True)
645
646			if os.path.isdir(patchdir):
647				patches.adddir(patchdir, wrksrc)
648			else:
649				continue
650
651		elif os.path.isfile(path):
652			portdir = locateportdir(os.path.dirname(path), '' , True)
653			wrksrc = querymakevar('WRKSRC', portdir, True)
654			patches.addpatchfile(path, wrksrc)
655
656		patch_cookie = querymakevar('PATCH_COOKIE', portdir, True)
657		if os.path.isfile(patch_cookie):
658			patch_cookiemtime = os.path.getmtime(patch_cookie)
659		else:
660			patch_cookiemtime = 0
661
662		for patchobj in patches.getpatchobjs():
663			write_msg('Updating patchfile: %s...' % \
664			  os.path.basename(patchobj.fullpath))
665
666			try:
667				retval = None
668				retval = patchobj.update(patch_cookiemtime, \
669				  ignoremtime)
670			finally:
671				if retval == None:
672					write_msg('OUCH!\n')
673
674			if retval[0] == False:
675				write_msg('skipped (%s)\n' % retval[1])
676			else:
677				write_msg('ok\n')
678
679
680if __name__ == '__main__':
681	try:
682		main()
683	except (PatchError, ECmdError, MakeVarError, LocatePDirError), msg:
684		sys.exit('ERROR: ' + str(msg))
685	except IOError, (code, msg):
686		sys.exit('ERROR: %s: %s' % (str(msg), os.strerror(code)))
687
688