3 SCons string substitution.
8 # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 The SCons Foundation
10 # Permission is hereby granted, free of charge, to any person obtaining
11 # a copy of this software and associated documentation files (the
12 # "Software"), to deal in the Software without restriction, including
13 # without limitation the rights to use, copy, modify, merge, publish,
14 # distribute, sublicense, and/or sell copies of the Software, and to
15 # permit persons to whom the Software is furnished to do so, subject to
16 # the following conditions:
18 # The above copyright notice and this permission notice shall be included
19 # in all copies or substantial portions of the Software.
21 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
22 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
23 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
25 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
26 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
27 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
30 __revision__ = "src/engine/SCons/Subst.py 3842 2008/12/20 22:59:52 scons"
40 from SCons.Util import is_String, is_Sequence
42 # Indexed by the SUBST_* constants below.
43 _strconv = [SCons.Util.to_String_for_subst,
44 SCons.Util.to_String_for_subst,
45 SCons.Util.to_String_for_signature]
49 AllowableExceptions = (IndexError, NameError)
51 def SetAllowableExceptions(*excepts):
52 global AllowableExceptions
53 AllowableExceptions = filter(None, excepts)
55 def raise_exception(exception, target, s):
56 name = exception.__class__.__name__
57 msg = "%s `%s' trying to evaluate `%s'" % (name, exception, s)
59 raise SCons.Errors.BuildError, (target[0], msg)
61 raise SCons.Errors.UserError, msg
66 """A wrapper for a string. If you use this object wrapped
67 around a string, then it will be interpreted as literal.
68 When passed to the command interpreter, all special
69 characters will be escaped."""
70 def __init__(self, lstr):
76 def escape(self, escape_func):
77 return escape_func(self.lstr)
79 def for_signature(self):
85 class SpecialAttrWrapper:
86 """This is a wrapper for what we call a 'Node special attribute.'
87 This is any of the attributes of a Node that we can reference from
88 Environment variable substitution, such as $TARGET.abspath or
89 $SOURCES[1].filebase. We implement the same methods as Literal
90 so we can handle special characters, plus a for_signature method,
91 such that we can return some canonical string during signature
92 calculation to avoid unnecessary rebuilds."""
94 def __init__(self, lstr, for_signature=None):
95 """The for_signature parameter, if supplied, will be the
96 canonical string we return from for_signature(). Else
97 we will simply return lstr."""
100 self.forsig = for_signature
107 def escape(self, escape_func):
108 return escape_func(self.lstr)
110 def for_signature(self):
113 def is_literal(self):
116 def quote_spaces(arg):
117 """Generic function for putting double quotes around any string that
118 has white space in it."""
119 if ' ' in arg or '\t' in arg:
124 class CmdStringHolder(UserString.UserString):
125 """This is a special class used to hold strings generated by
126 scons_subst() and scons_subst_list(). It defines a special method
127 escape(). When passed a function with an escape algorithm for a
128 particular platform, it will return the contained string with the
129 proper escape sequences inserted.
131 def __init__(self, cmd, literal=None):
132 UserString.UserString.__init__(self, cmd)
133 self.literal = literal
135 def is_literal(self):
138 def escape(self, escape_func, quote_func=quote_spaces):
139 """Escape the string with the supplied function. The
140 function is expected to take an arbitrary string, then
141 return it with all special characters escaped and ready
142 for passing to the command interpreter.
144 After calling this function, the next call to str() will
145 return the escaped string.
148 if self.is_literal():
149 return escape_func(self.data)
150 elif ' ' in self.data or '\t' in self.data:
151 return quote_func(self.data)
155 def escape_list(list, escape_func):
156 """Escape a list of arguments by running the specified escape_func
157 on every object in the list that has an escape() method."""
158 def escape(obj, escape_func=escape_func):
161 except AttributeError:
164 return e(escape_func)
165 return map(escape, list)
168 """A wrapper class that delays turning a list of sources or targets
169 into a NodeList until it's needed. The specified function supplied
170 when the object is initialized is responsible for turning raw nodes
171 into proxies that implement the special attributes like .abspath,
172 .source, etc. This way, we avoid creating those proxies just
173 "in case" someone is going to use $TARGET or the like, and only
174 go through the trouble if we really have to.
176 In practice, this might be a wash performance-wise, but it's a little
177 cleaner conceptually...
180 def __init__(self, list, func):
183 def _return_nodelist(self):
185 def _gen_nodelist(self):
189 elif not is_Sequence(list):
191 # The map(self.func) call is what actually turns
192 # a list into appropriate proxies.
193 self.nodelist = SCons.Util.NodeList(map(self.func, list))
194 self._create_nodelist = self._return_nodelist
196 _create_nodelist = _gen_nodelist
199 class Targets_or_Sources(UserList.UserList):
200 """A class that implements $TARGETS or $SOURCES expansions by in turn
201 wrapping a NLWrapper. This class handles the different methods used
202 to access the list, calling the NLWrapper to create proxies on demand.
204 Note that we subclass UserList.UserList purely so that the
205 is_Sequence() function will identify an object of this class as
206 a list during variable expansion. We're not really using any
207 UserList.UserList methods in practice.
209 def __init__(self, nl):
211 def __getattr__(self, attr):
212 nl = self.nl._create_nodelist()
213 return getattr(nl, attr)
214 def __getitem__(self, i):
215 nl = self.nl._create_nodelist()
217 def __getslice__(self, i, j):
218 nl = self.nl._create_nodelist()
219 i = max(i, 0); j = max(j, 0)
222 nl = self.nl._create_nodelist()
225 nl = self.nl._create_nodelist()
228 class Target_or_Source:
229 """A class that implements $TARGET or $SOURCE expansions by in turn
230 wrapping a NLWrapper. This class handles the different methods used
231 to access an individual proxy Node, calling the NLWrapper to create
234 def __init__(self, nl):
236 def __getattr__(self, attr):
237 nl = self.nl._create_nodelist()
241 # If there is nothing in the list, then we have no attributes to
242 # pass through, so raise AttributeError for everything.
243 raise AttributeError, "NodeList has no attribute: %s" % attr
244 return getattr(nl0, attr)
246 nl = self.nl._create_nodelist()
251 nl = self.nl._create_nodelist()
256 def subst_dict(target, source):
257 """Create a dictionary for substitution of special
258 construction variables.
260 This translates the following special arguments:
262 target - the target (object or array of objects),
263 used to generate the TARGET and TARGETS
264 construction variables
266 source - the source (object or array of objects),
267 used to generate the SOURCES and SOURCE
268 construction variables
273 def get_tgt_subst_proxy(thing):
275 subst_proxy = thing.get_subst_proxy()
276 except AttributeError:
277 subst_proxy = thing # probably a string, just return it
279 tnl = NLWrapper(target, get_tgt_subst_proxy)
280 dict['TARGETS'] = Targets_or_Sources(tnl)
281 dict['TARGET'] = Target_or_Source(tnl)
283 dict['TARGETS'] = None
284 dict['TARGET'] = None
287 def get_src_subst_proxy(node):
290 except AttributeError:
295 return node.get_subst_proxy()
296 except AttributeError:
297 return node # probably a String, just return it
298 snl = NLWrapper(source, get_src_subst_proxy)
299 dict['SOURCES'] = Targets_or_Sources(snl)
300 dict['SOURCE'] = Target_or_Source(snl)
302 dict['SOURCES'] = None
303 dict['SOURCE'] = None
307 # Constants for the "mode" parameter to scons_subst_list() and
308 # scons_subst(). SUBST_RAW gives the raw command line. SUBST_CMD
309 # gives a command line suitable for passing to a shell. SUBST_SIG
310 # gives a command line appropriate for calculating the signature
311 # of a command line...if this changes, we should rebuild.
316 _rm = re.compile(r'\$[()]')
317 _remove = re.compile(r'\$\([^\$]*(\$[^\)][^\$]*)*\$\)')
319 # Indexed by the SUBST_* constants above.
320 _regex_remove = [ _rm, None, _remove ]
323 #return [ l for l in list if not l in ('$(', '$)') ]
324 return filter(lambda l: not l in ('$(', '$)'), list)
326 def _remove_list(list):
328 do_append = result.append
331 do_append = lambda x: None
333 do_append = result.append
338 # Indexed by the SUBST_* constants above.
339 _list_remove = [ _rm_list, None, _remove_list ]
341 # Regular expressions for splitting strings and handling substitutions,
342 # for use by the scons_subst() and scons_subst_list() functions:
344 # The first expression compiled matches all of the $-introduced tokens
345 # that we need to process in some way, and is used for substitutions.
346 # The expressions it matches are:
351 # "$variable" [must begin with alphabetic or underscore]
354 # The second expression compiled is used for splitting strings into tokens
355 # to be processed, and it matches all of the tokens listed above, plus
356 # the following that affect how arguments do or don't get joined together:
359 # "non-white-space" [without any dollar signs]
360 # "$" [single dollar sign]
362 _dollar_exps_str = r'\$[\$\(\)]|\$[_a-zA-Z][\.\w]*|\${[^}]*}'
363 _dollar_exps = re.compile(r'(%s)' % _dollar_exps_str)
364 _separate_args = re.compile(r'(%s|\s+|[^\s\$]+|\$)' % _dollar_exps_str)
366 # This regular expression is used to replace strings of multiple white
367 # space characters in the string result from the scons_subst() function.
368 _space_sep = re.compile(r'[\t ]+(?![^{]*})')
370 def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={}, lvars={}, conv=None):
371 """Expand a string or list containing construction variable
374 This is the work-horse function for substitutions in file names
375 and the like. The companion scons_subst_list() function (below)
376 handles separating command lines into lists of arguments, so see
377 that function if that's what you're looking for.
379 if type(strSubst) == types.StringType and string.find(strSubst, '$') < 0:
383 """A class to construct the results of a scons_subst() call.
385 This binds a specific construction environment, mode, target and
386 source with two methods (substitute() and expand()) that handle
389 def __init__(self, env, mode, target, source, conv, gvars):
397 def expand(self, s, lvars):
398 """Expand a single "token" as necessary, returning an
399 appropriate string containing the expansion.
401 This handles expanding different types of things (strings,
402 lists, callables) appropriately. It calls the wrapper
403 substitute() method to re-expand things as necessary, so that
404 the results of expansions of side-by-side strings still get
405 re-evaluated separately, not smushed together.
410 except (IndexError, ValueError):
420 if key[0] == '{' or string.find(key, '.') >= 0:
424 s = eval(key, self.gvars, lvars)
425 except KeyboardInterrupt:
428 if e.__class__ in AllowableExceptions:
430 raise_exception(e, self.target, s)
432 if lvars.has_key(key):
434 elif self.gvars.has_key(key):
436 elif not NameError in AllowableExceptions:
437 raise_exception(NameError(key), self.target, s)
441 # Before re-expanding the result, handle
442 # recursive expansion by copying the local
443 # variable dictionary and overwriting a null
444 # string for the value of the variable name
447 # This could potentially be optimized by only
448 # copying lvars when s contains more expansions,
449 # but lvars is usually supposed to be pretty
450 # small, and deeply nested variable expansions
451 # are probably more the exception than the norm,
452 # so it should be tolerable for now.
454 var = string.split(key, '.')[0]
456 return self.substitute(s, lv)
458 def func(l, conv=self.conv, substitute=self.substitute, lvars=lvars):
459 return conv(substitute(l, lvars))
463 s = s(target=self.target,
466 for_signature=(self.mode != SUBST_CMD))
468 # This probably indicates that it's a callable
469 # object that doesn't match our calling arguments
471 if self.mode == SUBST_RAW:
474 return self.substitute(s, lvars)
480 def substitute(self, args, lvars):
481 """Substitute expansions in an argument or list of arguments.
483 This serves as a wrapper for splitting up a string into
486 if is_String(args) and not isinstance(args, CmdStringHolder):
487 args = str(args) # In case it's a UserString.
489 def sub_match(match, conv=self.conv, expand=self.expand, lvars=lvars):
490 return conv(expand(match.group(1), lvars))
491 result = _dollar_exps.sub(sub_match, args)
493 # If the internal conversion routine doesn't return
494 # strings (it could be overridden to return Nodes, for
495 # example), then the 1.5.2 re module will throw this
496 # exception. Back off to a slower, general-purpose
497 # algorithm that works for all data types.
498 args = _separate_args.findall(args)
501 result.append(self.conv(self.expand(a, lvars)))
505 result = string.join(map(str, result), '')
508 return self.expand(args, lvars)
511 conv = _strconv[mode]
513 # Doing this every time is a bit of a waste, since the Executor
514 # has typically already populated the OverrideEnvironment with
515 # $TARGET/$SOURCE variables. We're keeping this (for now), though,
516 # because it supports existing behavior that allows us to call
517 # an Action directly with an arbitrary target+source pair, which
518 # we use in Tool/tex.py to handle calling $BIBTEX when necessary.
519 # If we dropped that behavior (or found another way to cover it),
520 # we could get rid of this call completely and just rely on the
521 # Executor setting the variables.
522 d = subst_dict(target, source)
527 # We're (most likely) going to eval() things. If Python doesn't
528 # find a __builtins__ value in the global dictionary used for eval(),
529 # it copies the current global values for you. Avoid this by
530 # setting it explicitly and then deleting, so we don't pollute the
531 # construction environment Dictionary(ies) that are typically used
533 gvars['__builtins__'] = __builtins__
535 ss = StringSubber(env, mode, target, source, conv, gvars)
536 result = ss.substitute(strSubst, lvars)
539 del gvars['__builtins__']
543 if is_String(result):
544 # Remove $(-$) pairs and any stuff in between,
545 # if that's appropriate.
546 remove = _regex_remove[mode]
548 result = remove.sub('', result)
549 if mode != SUBST_RAW:
550 # Compress strings of white space characters into
552 result = string.strip(_space_sep.sub(' ', result))
553 elif is_Sequence(result):
554 remove = _list_remove[mode]
556 result = remove(result)
560 #Subst_List_Strings = {}
562 def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={}, lvars={}, conv=None):
563 """Substitute construction variables in a string (or list or other
564 object) and separate the arguments into a command list.
566 The companion scons_subst() function (above) handles basic
567 substitutions within strings, so see that function instead
568 if that's what you're looking for.
571 # Subst_List_Strings[strSubst] = Subst_List_Strings[strSubst] + 1
573 # Subst_List_Strings[strSubst] = 1
575 # SCons.Debug.caller_trace(1)
576 class ListSubber(UserList.UserList):
577 """A class to construct the results of a scons_subst_list() call.
579 Like StringSubber, this class binds a specific construction
580 environment, mode, target and source with two methods
581 (substitute() and expand()) that handle the expansion.
583 In addition, however, this class is used to track the state of
584 the result(s) we're gathering so we can do the appropriate thing
585 whenever we have to append another word to the result--start a new
586 line, start a new word, append to the current word, etc. We do
587 this by setting the "append" attribute to the right method so
588 that our wrapper methods only need ever call ListSubber.append(),
589 and the rest of the object takes care of doing the right thing
592 def __init__(self, env, mode, target, source, conv, gvars):
593 UserList.UserList.__init__(self, [])
601 if self.mode == SUBST_RAW:
602 self.add_strip = lambda x, s=self: s.append(x)
604 self.add_strip = lambda x, s=self: None
608 def expand(self, s, lvars, within_list):
609 """Expand a single "token" as necessary, appending the
610 expansion to the current result.
612 This handles expanding different types of things (strings,
613 lists, callables) appropriately. It calls the wrapper
614 substitute() method to re-expand things as necessary, so that
615 the results of expansions of side-by-side strings still get
616 re-evaluated separately, not smushed together.
622 except (IndexError, ValueError):
631 self.open_strip('$(')
633 self.close_strip('$)')
636 if key[0] == '{' or string.find(key, '.') >= 0:
640 s = eval(key, self.gvars, lvars)
641 except KeyboardInterrupt:
644 if e.__class__ in AllowableExceptions:
646 raise_exception(e, self.target, s)
648 if lvars.has_key(key):
650 elif self.gvars.has_key(key):
652 elif not NameError in AllowableExceptions:
653 raise_exception(NameError(), self.target, s)
657 # Before re-expanding the result, handle
658 # recursive expansion by copying the local
659 # variable dictionary and overwriting a null
660 # string for the value of the variable name
663 var = string.split(key, '.')[0]
665 self.substitute(s, lv, 0)
669 self.substitute(a, lvars, 1)
673 s = s(target=self.target,
676 for_signature=(self.mode != SUBST_CMD))
678 # This probably indicates that it's a callable
679 # object that doesn't match our calling arguments
681 if self.mode == SUBST_RAW:
685 self.substitute(s, lvars, within_list)
691 def substitute(self, args, lvars, within_list):
692 """Substitute expansions in an argument or list of arguments.
694 This serves as a wrapper for splitting up a string into
698 if is_String(args) and not isinstance(args, CmdStringHolder):
699 args = str(args) # In case it's a UserString.
700 args = _separate_args.findall(args)
702 if a[0] in ' \t\n\r\f\v':
710 self.expand(a, lvars, within_list)
712 self.expand(args, lvars, within_list)
715 """Arrange for the next word to start a new line. This
716 is like starting a new word, except that we have to append
717 another line to the result."""
718 UserList.UserList.append(self, [])
722 """Arrange for the next word to append to the end of the
723 current last word in the result."""
724 self.append = self.add_to_current_word
727 """Arrange for the next word to start a new word."""
728 self.append = self.add_new_word
730 def add_to_current_word(self, x):
731 """Append the string x to the end of the current last word
732 in the result. If that is not possible, then just add
733 it as a new word. Make sure the entire concatenated string
734 inherits the object attributes of x (in particular, the
735 escape function) by wrapping it as CmdStringHolder."""
737 if not self.in_strip or self.mode != SUBST_SIG:
739 current_word = self[-1][-1]
743 # All right, this is a hack and it should probably
744 # be refactored out of existence in the future.
745 # The issue is that we want to smoosh words together
746 # and make one file name that gets escaped if
747 # we're expanding something like foo$EXTENSION,
748 # but we don't want to smoosh them together if
749 # it's something like >$TARGET, because then we'll
750 # treat the '>' like it's part of the file name.
751 # So for now, just hard-code looking for the special
752 # command-line redirection characters...
754 last_char = str(current_word)[-1]
757 if last_char in '<>|':
762 # We used to treat a word appended to a literal
763 # as a literal itself, but this caused problems
764 # with interpreting quotes around space-separated
765 # targets on command lines. Removing this makes
766 # none of the "substantive" end-to-end tests fail,
767 # so we'll take this out but leave it commented
768 # for now in case there's a problem not covered
769 # by the test cases and we need to resurrect this.
770 #literal1 = self.literal(self[-1][-1])
771 #literal2 = self.literal(x)
774 #y = CmdStringHolder(y, literal1 or literal2)
775 y = CmdStringHolder(y, None)
778 def add_new_word(self, x):
779 if not self.in_strip or self.mode != SUBST_SIG:
780 literal = self.literal(x)
783 x = CmdStringHolder(x, literal)
785 self.append = self.add_to_current_word
787 def literal(self, x):
790 except AttributeError:
795 def open_strip(self, x):
796 """Handle the "open strip" $( token."""
800 def close_strip(self, x):
801 """Handle the "close strip" $) token."""
806 conv = _strconv[mode]
808 # Doing this every time is a bit of a waste, since the Executor
809 # has typically already populated the OverrideEnvironment with
810 # $TARGET/$SOURCE variables. We're keeping this (for now), though,
811 # because it supports existing behavior that allows us to call
812 # an Action directly with an arbitrary target+source pair, which
813 # we use in Tool/tex.py to handle calling $BIBTEX when necessary.
814 # If we dropped that behavior (or found another way to cover it),
815 # we could get rid of this call completely and just rely on the
816 # Executor setting the variables.
817 d = subst_dict(target, source)
822 # We're (most likely) going to eval() things. If Python doesn't
823 # find a __builtins__ value in the global dictionary used for eval(),
824 # it copies the current global values for you. Avoid this by
825 # setting it explicitly and then deleting, so we don't pollute the
826 # construction environment Dictionary(ies) that are typically used
828 gvars['__builtins__'] = __builtins__
830 ls = ListSubber(env, mode, target, source, conv, gvars)
831 ls.substitute(strSubst, lvars, 0)
834 del gvars['__builtins__']
840 def scons_subst_once(strSubst, env, key):
841 """Perform single (non-recursive) substitution of a single
842 construction variable keyword.
844 This is used when setting a variable when copying or overriding values
845 in an Environment. We want to capture (expand) the old value before
846 we override it, so people can do things like:
848 env2 = env.Clone(CCFLAGS = '$CCFLAGS -g')
850 We do this with some straightforward, brute-force code here...
852 if type(strSubst) == types.StringType and string.find(strSubst, '$') < 0:
855 matchlist = ['$' + key, '${' + key + '}']
856 val = env.get(key, '')
857 def sub_match(match, val=val, matchlist=matchlist):
862 return string.join(map(str, a))
866 if is_Sequence(strSubst):
877 result.append(_dollar_exps.sub(sub_match, arg))
881 elif is_String(strSubst):
882 return _dollar_exps.sub(sub_match, strSubst)