1 files changed, 122 insertions, 37 deletions

diff --git a/linden/scripts/template_verifier.py b/linden/scripts/template_verifier.py
index 1bc03e7..68d82ca 100755
--- a/linden/scripts/template_verifier.py
+++ b/linden/scripts/template_verifier.py
@@ -42,37 +42,49 @@ import os
 import sys
 import urllib
 
-from indra import compatibility
-from indra import llmessage
-
-def die(msg):
-    print >>sys.stderr, msg
-    sys.exit(1)
-
-MESSAGE_TEMPLATE = 'message_template.msg'
-
-PRODUCTION_ACCEPTABLE = (compatibility.Same, compatibility.Newer)
-DEVELOPMENT_ACCEPTABLE = (
-    compatibility.Same, compatibility.Newer,
-    compatibility.Older, compatibility.Mixed)   
+from indra.ipc import compatibility
+from indra.ipc import tokenstream
+from indra.ipc import llmessage
 
 def getstatusall(command):
     """ Like commands.getstatusoutput, but returns stdout and 
     stderr separately(to get around "killed by signal 15" getting 
     included as part of the file).  Also, works on Windows."""
     (input, out, err) = os.popen3(command, 't')
-    input.close() # send no input to the command
+    status = input.close() # send no input to the command
     output = out.read()
     error = err.read()
-    out.close()
-    status = err.close() # the status comes from the *last* pipe you close
+    status = out.close()
+    status = err.close() # the status comes from the *last* pipe that is closed
     return status, output, error
 
 def getstatusoutput(command):
     status, output, error = getstatusall(command)
     return status, output
 
-def compare(base, current, mode):
+
+def die(msg):
+    print >>sys.stderr, msg
+    sys.exit(1)
+
+MESSAGE_TEMPLATE = 'message_template.msg'
+
+PRODUCTION_ACCEPTABLE = (compatibility.Same, compatibility.Newer)
+DEVELOPMENT_ACCEPTABLE = (
+    compatibility.Same, compatibility.Newer,
+    compatibility.Older, compatibility.Mixed)   
+
+MAX_MASTER_AGE = 60 * 60 * 4   # refresh master cache every 4 hours
+
+def retry(times, function, *args, **kwargs):
+    for i in range(times):
+        try:
+            return function(*args, **kwargs)
+        except Exception, e:
+            if i == times - 1:
+                raise e  # we retried all the times we could
+
+def compare(base_parsed, current_parsed, mode):
     """Compare the current template against the base template using the given
     'mode' strictness:
 
@@ -85,10 +97,8 @@ def compare(base, current, mode):
     Returns a tuple of (bool, Compatibility)
     Return True if they are compatible in this mode, False if not.
     """
-    base = llmessage.parseTemplateString(base)
-    current = llmessage.parseTemplateString(current)
 
-    compat = current.compatibleWithBase(base)
+    compat = current_parsed.compatibleWithBase(base_parsed)
     if mode == 'production':
         acceptable = PRODUCTION_ACCEPTABLE
     else:
@@ -98,12 +108,61 @@ def compare(base, current, mode):
         return True, compat
     return False, compat
 
+def fetch(url):
+    if url.startswith('file://'):
+        # just open the file directly because urllib is dumb about these things
+        file_name = url[len('file://'):]
+        return open(file_name).read()
+    else:
+        # *FIX: this doesn't throw an exception for a 404, and oddly enough the sl.com 404 page actually gets parsed successfully
+        return ''.join(urllib.urlopen(url).readlines())   
+
+def cache_master(master_url):
+    """Using the url for the master, updates the local cache, and returns an url to the local cache."""
+    master_cache = local_master_cache_filename()
+    master_cache_url = 'file://' + master_cache
+    # decide whether to refresh the master cache based on its age
+    import time
+    if (os.path.exists(master_cache)
+        and time.time() - os.path.getmtime(master_cache) < MAX_MASTER_AGE):
+        return master_cache_url  # our cache is fresh
+    # new master doesn't exist or isn't fresh
+    print "Refreshing master cache from %s" % master_url
+    def get_and_test_master():
+        new_master_contents = fetch(master_url)
+        llmessage.parseTemplateString(new_master_contents)
+        return new_master_contents
+    try:
+        new_master_contents = retry(3, get_and_test_master)
+    except IOError, e:
+        # the refresh failed, so we should just soldier on
+        print "WARNING: unable to download new master, probably due to network error.  Your message template compatibility may be suspect."
+        print "Cause: %s" % e
+        return master_cache_url
+    try:
+        mc = open(master_cache, 'wb')
+        mc.write(new_master_contents)
+        mc.close()
+    except IOError, e:
+        print "WARNING: Unable to write master message template to %s, proceeding without cache." % master_cache
+        print "Cause: %s" % e
+        return master_url
+    return master_cache_url
+
 def local_template_filename():
     """Returns the message template's default location relative to template_verifier.py:
     ./messages/message_template.msg."""
     d = os.path.dirname(os.path.realpath(__file__))
     return os.path.join(d, 'messages', MESSAGE_TEMPLATE)
 
+def local_master_cache_filename():
+    """Returns the location of the master template cache (which is in the system tempdir)
+    <temp_dir>/master_message_template_cache.msg"""
+    import tempfile
+    d = tempfile.gettempdir()
+    return os.path.join(d, 'master_message_template_cache.msg')
+
+
 def run(sysargs):
     parser = optparse.OptionParser(
         usage="usage: %prog [FILE] [FILE]",
@@ -120,43 +179,69 @@ http://wiki.secondlife.com/wiki/Template_verifier.py
         '-u', '--master_url', type='string', dest='master_url',
         default='http://secondlife.com/app/message_template/master_message_template.msg',
         help="""The url of the master message template.""")
+    parser.add_option(
+        '-c', '--cache_master', action='store_true', dest='cache_master',
+        default=False,  help="""Set to true to attempt use local cached copy of the master template.""")
 
     options, args = parser.parse_args(sysargs)
 
+    if options.mode == 'production':
+        options.cache_master = False
+
     # both current and master supplied in positional params
     if len(args) == 2:
         master_filename, current_filename = args
-        print "base:", master_filename
+        print "master:", master_filename
         print "current:", current_filename
-        master = file(master_filename).read()
-        current = file(current_filename).read()
+        master_url = 'file://%s' % master_filename
+        current_url = 'file://%s' % current_filename
     # only current supplied in positional param
     elif len(args) == 1:
-        master = None
+        master_url = None
         current_filename = args[0]
-        print "base: <master template from repository>"
+        print "master:", options.master_url 
         print "current:", current_filename
-        current = file(current_filename).read()
+        current_url = 'file://%s' % current_filename
     # nothing specified, use defaults for everything
     elif len(args) == 0:
-        master = None
-        current = None
+        master_url  = None
+        current_url = None
     else:
         die("Too many arguments")
 
-    # fetch the master from the url (default or supplied)
-    if master is None:
-        master = urllib.urlopen(options.master_url).read()
-
-    # fetch the template for this build
-    if current is None:
+    if master_url is None:
+        master_url = options.master_url
+        
+    if current_url is None:
         current_filename = local_template_filename()
-        print "base: <master template from repository>"
+        print "master:", options.master_url
         print "current:", current_filename
-        current = file(current_filename).read()
+        current_url = 'file://%s' % current_filename
+
+    # retrieve the contents of the local template and check for syntax
+    current = fetch(current_url)
+    current_parsed = llmessage.parseTemplateString(current)
+
+    if options.cache_master:
+        # optionally return a url to a locally-cached master so we don't hit the network all the time
+        master_url = cache_master(master_url)
 
+    def parse_master_url():
+        master = fetch(master_url)
+        return llmessage.parseTemplateString(master)
+    try:
+        master_parsed = retry(3, parse_master_url)
+    except (IOError, tokenstream.ParseError), e:
+        if options.mode == 'production':
+            raise e
+        else:
+            print "WARNING: problems retrieving the master from %s."  % master_url
+            print "Syntax-checking the local template ONLY, no compatibility check is being run."
+            print "Cause: %s\n\n" % e
+            return 0
+        
     acceptable, compat = compare(
-        master, current, options.mode)
+        master_parsed, current_parsed, options.mode)
 
     def explain(header, compat):
         print header