Source Code for Module fcp.pseudopythonparser

  1  #!/usr/bin/env python 
  2  # encoding: utf-8 
  3   
  4  """ 
  5  Safe method for reading files in the pseudo python syntax used for storing pyFreenet configurations. 
  6   
  7  This CANNOT read all kinds of python files. It is purely a specialized 
  8  reader for a very restricted subset of python code. 
  9   
 10  It uses json for reading more complex assignments. 
 11  """ 
 12   
 13  # this requires at least python 2.6. 
 14  import json 
 15   
 16  # Firstoff we need a reader which can be given consecutive lines and parse these into a dictionary of variables. 
 17 -class Parser: 
 18 -    def __init__(self): 
 19          """Reads config files in Pseudo-Python-Syntax. 
 20   
 21          >>> p = Parser() 
 22          >>> p.parse("a = 1") 
 23          {'a': 1} 
 24          >>> p = Parser() 
 25          >>> p.parse("b = [1,2,3,'a']") 
 26          {'b': [1, 2, 3, u'a']} 
 27          >>> p = Parser() 
 28          >>> p.parse('''c = [ { 'a': 1, 
 29          ...   'b': "c", 
 30          ...   'd': [1, 2, 3, None, False, True, "e"]}] 
 31          ...   ''') 
 32          {'c': [{u'a': 1, u'b': u'c', u'd': [1, 2, 3, None, False, True, u'e']}]} 
 33          """ 
 34          self.data = {} 
 35          self.unparsed = [] 
 36          self.endunparsed = None 
 37          self.unparsedvariable = None 
 38       
 39 -    def parse(self, text): 
 40          for line in text.splitlines(): 
 41              self.readline(line) 
 42          # if unparsed code remains, that is likely an error in the code. 
 43          if self.unparsedstring.strip() or self.endunparsed: 
 44              raise ValueError("Invalid or too complex code: " + self.endunparsed + "\n" + self.unparsed) 
 45          return self.data 
 46       
 47 -    def jsonload(self, text): 
 48          # replace entities which json encodes differently from python. 
 49          text = text.replace( 
 50              " None", " null").replace( 
 51              " True", " true").replace( 
 52              " False", " false").replace( 
 53              # json does not support tuples, so parse them as list. 
 54              " (", " [").replace( 
 55              "),", "],").replace( 
 56              # json cannot handle unicode string markers 
 57              ' u"', ' "').replace( 
 58              ' [u"', ' ["').replace( 
 59              " u'", " '").replace( 
 60              " [u'", " ['") 
 61          try: 
 62              return json.loads(text+"\n") 
 63          except ValueError: 
 64              # on old freesitemrg site entries json can break, so it 
 65              # needs some manual conversion. 
 66              # json uses " for strings but never '. Python may use ' 
 67              # for backwards compatibility we have to treat this correctly. 
 68              # If there are two " in a line, then every ' must be escaped  
 69              # as \' instead of being replaced by ". This requires some care. 
 70              lines = text.splitlines() 
 71              for n, l in enumerate(lines[:]): 
 72                  if l.count('"') and not l.count('"') % 2: # even number 
 73                      l = l.replace("'", "\\'") 
 74                      lines[n] = l 
 75                  elif "'" in l: 
 76                      l = l.replace("'", '"') 
 77                      lines[n] = l 
 78              text = "\n".join(lines) 
 79              try: 
 80                  return json.loads(text+"\n") 
 81              except ValueError: 
 82                  print text 
 83                  raise 
 84       
 85      # FIXME: Using a property here might be confusing, because I assign 
 86      #        directly to unparsed. Check whether there’s a cleaner way. 
 87      @property 
 88 -    def unparsedstring(self): 
 89          """Join and return self.unparsed as a string.""" 
 90          return "\n".join(self.unparsed) 
 91       
 92 -    def checkandprocessunprocessed(self): 
 93          """Check if the rest of self.unprocessed finishes the line.""" 
 94          if self.endunparsed in self.unparsedstring: 
 95              self.data[self.unparsedvariable] = self.jsonload(self.unparsedstring) 
 96              self.unparsed, self.unparsedvariable, self.endunparsed = [], "", "" 
 97       
 98 -    def readline(self, line): 
 99          """Read one line of text.""" 
100          # if we have unparsed code and this line does not end it, we just add the code to the unparsed code. 
101          if self.unparsed and not self.endunparsed: 
102              raise ValueError("We have unparsed data but we do not know how it ends. THIS IS A BUG.") 
103           
104          if self.unparsed and self.endunparsed: 
105              self.unparsed.append(line) 
106           
107          if self.unparsed and self.endunparsed and not line.strip().endswith(self.endunparsed): 
108              return # line is already processed as far as possible 
109          if self.unparsed and self.endunparsed and line.strip().endswith(self.endunparsed): 
110              # json uses null for None, true for True and false for False.  
111              # We have to replace those in the content and hope that nothing will break. 
112              data = self.jsonload(self.unparsedstring) 
113              self.data[self.unparsedvariable] = data 
114              self.unparsed, self.endunparsed = [], "" 
115              return 
116           
117          # start reading complex datastructures 
118          if " = [" in line: 
119              start = line.index(" = [") 
120              self.unparsedvariable = line[:start] 
121              self.unparsed = [line[start+3:]] 
122              self.endunparsed = "]" 
123              self.checkandprocessunprocessed() 
124              return 
125          elif " = {" in line: 
126              start = line.index(" = {") 
127              self.unparsedvariable = line[:start] 
128              self.unparsed = [line[start+3:]] 
129              self.endunparsed = "}" 
130              self.checkandprocessunprocessed() 
131              return 
132           
133          # handle the easy cases 
134          # ignore empty lines 
135          if not line.strip(): 
136              return 
137          # ignore comments 
138          if line.strip().startswith("#"): 
139              return 
140          # the only thing left to care for are variable assignments 
141          if not " = " in line: 
142              return 
143           
144          # prepare reading variables 
145          start = line.index(" = ") 
146          variable = line[:start] 
147          forbiddenvariablechars = " ", ".", "+", "-", "=", "*", "/" 
148          if True in [i in variable for i in forbiddenvariablechars]: 
149              raise ValueError("Variables must not contain any of the forbidden characters: " + str(forbiddenvariablechars)) 
150          rest = line[start+3:].strip() 
151           
152          # handle literal values: these are safe to eval 
153          safevalues = "True", "False", "None"  
154          if rest in safevalues: 
155              self.data[variable] = eval(rest) 
156              return 
157   
158          # handle json literals 
159          safevalues = "true", "false", "null"  
160          if rest in safevalues: 
161              self.data[variable] = json.loads(rest) 
162              return 
163           
164          # handle numbers: these are safe to eval, too 
165          numberchars = set(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "."]) 
166          if not False in [i in numberchars for i in rest]: 
167              self.data[variable] = eval(rest) 
168              return 
169           
170          # finally handle strings 
171          if rest.startswith("'") and rest.endswith("'") or rest.startswith('"') and rest.endswith('"'): 
172              self.data[variable] = rest[1:-1] 
173              return 
174           
175           
176          # if we did not return by now, the file is malformed (or too complex) 
177          raise ValueError("Invalid or too complex code: " + line) 
178   
179  if __name__ == "__main__": 
180      from doctest import testmod 
181      testmod() 
182