Changeset 1167


Ignore:
Timestamp:
Apr 26, 2011, 9:20:34 PM (8 years ago)
Author:
ksherdy
Message:

Refactor. Remove utility code from grammar.

Location:
proto/RE2PBS
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • proto/RE2PBS/Makefile

    r1166 r1167  
    1 GRAMMAR_NAME    = re2pbs
    2 CLASSPATH       = lib/antlr-3.1.3.jar
    3 ANTLRWORKS_JAR  = lib/antlrworks-1.4.2.jar
    4 RE2PBS_OUT_DIR  = bin
    5 RE2PBS_PROTO    = prototype.py
    6 RE2PBS_COMP     = compilable.py
    7 RE2PBS_DEF_FILE = test/HexCharRefDef.txt
    8 RE2PBS_TEST_FILE= test/CharRefTest.txt
     1GRAMMAR_NAME            = re2pbs
     2CLASSPATH               = lib/antlr-3.1.3.jar
     3ANTLRWORKS_JAR          = lib/antlrworks-1.4.2.jar
     4RE2PBS_OUT_DIR          = bin
     5RE2PBS_PROTO_FILE       = prototype.py
     6RE2PBS_COMP_FILE        = compilable.py
     7RE2PBS_DEF_FILE         = test/HexCharRefDef.txt
     8RE2PBS_TEST_FILE        = test/CharRefTest.txt
    99
    10 PABLO_SRC_FILE  = compilable.py
     10PABLO_SRC_FILE  = $(RE2PBS_COMP_FILE)
    1111PABLO_COMPILER  = ../Compiler/compiler2.py
    1212PABLO_TEMPLATE  = re2pbs_demo_template.cpp
    13 PABLO_OUT_FILE  = src/test.cpp
     13PABLO_OUT_FILE  = src/demo.cpp
    1414
    1515antlr: 
     
    2020
    2121proto:
    22         python $(RE2PBS_PROTO) $(RE2PBS_TEST_FILE)
     22        python $(RE2PBS_PROTO_FILE) $(RE2PBS_TEST_FILE)
    2323
    2424pablo:  $(PABLO_SRC_FILE)
  • proto/RE2PBS/bin/codepoint_symbol_table.py

    r1143 r1167  
    5151        print symbol.debug()
    5252
     53# --------------------------------------------------------------------------------
    5354class CodePointSymbol():   
    5455   
     
    9899        return "Key: %s Value: %s Invert: %s" % (self.key(), self.codepoints, self.invert)
    99100
     101# --------------------------------------------------------------------------------
     102def generate_chardefsets(symbol_table_items, prefix=''):
     103        defs = []
     104        for key, symbol in symbol_table_items:
     105                defs.append(CharSetDef(prefix + key, codepoints2chardefsets(symbol.codepoints), symbol.invert))
     106        return defs
     107
     108def codepoints2chardefsets(codepointss):
     109        """ Translates 'lists of lists' of code points Character Class Compiler CharSetDef argument list syntax """       
     110        lst = []       
     111               
     112        for codepoints in codepointss: 
     113          lgth = len(codepoints)
     114
     115          if lgth < 1:
     116            raise BadCodePointException
     117          elif lgth == 1:
     118            lst.append(chr(codepoints[0]))
     119          elif lgth == 2:
     120            lst.append(chr(codepoints[0]) + '-' + chr(codepoints[1]))
     121          else:
     122            raise BadCodePointException
     123       
     124        return lst
     125
    100126       
    101127def test():
  • proto/RE2PBS/bin/re2pbs.py

    r1166 r1167  
    99#
    1010
     11import sys
     12import string
     13
     14# --------------------------------------------------------------------------------
    1115class Node:
    1216    def debug(self):
     
    6872      return 'Single() -> (%s)' % (self.codepoint)
    6973
     74# --------------------------------------------------------------------------------
    7075class PabloCode:
    7176    """ Methods to generate and format Pablo prototype statements and code. """
     
    97102    def indent(cls, level):
    98103      return ''.join(['\t' for i in range(0, level)])
    99      
     104
     105# --------------------------------------------------------------------------------
    100106class GeneratorFactory:
    101107    """ Parallel bit stream equation generator factory class.
     
    118124      return PreValidator(level, cursor, error)
    119125
    120 import sys
    121 
     126# --------------------------------------------------------------------------------
    122127class Generator:
    123128    """ Parallel bit streams equation generator base class."""
     
    174179      print self.__class__.__name__ + ' not implemented.'
    175180      sys.exit(1)
    176      
    177     @classmethod         
    178     def make_lexical_items(cls, strm_lst, level=1):
    179         strms = ""
    180         for strm in strm_lst:
    181           strms += PabloCode.indent(level) + PabloCode.Assign(strm, '0')  +'\n'   
    182         return strms 
    183      
    184      
    185 
     181
     182# --------------------------------------------------------------------------------
    186183class PreValidator(Generator):
    187184   
     
    360357      return '\n'.join(code) + '\n'
    361358
    362 
    363359    def make_debug_stmts(self):     
    364360      code = ""
    365361      if(self.debug):
    366362        code = self.indent_level + self.re2pbs_debug_generator.make_debug_stmts()
    367 
    368       return code
    369 
     363      return code
     364
     365# --------------------------------------------------------------------------------
    370366class PostValidator(Generator):
    371367   
     
    441437      #code = self.make_single_stmts(strm)
    442438      code = self.make_kleene_star_stmts(strm, follow_set_strm)
     439      return code
     440
     441    def make_debug_stmts(self):     
     442      code = ""
     443      if(self.debug):
     444        code = self.indent_level + self.re2pbs_debug_generator.make_debug_stmts()
    443445      return code
    444446
     
    460462
    461463# --------------------------------------------------------------------------------
    462 import string     
    463      
    464464class Program:
    465   def __init__(self, template = "template.py",  out_file='out.py'):
     465  def __init__(self, factory, template = "template.py",  out_file='out.py', level=1):
    466466      """ Object to generates Python parallel bit stream template code via search and replace of template labels '@label_name' """
     467      self.factory = factory
    467468      self.template = template
    468469      self.out_file = out_file
    469470      self.contents = readfile(template)     
     471      self.indent_level = PabloCode.indent(level)
     472      self.level = level
    470473   
    471474  def generate(self, lex_strms, cc_code, pbs_code, pbs_debug=''):   
     
    476479      writefile(self.out_file, self.contents)
    477480
     481  def generate_lex_strms(self, strm_lst):
     482      """ @lex_strms """
     483      code = []
     484      for strm in strm_lst:
     485        op = self.indent_level + PabloCode.Assign(strm, '0')
     486        code.append(op)
     487      return '\n'.join(code) + '\n'
     488
     489  def generate_pbs_code_pbs_debug(self, concats, prefix=''):
     490      """ @pbs_code, @pbs_debug """
     491      pbs_code = ''
     492      pbs_debug = ''
     493      lgth = len(concats)
     494
     495      for i in range(0,lgth):
     496        pre_validator_generator = self.factory.build_pre_validator(self.level, 'cursor', 'error') 
     497        (code, debug) = self.generate_pbs_concat(pre_validator_generator, concats[i].atom_list, prefix)
     498        pbs_code += code
     499        pbs_debug += debug
     500
     501      #TODO - glue concatenation list pbs code, glue debug code
     502
     503      return (pbs_code, pbs_debug)
     504
     505  # generate_pbs_code_pbs_debug helper
     506  def generate_pbs_concat(self, generator, atoms, prefix=''):
     507      pbs_code = ''
     508      pbs_debug = ''
     509      lgth = len(atoms)
     510
     511      if lgth > 0:
     512        pbs_code = generator.make_init_stmts(prefix + atoms[0].strm_name)
     513
     514        for i in range(0,lgth):
     515          pbs_code += generator.make_match_stmts(prefix + atoms[i].strm_name, atoms[i].quantifier, None)
     516
     517        pbs_code += generator.make_finalization_stmts()
     518        pbs_debug += generator.make_debug_stmts()
     519
     520      return (pbs_code, pbs_debug)
     521
     522
    478523# --------------------------------------------------------------------------------
    479524def readfile(filename):
     
    493538    factory = GeneratorFactory()
    494539   
    495     re2pbs_code_generator = factory.build_post_validator()
     540    re2pbs_code_generator = factory.build_pre_validator()
    496541    pbs_code = re2pbs_code_generator.make_init_stmts('lex.Ampersand')                           # init   '&'
    497542    pbs_code += re2pbs_code_generator.make_match_stmts('lex.Ampersand', None, 'lex.Hash')       # expect '&'
  • proto/RE2PBS/compilable_template.py

    r1166 r1167  
    2727@lex_strms 
    2828
    29 
    3029def Classify_bytes(basis_bits,lex):
    31 
    3230@cc_code
    33 
    3431        return lex
    3532
    3633def Demo(u8data, lex):
    3734@pbs_code
    38 
    3935        return
    4036
    4137def Main(basis_bits, lex):     
    42                                
    4338        Classify_bytes(basis_bits, lex)
    4439        Demo(lex, scope1)
  • proto/RE2PBS/prototype.py

    r1166 r1167  
    1111# (at)cc_code   - Character Class compiler equations.
    1212# (at)pbs_code  - Parallel bit stream equations.
     13# (at)pbs_debug - Parallel bit stream debug statements.
    1314#
    1415#----------------------------------------------------------------------------
     
    3637        s38 = (0)
    3738        s59 = (0)
    38 
     39 
    3940
    4041def Classify_bytes(basis_bits,lex):
    41 
    4242        temp1 = (basis_bits.bit_0 | basis_bits.bit_1)
    4343        temp2 = (basis_bits.bit_2 &~ basis_bits.bit_3)
     
    6868
    6969def Demo(u8data, lex):
     70
    7071        global lgth
    7172        lgth = len(u8data)
    72        
    7373        error_1_0 = (0)
    7474        cursor_1_0 = ((lex.s38) &~ (bitutil.Advance(lex.s38)))
     
    144144        ('      error_1_6 |= (error_1_5)', bitutil.bitstream2string(error_1_6, lgth+1))])
    145145
    146 
    147146        return
    148147
  • proto/RE2PBS/prototype_template.py

    r1166 r1167  
    3434@lex_strms 
    3535
    36 
    3736def Classify_bytes(basis_bits,lex):
    38 
    3937@cc_code
    40 
    4138        return lex
    4239
    4340def Demo(u8data, lex):
     41
    4442        global lgth
    4543        lgth = len(u8data)
    46        
    4744@pbs_code
    4845@pbs_debug
    49 
    5046        return
    5147
  • proto/RE2PBS/re2pbs.g

    r1166 r1167  
    5454
    5555@header {
    56         # --------------------------------------------------------------------------------
    57         # Imports       
    58         # --------------------------------------------------------------------------------
    5956        from re2pbs import *
    60         from codepoint_symbol_table import CodePointSymbolTable, CodePointSymbol
     57        from codepoint_symbol_table import *
    6158        from charset_compiler import chardeflist2py
    6259        from charset_def import CharSetDef
    63 
    64         # --------------------------------------------------------------------------------
    65         # Methods
    66         # --------------------------------------------------------------------------------
    67         def codepoints2chardefsets(codepointss):
    68                 """ Translates 'lists of lists' of code points Character Class Compiler CharSetDef argument list syntax """       
    69                 lst = []       
    70                        
    71                 for codepoints in codepointss: 
    72                   lgth = len(codepoints)
    73 
    74                   if lgth < 1:
    75                     raise BadCodePointException
    76                   elif lgth == 1:
    77                     lst.append(chr(codepoints[0]))
    78                   elif lgth == 2:
    79                     lst.append(chr(codepoints[0]) + '-' + chr(codepoints[1]))
    80                   else:
    81                     raise BadCodePointException
    82                
    83                 return lst
    84 
    85 
    86         def generate_chardefsets(symbol_table_items, prefix=''):
    87                 defs = []
    88                 for key, symbol in symbol_table_items:
    89                         defs.append(CharSetDef(prefix + key, codepoints2chardefsets(symbol.codepoints), symbol.invert))
    90                 return defs
    91 
    92         # TODO - relocate
    93         def generate_pbs_code_concatenations(generator_factory, concatenations, prefix='', level=1, debug=True):
    94                 pbs_code = ''
    95                 pbs_debug = ''
    96                 lgth = len(concatenations)
    97 
    98                 for i in range(0,lgth):
    99                         pre_validator_generator = generator_factory.build_pre_validator(level, 'cursor','error') 
    100                         (code, debug) = generate_pbs_code_atoms(pre_validator_generator, concatenations[i].atom_list, prefix)
    101                         pbs_code += code
    102                         pbs_debug += debug
    103 
    104                 #TODO - glue concatenation list pbs code, glue debug code
    105 
    106                 return (pbs_code, pbs_debug)
    107 
    108         # TODO - relocate
    109         def generate_pbs_code_atoms(generator, atoms, prefix='', debug=True):
    110                 pbs_code = ''
    111                 pbs_debug = ''
    112                 lgth = len(atoms)
    113 
    114                 if lgth > 0:
    115                         pbs_code = generator.make_init_stmts(prefix + atoms[0].strm_name)
    116                
    117                         for i in range(0,lgth):
    118                                 pbs_code += generator.make_match_stmts(prefix + atoms[i].strm_name, atoms[i].quantifier, None)
    119 
    120                         pbs_code += generator.make_finalization_stmts()
    121                         pbs_debug += generator.make_debug_stmts()
    122 
    123                 return (pbs_code, pbs_debug)
    12460}
    12561
     
    13672prog    :       r = re EOF
    13773{
    138 
    139         prefix = 'lex.'
    14074        lex_strms = ''
    14175        cc_code = ''
     
    14377        pbs_debug = ''
    14478
    145         factory = GeneratorFactory()
    146 
    147         lex_strms = Generator.make_lexical_items(self.symbol_table.symbols.keys())
     79        prefix = 'lex.'
     80
     81        # Character Class compiler dependency
     82        lexical_item_names = self.symbol_table.symbols.keys()
    14883        cc_code = chardeflist2py(generate_chardefsets(self.symbol_table.symbols.items(), prefix))
    149         (pbs_code, pbs_debug) = generate_pbs_code_concatenations(factory, r.union.concatenation_list, prefix)   
    150        
     84
    15185        # Generate prototype.py based on the template file 'prototype_template.py'.
    152         prototype = Program('prototype_template.py', 'prototype.py')
     86        factory = GeneratorFactory()   
     87        prototype = Program(factory, 'prototype_template.py', 'prototype.py')
     88        lex_strms = prototype.generate_lex_strms(lexical_item_names)
     89       
     90        (pbs_code, pbs_debug) = prototype.generate_pbs_code_pbs_debug(r.union.concatenation_list, prefix)               
    15391        prototype.generate(lex_strms, cc_code, pbs_code, pbs_debug)
    15492
    15593        # Generate 'compilable.py' based on the template file 'compilable_template.py'.
    156         compilable = Program('compilable_template.py', 'compilable.py')
     94        factory = GeneratorFactory()   
     95        compilable = Program(factory, 'compilable_template.py', 'compilable.py')
     96        lex_strms = compilable.generate_lex_strms(lexical_item_names)
     97        (pbs_code, pbs_debug) = compilable.generate_pbs_code_pbs_debug(r.union.concatenation_list, prefix)             
    15798        compilable.generate(lex_strms, cc_code, pbs_code, pbs_debug)
    15899}
Note: See TracChangeset for help on using the changeset viewer.