Changeset 4462

Timestamp:

Feb 5, 2015, 7:34:28 PM (4 years ago)

Author:

cameron

Message:

Simplify generated tests

Location:

proto/charsetcompiler

Files:

• UCD/UCD_parser.py (modified) (1 diff)
• generate_UCD_tests.py (modified) (3 diffs)

proto/charsetcompiler/UCD/UCD_parser.py

@@ -208 +208 @@
         if not v in name_list_order: 
             #raise Exception("Property %s value %s missing" % (self.full_name_map[property_code], v))
-            print("Warning: property %s has no instance of value %s" % (property_code, v))
+            #print("Warning: property %s has no instance of value %s" % (property_code, v))
             value_map[v] = empty_uset()
             name_list_order.append(v)

proto/charsetcompiler/generate_UCD_tests.py

@@ -61 +61 @@
 
     def load_all(self):
-        # First parse all property names and their aliases
-        self.load_property_name_info()
-        #
-        # Next parse all property value names and their aliases.  Generate the data.
-        self.load_property_value_info()
-        #
-        # The Block property
-        self.load_enumerated_property_data('Blocks', 'blk')
-        #
-        # Scripts
-        self.load_enumerated_property_data('Scripts', 'sc')
-        #
-        # Script Extensions
-        self.load_ScriptExtensions_data()
-        #
-        # General Category
-        self.load_enumerated_property_data('extracted/DerivedGeneralCategory', 'gc')
-        #
-        # Core Properties
-        self.load_binary_properties_data('DerivedCoreProperties')
-        #
-        self.load_binary_properties_data('PropList')
-
+        # First parse all property names and their aliases
+        self.load_property_name_info()
+        #
+        # Next parse all property value names and their aliases.  Generate the data.
+        self.load_property_value_info()
+        #
+        # The Block property
+        self.load_enumerated_property_data('Blocks', 'blk')
+        #
+        # Scripts
+        self.load_enumerated_property_data('Scripts', 'sc')
+        #
+        # Script Extensions
+        self.load_ScriptExtensions_data()
+        #
+        # General Category
+        self.load_enumerated_property_data('extracted/DerivedGeneralCategory', 'gc')
+        #
+        # Core Properties
+        self.load_binary_properties_data('DerivedCoreProperties')
+        #
+        self.load_binary_properties_data('PropList')
         self.load_others()
 
-    def generate_level_1_property_terms(self):
+    def generate_level_1_property_terms(self, negated_per_10 = 5, propgroups=['others', 'sc', 'scx', 'gc']):
         template = r"""<grepcase regexp="^\%s{%s}$" datafile="All_good" grepcount="%i"/>"""
         terms = []
-        for p in self.others:
-            s = self.binary_value_map[p]
-            lbl = 'p'
-            neg = randint(0,1)
-            if neg == 1:
-                s = uset_complement(s)
-                lbl = 'P'
-            terms.append(template % (lbl, p, uset_popcount(uset_intersection(self.all_good_set, s))))
-        for v in self.property_value_list['gc']:
-            s = self.enum_value_map['gc'][v]
-            lbl = 'p'
-            neg = randint(0,1)
-            if neg == 1:
-                s = uset_complement(s)
-                lbl = 'P'
-            terms.append(template % (lbl, v, uset_popcount(uset_intersection(self.all_good_set, s))))
-        for v in self.property_value_list['sc']:
-            s = self.enum_value_map['sc'][v]
-            lbl = 'p'
-            neg = randint(0,1)
-            if neg == 1:
-                s = uset_complement(s)
-                lbl = 'P'
-            terms.append(template % (lbl, v, uset_popcount(uset_intersection(self.all_good_set, s))))
-        for v in self.property_value_list['sc']:
-            s = self.enum_value_map['scx'][v]
-            lbl = 'p'
-            neg = randint(0,1)
-            if neg == 1:
-                s = uset_complement(s)
-                lbl = 'P'
-            terms.append(template % (lbl, "scx=" + v, uset_popcount(uset_intersection(self.all_good_set, s))))
+        if 'others' in propgroups:
+            for p in self.others:
+                s = self.binary_value_map[p]
+                lbl = 'p'
+                if randint(1,10) <= negated_per_10:
+                    s = uset_complement(s)
+                    lbl = 'P'
+                terms.append(template % (lbl, p, uset_popcount(uset_intersection(self.all_good_set, s))))
+        if 'gc' in propgroups:
+            for v in self.property_value_list['gc']:
+                s = self.enum_value_map['gc'][v]
+                lbl = 'p'
+                if randint(1,10) <= negated_per_10:
+                    s = uset_complement(s)
+                    lbl = 'P'
+                terms.append(template % (lbl, v, uset_popcount(uset_intersection(self.all_good_set, s))))
+        if 'sc' in propgroups:
+            for v in self.property_value_list['sc']:
+                s = self.enum_value_map['sc'][v]
+                vname = self.property_value_full_name_map['sc'][v]
+                lbl = 'p'
+                if randint(1,10) <= negated_per_10:
+                    s = uset_complement(s)
+                    lbl = 'P'
+                terms.append(template % (lbl, vname, uset_popcount(uset_intersection(self.all_good_set, s))))
+        if 'scx' in propgroups:
+            for v in self.property_value_list['sc']:
+                s = self.enum_value_map['scx'][v]
+                vname = self.property_value_full_name_map['sc'][v]
+                lbl = 'p'
+                if randint(1,10) <= negated_per_10:
+                    s = uset_complement(s)
+                    lbl = 'P'
+                terms.append(template % (lbl, "scx=" + vname, uset_popcount(uset_intersection(self.all_good_set, s))))
         return terms
 
-    def random_binary(self, a1, a2):
-        (p1, t1) = a1
-        (p2, t2) = a2
-        neg1 = randint(0,1)
-        neg2 = randint(0,1)
-        op = randint(0,2)
-        s1 = self.enum_value_map[p1][t1]
-        if p2 == 'others': 
+    def random_binary(self, a1, a2, useLookbehindAssertions = False):
+        (p1, t1) = a1
+        (p2, t2) = a2
+        op = randint(0,2)
+        s1 = self.enum_value_map[p1][t1]
+        if p2 == 'others':
             s2 = self.binary_value_map[t2]
         else: s2 = self.enum_value_map[p2][t2]
-        if neg1 == 1: s1 = uset_complement(s1)
-        if neg2 == 1: s2 = uset_complement(s2)
         if op == 0: s3 = uset_intersection(s1, s2)
         elif op == 1: s3 = uset_difference(s1, s2)
         elif op == 2: s3 = uset_union(s1, s2)
         s3 = uset_intersection(s3, self.all_good_set)
+        if p1 == 'sc' or p1 == 'scx': t1 = self.property_value_full_name_map['sc'][t1]
+        if p2 == 'sc' or p2 == 'scx': t2 = self.property_value_full_name_map['sc'][t2]
         if p1 == 'scx': t1 = 'scx=' + t1
         if p2 == 'scx': t2 = 'scx=' + t2
-        v1 = "\\%s{%s}" % (['p', 'P'][neg1], t1)
-        v2 = "\\%s{%s}" % (['p', 'P'][neg2], t2)
-        opr = ["&amp;&amp;", "--", ""][op]
-        return r"""<grepcase regexp="^[%s%s%s]$" datafile="All_good" grepcount="%i"/>""" % (v1, opr, v2, uset_popcount(s3))
+        v1 = "\\p{%s}" % (t1)
+        v2 = "\\p{%s}" % (t2)
+        if not useLookbehindAssertions:
+            opr = ["&amp;&amp;", "--", ""][op]
+            return r"""<grepcase regexp="^[%s%s%s]$" datafile="All_good" grepcount="%i"/>""" % (v1, opr, v2, uset_popcount(s3))
+        if op == 0:
+            return r"""<grepcase regexp="^%s(?&lt;=%s)$" datafile="All_good" grepcount="%i"/>""" % (v1, v2, uset_popcount(s3))
+        elif op == 1:
+            return r"""<grepcase regexp="^%s(?&lt;!%s)$" datafile="All_good" grepcount="%i"/>""" % (v1, v2, uset_popcount(s3))
+        else:
+            return r"""<grepcase regexp="^[%s%s]$" datafile="All_good" grepcount="%i"/>""" % (v1, v2, uset_popcount(s3))
 
-    def generate_random_property_expressions(self):
+    def generate_random_property_expressions(self, useLookbehindAssertions = False):
         gc = self.property_value_list['gc']
         sc = self.property_value_list['sc']
@@ -153 +159 @@
         for p in gc:
            s = sc[randint(0, len(sc)-1)]
-           exprs.append(self.random_binary(('gc', p), ('sc', s)))           
-           sx = sc[randint(0, len(sc)-1)]
-           exprs.append(self.random_binary(('gc', p), ('scx', sx)))
-           othr = others[randint(0, len(others)-1)]
-           exprs.append(self.random_binary(('gc', p), ('others', othr)))
+           exprs.append(self.random_binary(('gc', p), ('sc', s), useLookbehindAssertions))           
+           #sx = sc[randint(0, len(sc)-1)]
+           #exprs.append(self.random_binary(('gc', p), ('scx', sx), useLookbehindAssertions))
+           #othr = others[randint(0, len(others)-1)]
+           #exprs.append(self.random_binary(('gc', p), ('others', othr), useLookbehindAssertions))
         for p in sc:
            g = gc[randint(0, len(gc)-1)]
-           exprs.append(self.random_binary(('sc', p), ('gc', g)))           
-           sx = sc[randint(0, len(sc)-1)]
-           exprs.append(self.random_binary(('sc', p), ('scx', sx)))
-           othr = others[randint(0, len(others)-1)]
-           exprs.append(self.random_binary(('sc', p), ('others', othr)))
-        for p in others:
-           s = sc[randint(0, len(sc)-1)]
-           exprs.append(self.random_binary(('sc', s), ('others', p)))
-           sx = sc[randint(0, len(sc)-1)]
-           exprs.append(self.random_binary(('scx', sx), ('others', p)))
+           exprs.append(self.random_binary(('sc', p), ('gc', g), useLookbehindAssertions))           
+           #sx = sc[randint(0, len(sc)-1)]
+           #exprs.append(self.random_binary(('sc', p), ('scx', sx), useLookbehindAssertions))
+           #othr = others[randint(0, len(others)-1)]
+           #exprs.append(self.random_binary(('sc', p), ('others', othr), useLookbehindAssertions))
+        #for p in others:
+           #s = sc[randint(0, len(sc)-1)]
+           #exprs.append(self.random_binary(('sc', s), ('others', p), useLookbehindAssertions))
+           #sx = sc[randint(0, len(sc)-1)]
+           #exprs.append(self.random_binary(('scx', sx), ('others', p), useLookbehindAssertions))
         return exprs
 
@@ -175 +181 @@
     ucd = UCD_test_generator()
     ucd.load_all()
-    for t in ucd.generate_level_1_property_terms():
+    print "<greptest>"
+    for t in ucd.generate_level_1_property_terms(1, ['sc', 'gc']):
         print t
-    for p in ucd.generate_random_property_expressions():
+    for p in ucd.generate_random_property_expressions(True):
         print p
+    print "</greptest>"
 
 if __name__ == "__main__":