source: proto/charsetcompiler/UCD/UCD_properties.py @ 4174

#
# UCD_properties.py - parsing Unicode Character Database (UCD) files
# and generating C headers for property data using a compact bitset
# representation.
#
# Robert D. Cameron
# September 10, 2014
#
# Licensed under Open Software License 3.0.
#
#
import re, string
from unicode_set import *

UCD_dir = "7.0.0"


#
#  Processing files of the UCD
#
#  General format for skippable comments, blank lines
UCD_skip = re.compile("^#.*$|^\s*$")

#
#  UCD Property File Format 1: property aliases
#  PropertyAliases.txt
#
UCD_property_alias_regexp = re.compile("^([-A-Za-z_0-9]+)\s*;\s*([-A-Za-z_0-9]+)([^#]*)")

def parse_PropertyAlias_txt():
   property_enum_name_list = []
   full_name_map = {}
   property_lookup_map = {}
   f = open(UCD_dir + "/" + 'PropertyAliases.txt')
   lines = f.readlines()
   for t in lines:
      if UCD_skip.match(t): continue  # skip comment and blank lines
      m = UCD_property_alias_regexp.match(t)
      if not m: raise Exception("Unknown property alias syntax: %s" % t)
      prop_enum = m.group(1)
      prop_preferred_full_name = m.group(2)
      prop_extra = m.group(3)
      prop_aliases = re.findall("[-A-Za-z_0-9]+", prop_extra)
      property_enum_name_list.append(prop_enum)
      full_name_map[prop_enum] = prop_preferred_full_name
      property_lookup_map[canonicalize(prop_enum)] = prop_enum
      property_lookup_map[canonicalize(prop_preferred_full_name)] = prop_enum
      for a in prop_aliases: property_lookup_map[canonicalize(a)] = prop_enum
   return (property_enum_name_list, full_name_map, property_lookup_map)

trivial_name_char_re = re.compile('[-_\s]')
def canonicalize(property_string):
   c = trivial_name_char_re.sub('', property_string.lower())
   if len(c) > 2 and c[0:2] == "is": return c[2:]
   else: return c


header_template = r"""#ifndef %s
#define %s
/*
 *  Copyright (c) 2014 International Characters, Inc.
 *  This software is licensed to the public under the Open Software License 3.0.
 *  icgrep is a trademark of International Characters, Inc.
 *
 *  This file is generated by UCD_properties.y - manual edits may be lost.
 */

#include <string>
#include <unordered_map>

"""

def open_header_file_for_write(filename):
   f = open(filename + '.h', 'w')
   hname = filename.upper() + '_H'
   f.write(header_template % (hname, hname))
   return f

def close_header_file(f):
   f.write("#endif\n")
   f.close()

PropertyAliases_template = r"""
namespace UCD {
  enum property_t {
%s
  };
  const std::string property_full_name[] = {
%s
  };
  const std::unordered_map<std::string, property_t> alias_map = {
%s
  };
}
"""

def multiline_join(item_list, items_per_line, separator = ",", closer='', indent = 4):
  lines = ""
  sep_with_space = separator + " "
  while len(item_list) > items_per_line:
    line_items = item_list[:items_per_line]
    lines += (" " * indent) + sep_with_space.join(line_items) + separator + "\n"
    item_list = item_list[items_per_line:]
  lines += (" " * indent) + sep_with_space.join(item_list) + closer
  return lines

enums_per_line = 4
def generate_PropertyAliases_h():
   (property_enum_name_list, full_name_map, property_lookup_map) = parse_PropertyAlias_txt()
   f = open_header_file_for_write('PropertyAliases')
   enum_text = multiline_join([e.lower() for e in property_enum_name_list], enums_per_line, ',')
   full_name_text = multiline_join(['"%s"' % full_name_map[e] for e in property_enum_name_list], 2, ',')
   map_text = multiline_join(['{"%s", %s}' % (k, property_lookup_map[k].lower()) for k in sorted(property_lookup_map.keys())], 2, ',')
   f.write(PropertyAliases_template % (enum_text, full_name_text, map_text))
   close_header_file(f)
   
#
#  UCD Property File Format 2: property value aliases
#  PropertyValueAliases.txt
#
#  This file records value aliases for property values for
#  each enumerated property, with the following additional notes:
#  (1) The corresponding integer value of the enum constant is
#      also specified for ccc (second field).
#  (2) The Age property is a numeric type which has decimal float
#      values as the enum constants: these won't be legal in enum syntax.
#  (3) Binary properties also have enumerated values and aliases listed,
#      although this is redundant, because all binary properties have the
#      same value space.
#

UCD_property_value_alias_regexp = re.compile("^([-A-Za-z_0-9.]+)\s*;\s*([-A-Za-z_0-9.]+)\s*;\s*([-A-Za-z_0-9.]+)([^#]*)")

def parse_PropertyValueAlias_txt():
    property_value_list = {}
    property_value_enum_integer = {}
    property_value_full_name_map = {}
    property_value_lookup_map = {}
    f = open(UCD_dir + "/" + 'PropertyValueAliases.txt')
    lines = f.readlines()
    for t in lines:
        if UCD_skip.match(t): continue  # skip comment and blank lines
        m = UCD_property_value_alias_regexp.match(t)
        if not m: raise Exception("Unknown property value alias syntax: %s" % t)
        prop_code = m.group(1)
        if not property_value_list.has_key(prop_code):
          property_value_list[prop_code] = []
          property_value_enum_integer[prop_code] = {}
          property_value_full_name_map[prop_code] = {}
          property_value_lookup_map[prop_code] = {}
          enum_integer = 0
        # Special case for ccc: second field is enum integer value
        if prop_code == 'ccc':
          enum_integer = int(m.group(2))
          value_enum = m.group(3)
          extra = m.group(4)
          extra_list = re.findall("[-A-Za-z_0-9.]+", extra)
          value_preferred_full_name = extra_list[0]
          value_aliases = extra_list[1:]
        # Special case for age: second field is numeric, third field is enum
        # treat numeric value as an alias string
        elif prop_code == 'age':
          value_enum = m.group(3)
          value_preferred_full_name = m.group(3)
          extra = m.group(4)
          value_aliases = [m.group(2)] + re.findall("[-A-Za-z_0-9]+", extra)
        else:
          value_enum = m.group(2)
          value_preferred_full_name = m.group(3)
          extra = m.group(4)
          value_aliases = re.findall("[-A-Za-z_0-9]+", extra)
        property_value_list[prop_code].append(value_enum)
        property_value_enum_integer[prop_code][value_enum] = enum_integer
        enum_integer += 1
        property_value_full_name_map[prop_code][value_enum] = value_preferred_full_name
        property_value_lookup_map[prop_code][canonicalize(value_enum)] = value_enum
        property_value_lookup_map[prop_code][canonicalize(value_preferred_full_name)] = value_enum
        for a in value_aliases: property_value_lookup_map[prop_code][canonicalize(a)] = value_enum
    return (property_value_list, property_value_enum_integer, property_value_full_name_map, property_value_lookup_map)


PropertyValueAliases_template = r"""
namespace UCD {
  namespace %s {
    enum value_t {
%s
    };
    const std::string value_name[] = {
%s
    };
    const std::unordered_map<std::string, value_t> alias_map = {
%s
    };
  }
}
"""

def generate_PropertyValueAliases_h():
   (property_enum_name_list, full_name_map, property_lookup_map) = parse_PropertyAlias_txt()
   (property_value_list, property_value_enum_integer, property_value_full_name_map, property_value_lookup_map) = parse_PropertyValueAlias_txt()
   f = open_header_file_for_write('PropertyValueAliases')
   #  Generate the aliases for all Binary properties.
   enum_text = multiline_join(['N', 'Y'], 4, ',','', 6)
   full_name_text = multiline_join(['"No"', '"Yes"'], 4, ',',  '', 6)
   map_text = multiline_join(['{"n", N}', '{"y", Y}', '{"no", N}', '{"yes", Y}', '{"f", N}', '{"t", Y}', '{"false", N}', '{"true", Y}'], 4, ',', '', 6)
   f.write(PropertyValueAliases_template % ('Binary', enum_text, full_name_text, map_text))
   #
   for p in property_enum_name_list:
     if property_value_list.has_key(p):
       if property_value_list[p] == ['N', 'Y']: continue # skip boolean properties
       enum_text = multiline_join(property_value_list[p], 4, ',','', 6)
       if p == 'ccc': # Special case: add numeric value information for ccc.
         enum_text += r"""
    };
    const uint8_t enum_val[] = {
"""
         enum_text += multiline_join(["%s" % (property_value_enum_integer[p][e]) for e in property_value_list['ccc']], 4, ',', '', 6)
       full_name_text = multiline_join(['"%s"' % (property_value_full_name_map[p][e]) for e in property_value_list[p]], 4, ',',  '', 6)
       map_text = multiline_join(['{"%s", %s}' % (k, property_value_lookup_map[p][k]) for k in sorted(property_value_lookup_map[p].keys())], 4, ',', '', 6)
       f.write(PropertyValueAliases_template % (p.upper(), enum_text, full_name_text, map_text))
   close_header_file(f)
   


#
#  UCD Property File Format 3:  codepoint -> name maps
# 
UCD_skip = re.compile("^#.*$|^\s*$")
UCD_point_name_regexp = re.compile("^([0-9A-F]{4,6})\s*;\s*((?:[-A-Za-z0-9_]+\s+)*[-A-Za-z0-9_]+)\s*(?:[;#]|$)")
UCD_range_name_regexp = re.compile("^([0-9A-F]{4,6})[.][.]([0-9A-F]{4,6})\s*;\s*((?:[-A-Za-z0-9_]+\s+)*[-A-Za-z0-9_]+)\s*(?:[;#]|$)")

def parse_UCD_codepoint_name_map(mapfile):
   name_map = {}
   name_list_order = []
   f = open(UCD_dir + "/" + mapfile)
   lines = f.readlines()
   for t in lines:
      if UCD_skip.match(t): continue  # skip comment and blank lines
      m = UCD_point_name_regexp.match(t)
      if m:
        (codepoint, name) = (int(m.group(1), 16), m.group(2))
        newset = singleton_set(codepoint)
      else: 
        m = UCD_range_name_regexp.match(t)
        if not m: raise Exception("Unknown syntax: %s" % t)
        (cp_lo, cp_hi, name) = (int(m.group(1), 16), int(m.group(2), 16), m.group(3))
        newset = make_range_set(cp_lo, cp_hi)
      if not name_map.has_key(name):
        name_map[name] = newset
        name_list_order.append(name)
      else: name_map[name] = union(name_map[name], newset)
   return (name_list_order, name_map)

def generate_PropList_h():
   (props, prop_map) = parse_UCD_codepoint_name_map('PropList.txt')
   f = open_header_file_for_write('PropList')
   for k in props:
     f.write(prop_map[k].showC(k))
   close_header_file(f)

def generate_Blocks_h():
   (blocks, block_map) = parse_UCD_codepoint_name_map('Blocks.txt')
   f = open_header_file_for_write('Blocks')
   for k in blocks:
     f.write(block_map[k].showC('block["%s"]' % k))
   close_header_file(f)

def generate_Scripts_h():
   (scripts, script_map) = parse_UCD_codepoint_name_map('Scripts.txt')
   f = open_header_file_for_write('Scripts')
   for k in scripts:
     f.write(script_map[k].showC('script["%s"]' % k))
   close_header_file(f)
   
def generate_ScriptExtensions_h():
   (scx_sets, scx_map) = parse_UCD_codepoint_name_map('ScriptExtensions.txt')
   map2 = {}
   f = open_header_file_for_write('ScriptExtensions')
   for scx_list in scx_sets:
     scx_items = scx_list.split(" ")
     for scx in scx_items:
        if map2.has_key(scx): 
           map2[scx] = union(map2[scx], scx_map[scx_list])
        else: map2[scx] = scx_map[scx_list]
   for k in sorted(map2.keys()):
     f.write(map2[k].showC('scx["%s"]' % k))
   close_header_file(f)

def generate_DerivedGeneralCategory_h():
   (categories, cat_map) = parse_UCD_codepoint_name_map('extracted/DerivedGeneralCategory.txt')
   f = open_header_file_for_write('DerivedGeneralCategory')
   for k in categories:
     f.write(cat_map[k].showC('GC["%s"]' % k))
   close_header_file(f)

def generate_DerivedCoreProperties_h():
   (properties, prop_map) = parse_UCD_codepoint_name_map('DerivedCoreProperties.txt')
   f = open_header_file_for_write('DerivedCoreProperties')
   for k in properties:
     f.write(prop_map[k].showC(k))
   close_header_file(f)