source: proto/JSON/u8u16.py @ 682

Last change on this file since 682 was 682, checked in by ksherdy, 9 years ago

Add UTF8 validation. Add UTF8 character set definitions.

File size: 7.4 KB
Line 
1#!/usr/bin/python
2# -*- coding: utf-8 -*-
3
4# u8u16.py
5#
6# Python prototype implementation
7# Robert D. Cameron
8# Revised October 29, 2009 - Make consistent with character compiler UTF8 definition variable names.
9#
10#----------------------------------------------------------------------------
11#
12# We use python's unlimited precision integers for unbounded bit streams.
13# This permits simple logical operations on the entire stream.
14# Assumption: bitstreams are little-endian (e.g., as on x86).
15#
16#----------------------------------------------------------------------------
17#
18
19import bitutil
20
21import byteclass
22
23def ShiftBack(stream):
24        return stream >> 1
25
26def validate_utf8(u8): 
27        u8.scope22 = bitutil.Advance(u8.prefix2)
28        u8.scope32 = bitutil.Advance(u8.prefix3)
29        u8.scope33 = bitutil.Advance(u8.scope32)
30        u8.scope42 = bitutil.Advance(u8.prefix4)
31        u8.scope43 = bitutil.Advance(u8.scope42)
32        u8.scope44 = bitutil.Advance(u8.scope43)
33               
34        u8lastscope = u8.scope22 | u8.scope33 | u8.scope44
35        u8anyscope = u8lastscope | u8.scope32 | u8.scope42 | u8.scope43
36             
37        # C0-C1 and F5-FF are illegal
38        error_mask = u8.badprefix
39             
40        error_mask |= bitutil.Advance(u8.xE0) & u8.x80_x9F
41        error_mask |= bitutil.Advance(u8.xED) & u8.xA0_xBF
42        error_mask |= bitutil.Advance(u8.xF0) & u8.x80_x8F
43        error_mask |= bitutil.Advance(u8.xF4) & u8.x90_xBF
44             
45        error_mask |= u8anyscope ^ u8.suffix
46        u8.error = error_mask   
47        return u8
48
49#
50# The following calculation of UTF-16 bit streams is consistent
51# with the original u8u16, calculating streams at u8scope42 and
52# u8scope44 positions.
53#
54
55def u8_u16_old(u8, u8bit):
56        u16hi = [0,0,0,0,0,0,0,0]
57        u16lo = [0,0,0,0,0,0,0,0]
58       
59        u8lastscope = u8.scope22 | u8.scope33 | u8.scope44
60        u8lastbyte = u8.unibyte | u8lastscope
61        u16lo[2] = u8lastbyte & u8bit[2]
62        u16lo[3] = u8lastbyte & u8bit[3]
63        u16lo[4] = u8lastbyte & u8bit[4]
64        u16lo[5] = u8lastbyte & u8bit[5]
65        u16lo[6] = u8lastbyte & u8bit[6]
66        u16lo[7] = u8lastbyte & u8bit[7]
67        u16lo[1] = (u8.unibyte & u8bit[1]) | (u8lastscope & bitutil.Advance(u8bit[7]))
68        u16lo[0] = u8lastscope & bitutil.Advance(u8bit[6])
69       
70        u16hi[5] = u8lastscope & bitutil.Advance(u8bit[3])
71        u16hi[6] = u8lastscope & bitutil.Advance(u8bit[4])
72        u16hi[7] = u8lastscope & bitutil.Advance(u8bit[5])
73        u16hi[0] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[4]))
74        u16hi[1] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[5]))
75        u16hi[2] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[6]))
76        u16hi[3] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[7]))
77        u16hi[4] = u8.scope33 & bitutil.Advance(u8bit[2])
78
79        u8surrogate = u8.scope42 | u8.scope44
80        u16hi[0] = u16hi[0] | u8surrogate       
81        u16hi[1] = u16hi[1] | u8surrogate       
82        u16hi[3] = u16hi[3] | u8surrogate       
83        u16hi[4] = u16hi[4] | u8surrogate       
84        u16hi[5] = u16hi[5] | u8.scope44
85
86        s42lo1 = ~u8bit[3] # subtract 1
87        u16lo[1] = u16lo[1] | (u8.scope42 & s42lo1)
88        s42lo0 = u8bit[2] ^ s42lo1 # borrow *
89        u16lo[0] = u16lo[0] | (u8.scope42 & s42lo0)
90        borrow1 = s42lo1 & ~u8bit[2]
91        s42hi7 = bitutil.Advance(u8bit[7]) ^ borrow1
92        u16hi[7]= u16hi[7] | (u8.scope42 & s42hi7)
93        borrow2 = borrow1 & ~bitutil.Advance(u8bit[7])
94        s42hi6 = bitutil.Advance(u8bit[6]) ^ borrow2
95        u16hi[6] = u16hi[6] | (u8.scope42 & s42hi6)
96
97        u16lo[2] = u16lo[2] | (u8.scope42 & u8bit[4])
98        u16lo[3] = u16lo[3] | (u8.scope42 & u8bit[5])
99        u16lo[4] = u16lo[4] | (u8.scope42 & u8bit[6])
100        u16lo[5] = u16lo[5] | (u8.scope42 & u8bit[7])
101        u16lo[6] = u16lo[6] | (u8.scope42 & ShiftBack(u8bit[2]))
102        u16lo[7] = u16lo[7] | (u8.scope42 & ShiftBack(u8bit[3]))
103
104        delmask = u8.prefix | u8.scope32 | u8.scope43
105        return (u16hi, u16lo, delmask)
106
107
108
109
110#
111# The following calculation of UTF-16 bit streams uses the
112# u8scope43 position rather than the u8scope42 position for
113# the bits of the first UTF-16 code unit of a surrogate pair.
114# This requires more shifting than with the use of u8scope42,
115# but has the advantage that all shifts are in the forward
116# direction only and can hence be implemented using addition
117# on little-endian architecture.
118#
119
120def u8u16(u8, u8bit):
121        u16hi = [0,0,0,0,0,0,0,0]
122        u16lo = [0,0,0,0,0,0,0,0]
123       
124        u8lastscope = u8.scope22 | u8.scope33 | u8.scope44
125        u8lastbyte = u8.unibyte | u8lastscope
126        u16lo[2] = u8lastbyte & u8bit[2]
127        u16lo[3] = u8lastbyte & u8bit[3]
128        u16lo[4] = u8lastbyte & u8bit[4]
129        u16lo[5] = u8lastbyte & u8bit[5]
130        u16lo[6] = u8lastbyte & u8bit[6]
131        u16lo[7] = u8lastbyte & u8bit[7]
132        u16lo[1] = (u8.unibyte & u8bit[1]) | (u8lastscope & bitutil.Advance(u8bit[7]))
133        u16lo[0] = u8lastscope & bitutil.Advance(u8bit[6])
134       
135        u16hi[5] = u8lastscope & bitutil.Advance(u8bit[3])
136        u16hi[6] = u8lastscope & bitutil.Advance(u8bit[4])
137        u16hi[7] = u8lastscope & bitutil.Advance(u8bit[5])
138        u16hi[0] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[4]))
139        u16hi[1] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[5]))
140        u16hi[2] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[6]))
141        u16hi[3] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[7]))
142        u16hi[4] = u8.scope33 & bitutil.Advance(u8bit[2])
143
144        u8surrogate = u8.scope43 | u8.scope44
145        u16hi[0] = u16hi[0] | u8surrogate       
146        u16hi[1] = u16hi[1] | u8surrogate       
147        u16hi[3] = u16hi[3] | u8surrogate       
148        u16hi[4] = u16hi[4] | u8surrogate       
149        u16hi[5] = u16hi[5] | u8.scope44
150
151
152        s42lo1 = ~u8bit[3] # subtract 1
153        u16lo[1] = u16lo[1] | (u8.scope43 & bitutil.Advance(s42lo1))
154        s42lo0 = u8bit[2] ^ s42lo1 # borrow *
155        u16lo[0] = u16lo[0] | (u8.scope43 & bitutil.Advance(s42lo0))
156        borrow1 = s42lo1 & ~u8bit[2]
157        Advance_bit7 = bitutil.Advance(u8bit[7])
158        s42hi7 = Advance_bit7 ^ borrow1
159        u16hi[7]= u16hi[7] | (u8.scope43 & bitutil.Advance(s42hi7))
160        borrow2 = borrow1 & ~Advance_bit7
161        s42hi6 = bitutil.Advance(u8bit[6]) ^ borrow2
162        u16hi[6] = u16hi[6] | (u8.scope43 & bitutil.Advance(s42hi6))
163
164        u16lo[2] = u16lo[2] | (u8.scope43 & bitutil.Advance(u8bit[4]))
165        u16lo[3] = u16lo[3] | (u8.scope43 & bitutil.Advance(u8bit[5]))
166        u16lo[4] = u16lo[4] | (u8.scope43 & bitutil.Advance(u8bit[6]))
167        u16lo[5] = u16lo[5] | (u8.scope43 & bitutil.Advance(u8bit[7]))
168        u16lo[6] = u16lo[6] | (u8.scope43 & u8bit[2])
169        u16lo[7] = u16lo[7] | (u8.scope43 & u8bit[3])
170
171        delmask = u8.prefix | u8.scope32 | u8.scope42
172        return (u16hi, u16lo, delmask)
173
174
175
176#
177# Messages to duplicate u8u16 error reporting.
178#
179def IllegalSequenceMessage(pos):
180        return "Illegal UTF-8 sequence at position %i in source.\n" % pos
181
182def IncompleteSequenceMessage(pos):
183        return "EOF with incomplete UTF-8 sequence at position %i in source.\n" % pos
184
185
186import sys
187def main():
188
189        if len(sys.argv) < 2:
190                sys.stderr.write("Usage: u8u16.py u8file [u16file]\n")
191                exit
192        if len(sys.argv) == 3:
193                outfile = open(sys.argv[2],"w")
194        else: outfile = sys.stdout
195        u8data = bitutil.readfile(sys.argv[1])
196        u8len = len(u8data)
197        (u8bit, EOF_mask) = bitutil.transpose_streams(u8data)
198        (u8, control, lex) = byteclass.classify_bytes(u8bit)
199        u8 = validate_utf8(u8)
200        if u8.error != 0:
201                err_pos = bitutil.count_leading_zeroes(u8.error)
202                at_EOF = err_pos == len(u8data)
203                if (err_pos >= 1) and ord(u8data[err_pos-1]) >= 0xC0: err_pos -= 1
204                elif err_pos >= 2 and ord(u8data[err_pos-2]) >= 0xE0: err_pos -= 2
205                elif err_pos >= 3 and ord(u8data[err_pos-3]) >= 0xF0: err_pos -= 3     
206                if at_EOF:
207                        sys.stderr.write(IncompleteSequenceMessage(err_pos))
208                else:
209                        sys.stderr.write(IllegalSequenceMessage(err_pos))
210                u8len = err_pos
211        (u16hi, u16lo, delmask) = u8u16(u8, u8bit)
212        U16H = bitutil.filter_bytes(bitutil.inverse_transpose(u16hi, u8len), delmask)
213        U16L = bitutil.filter_bytes(bitutil.inverse_transpose(u16lo, u8len), delmask)
214        U16final = bitutil.merge_bytes(U16H, U16L)
215        outfile.write(U16final)
216        outfile.close()
217               
218if __name__ == "__main__": main()
219
220
Note: See TracBrowser for help on using the repository browser.