source: proto/parabix2/u8u16.py @ 314

Last change on this file since 314 was 314, checked in by ksherdy, 10 years ago

Move readfile to bitutil.py. Revert transpose_streams method name.

  • Property svn:executable set to *
File size: 7.3 KB
Line 
1#!/usr/bin/python
2
3# u8u16.py
4#
5# Python prototype implementation
6# Robert D. Cameron
7# revised August 20, 2009 - make consistent with parabix2 prototype
8#
9#----------------------------------------------------------------------------
10#
11# We use python's unlimited precision integers for unbounded bit streams.
12# This permits simple logical operations on the entire stream.
13# Assumption: bitstreams are little-endian (e.g., as on x86).
14#
15#----------------------------------------------------------------------------
16#
17
18import bitutil
19
20import byteclass
21
22def ShiftBack(stream):
23        return stream >> 1
24
25def validate_utf8(u8):
26        u8.scope22 = bitutil.Advance(u8.prefix2)
27        u8.scope32 = bitutil.Advance(u8.prefix3)
28        u8.scope33 = bitutil.Advance(u8.scope32)
29        u8.scope42 = bitutil.Advance(u8.prefix4)
30        u8.scope43 = bitutil.Advance(u8.scope42)
31        u8.scope44 = bitutil.Advance(u8.scope43)
32        u8lastscope = u8.scope22 | u8.scope33 | u8.scope44
33        u8anyscope = u8lastscope | u8.scope32 | u8.scope42 | u8.scope43
34       
35        # C0-C1 and F5-FF are illegal
36        error_mask = u8.badprefix
37       
38        error_mask |= bitutil.Advance(u8.xE0) & u8.x80_x9F
39        error_mask |= bitutil.Advance(u8.xED) & u8.xA0_xBF
40        error_mask |= bitutil.Advance(u8.xF0) & u8.x80_x8F
41        error_mask |= bitutil.Advance(u8.xF4) & u8.x90_xBF
42       
43        error_mask |= u8anyscope ^ u8.suffix
44        u8.error = error_mask
45        return u8
46
47
48#
49# The following calculation of UTF-16 bit streams is consistent
50# with the original u8u16, calculating streams at u8scope42 and
51# u8scope44 positions.
52#
53
54def u8_u16_old(u8, u8bit):
55        u16hi = [0,0,0,0,0,0,0,0]
56        u16lo = [0,0,0,0,0,0,0,0]
57       
58        u8lastscope = u8.scope22 | u8.scope33 | u8.scope44
59        u8lastbyte = u8.unibyte | u8lastscope
60        u16lo[2] = u8lastbyte & u8bit[2]
61        u16lo[3] = u8lastbyte & u8bit[3]
62        u16lo[4] = u8lastbyte & u8bit[4]
63        u16lo[5] = u8lastbyte & u8bit[5]
64        u16lo[6] = u8lastbyte & u8bit[6]
65        u16lo[7] = u8lastbyte & u8bit[7]
66        u16lo[1] = (u8.unibyte & u8bit[1]) | (u8lastscope & bitutil.Advance(u8bit[7]))
67        u16lo[0] = u8lastscope & bitutil.Advance(u8bit[6])
68       
69        u16hi[5] = u8lastscope & bitutil.Advance(u8bit[3])
70        u16hi[6] = u8lastscope & bitutil.Advance(u8bit[4])
71        u16hi[7] = u8lastscope & bitutil.Advance(u8bit[5])
72        u16hi[0] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[4]))
73        u16hi[1] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[5]))
74        u16hi[2] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[6]))
75        u16hi[3] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[7]))
76        u16hi[4] = u8.scope33 & bitutil.Advance(u8bit[2])
77
78        u8surrogate = u8.scope42 | u8.scope44
79        u16hi[0] = u16hi[0] | u8surrogate       
80        u16hi[1] = u16hi[1] | u8surrogate       
81        u16hi[3] = u16hi[3] | u8surrogate       
82        u16hi[4] = u16hi[4] | u8surrogate       
83        u16hi[5] = u16hi[5] | u8.scope44
84
85        s42lo1 = ~u8bit[3] # subtract 1
86        u16lo[1] = u16lo[1] | (u8.scope42 & s42lo1)
87        s42lo0 = u8bit[2] ^ s42lo1 # borrow *
88        u16lo[0] = u16lo[0] | (u8.scope42 & s42lo0)
89        borrow1 = s42lo1 & ~u8bit[2]
90        s42hi7 = bitutil.Advance(u8bit[7]) ^ borrow1
91        u16hi[7]= u16hi[7] | (u8.scope42 & s42hi7)
92        borrow2 = borrow1 & ~bitutil.Advance(u8bit[7])
93        s42hi6 = bitutil.Advance(u8bit[6]) ^ borrow2
94        u16hi[6] = u16hi[6] | (u8.scope42 & s42hi6)
95
96        u16lo[2] = u16lo[2] | (u8.scope42 & u8bit[4])
97        u16lo[3] = u16lo[3] | (u8.scope42 & u8bit[5])
98        u16lo[4] = u16lo[4] | (u8.scope42 & u8bit[6])
99        u16lo[5] = u16lo[5] | (u8.scope42 & u8bit[7])
100        u16lo[6] = u16lo[6] | (u8.scope42 & ShiftBack(u8bit[2]))
101        u16lo[7] = u16lo[7] | (u8.scope42 & ShiftBack(u8bit[3]))
102
103        delmask = u8.prefix | u8.scope32 | u8.scope43
104        return (u16hi, u16lo, delmask)
105
106
107
108
109#
110# The following calculation of UTF-16 bit streams uses the
111# u8scope43 position rather than the u8scope42 position for
112# the bits of the first UTF-16 code unit of a surrogate pair.
113# This requires more shifting than with the use of u8scope42,
114# but has the advantage that all shifts are in the forward
115# direction only and can hence be implemented using addition
116# on little-endian architecture.
117#
118
119def u8u16(u8, u8bit):
120        u16hi = [0,0,0,0,0,0,0,0]
121        u16lo = [0,0,0,0,0,0,0,0]
122       
123        u8lastscope = u8.scope22 | u8.scope33 | u8.scope44
124        u8lastbyte = u8.unibyte | u8lastscope
125        u16lo[2] = u8lastbyte & u8bit[2]
126        u16lo[3] = u8lastbyte & u8bit[3]
127        u16lo[4] = u8lastbyte & u8bit[4]
128        u16lo[5] = u8lastbyte & u8bit[5]
129        u16lo[6] = u8lastbyte & u8bit[6]
130        u16lo[7] = u8lastbyte & u8bit[7]
131        u16lo[1] = (u8.unibyte & u8bit[1]) | (u8lastscope & bitutil.Advance(u8bit[7]))
132        u16lo[0] = u8lastscope & bitutil.Advance(u8bit[6])
133       
134        u16hi[5] = u8lastscope & bitutil.Advance(u8bit[3])
135        u16hi[6] = u8lastscope & bitutil.Advance(u8bit[4])
136        u16hi[7] = u8lastscope & bitutil.Advance(u8bit[5])
137        u16hi[0] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[4]))
138        u16hi[1] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[5]))
139        u16hi[2] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[6]))
140        u16hi[3] = u8.scope33 & bitutil.Advance(bitutil.Advance(u8bit[7]))
141        u16hi[4] = u8.scope33 & bitutil.Advance(u8bit[2])
142
143        u8surrogate = u8.scope43 | u8.scope44
144        u16hi[0] = u16hi[0] | u8surrogate       
145        u16hi[1] = u16hi[1] | u8surrogate       
146        u16hi[3] = u16hi[3] | u8surrogate       
147        u16hi[4] = u16hi[4] | u8surrogate       
148        u16hi[5] = u16hi[5] | u8.scope44
149
150
151        s42lo1 = ~u8bit[3] # subtract 1
152        u16lo[1] = u16lo[1] | (u8.scope43 & bitutil.Advance(s42lo1))
153        s42lo0 = u8bit[2] ^ s42lo1 # borrow *
154        u16lo[0] = u16lo[0] | (u8.scope43 & bitutil.Advance(s42lo0))
155        borrow1 = s42lo1 & ~u8bit[2]
156        Advance_bit7 = bitutil.Advance(u8bit[7])
157        s42hi7 = Advance_bit7 ^ borrow1
158        u16hi[7]= u16hi[7] | (u8.scope43 & bitutil.Advance(s42hi7))
159        borrow2 = borrow1 & ~Advance_bit7
160        s42hi6 = bitutil.Advance(u8bit[6]) ^ borrow2
161        u16hi[6] = u16hi[6] | (u8.scope43 & bitutil.Advance(s42hi6))
162
163        u16lo[2] = u16lo[2] | (u8.scope43 & bitutil.Advance(u8bit[4]))
164        u16lo[3] = u16lo[3] | (u8.scope43 & bitutil.Advance(u8bit[5]))
165        u16lo[4] = u16lo[4] | (u8.scope43 & bitutil.Advance(u8bit[6]))
166        u16lo[5] = u16lo[5] | (u8.scope43 & bitutil.Advance(u8bit[7]))
167        u16lo[6] = u16lo[6] | (u8.scope43 & u8bit[2])
168        u16lo[7] = u16lo[7] | (u8.scope43 & u8bit[3])
169
170        delmask = u8.prefix | u8.scope32 | u8.scope42
171        return (u16hi, u16lo, delmask)
172
173
174
175#
176# Messages to duplicate u8u16 error reporting.
177#
178def IllegalSequenceMessage(pos):
179        return "Illegal UTF-8 sequence at position %i in source.\n" % pos
180
181def IncompleteSequenceMessage(pos):
182        return "EOF with incomplete UTF-8 sequence at position %i in source.\n" % pos
183
184
185import sys
186def main():
187
188        if len(sys.argv) < 2:
189                sys.stderr.write("Usage: u8u16.py u8file [u16file]\n")
190                exit
191        if len(sys.argv) == 3:
192                outfile = open(sys.argv[2],"w")
193        else: outfile = sys.stdout
194        u8data = bitutil.readfile(sys.argv[1])
195        u8len = len(u8data)
196        (u8bit, EOF_mask) = bitutil.transpose_streams(u8data)
197        (u8, control, lex) = byteclass.classify_bytes(u8bit)
198        u8 = validate_utf8(u8)
199        if u8.error != 0:
200                err_pos = bitutil.count_leading_zeroes(u8.error)
201                at_EOF = err_pos == len(u8data)
202                if (err_pos >= 1) and ord(u8data[err_pos-1]) >= 0xC0: err_pos -= 1
203                elif err_pos >= 2 and ord(u8data[err_pos-2]) >= 0xE0: err_pos -= 2
204                elif err_pos >= 3 and ord(u8data[err_pos-3]) >= 0xF0: err_pos -= 3     
205                if at_EOF:
206                        sys.stderr.write(IncompleteSequenceMessage(err_pos))
207                else:
208                        sys.stderr.write(IllegalSequenceMessage(err_pos))
209                u8len = err_pos
210        (u16hi, u16lo, delmask) = u8u16(u8, u8bit)
211        U16H = bitutil.filter_bytes(bitutil.inverse_transpose(u16hi, u8len), delmask)
212        U16L = bitutil.filter_bytes(bitutil.inverse_transpose(u16lo, u8len), delmask)
213        U16final = bitutil.merge_bytes(U16H, U16L)
214        outfile.write(U16final)
215        outfile.close()
216               
217if __name__ == "__main__": main()
218
219
Note: See TracBrowser for help on using the repository browser.