source: trunk/src/byteplex.h @ 169

Last change on this file since 169 was 169, checked in by lindanl, 11 years ago

UTF-16/32 validation.

File size: 7.6 KB
Line 
1/*  byteplex.h - Parallel byte stream module.
2    Copyright (c) 2008,  Robert D. Cameron.
3    Licensed to the public under the Open Software License 3.0.
4    Licensed to International Characters, Inc., under the Academic
5    Free License 3.0.
6
7    This module has as its goal the buffering of XML byte data and
8    transformation of 16-bit and 32-bit code unit data so that the
9    parsing engine is provided a uniform representation based on
10    the concept of an 8-bit pseudo-ASCII representation (x8data).
11
12    A Byteplex object provides buffers for one to six parallel data
13    streams based for an XML input entity, depending on the size of
14    character code units. 
15       1.  In the case of 8-bit code units, a single byte stream
16           consisting of unmodified input data is maintained.
17           x8data = src_buffer
18       2.  In the case of 16-bit code units (UTF-16 and UCS-2 families),
19             (a) the original code unit stream is maintained unmodified,
20             (b) the x16hi byte stream is established for the high byte
21                 of each code unit,
22             (c) the x16lo byte stream is established for the low byte
23                 of each code unit, and
24             (d) x8data is established as the pseudo-ASCII byte stream,
25                 with ASCII code units having their proper 8-bit values,
26                 and all others having bit 0 set to 1.
27       3.  In the case of 32-bit code units (UTF-32 family),
28             (a) the original code unit stream is maintained unmodified,
29             (b) the x32hh byte stream has high bytes of each code unit
30             (c) the x32hl byte stream has second bytes of each code unit
31             (d) the x32lh byte stream has third bytes of each code unit
32             (e) the x32hh byte stream has low bytes of each code unit, and
33             (f) x8data is established as the pseudo-ASCII byte stream,
34                 with ASCII code units having their proper 8-bit values,
35                 and all others having bit 0 set to 1.
36
37    The pseudo-ASCII representation is defined for both ASCII-based
38    and EBCDIC-based character sets such that all characters in
39    the ASCII repertoire (i.e., having Unicode code points from 0x00
40    to 0x7F), are represented as themselves and no non-ASCII character
41    is represented as a character in the ASCII repertoire.
42
43*/
44
45#ifndef BYTEPLEX_H
46#define BYTEPLEX_H
47
48#include "xmldecl.h"
49#include "../lib/lib_simd.h"
50
51/* The BytePack and the BitBlock are the two fundamental
52   types used by the parabix program for data held in
53   SIMD registers, representing, respectively, the byte-oriented
54   and bit-oriented views of character data.*/
55
56typedef SIMD_type BytePack;
57typedef SIMD_type BitBlock;
58const int PACKSIZE = sizeof(SIMD_type);
59const int BLOCKSIZE = sizeof(SIMD_type) * 8;
60
61/* Define the size of buffer used for lexical analysis/parsing. */
62const int BUFFER_BLOCKS = 781;
63const int BUFFER_SIZE = BUFFER_BLOCKS * BLOCKSIZE;
64
65/* When working near the end of a buffer, a bytespace test may involve
66   a multibyte literal.  The bytespace buffer must always make available
67   a number of lookahead bytes at least equal to the maximum length of any
68   such literal. */
69
70const int LOOKAHEAD_POSITIONS = 16;
71const int BYTEPLEX_SIZE = BUFFER_SIZE + LOOKAHEAD_POSITIONS;
72
73class Byteplex {
74public:
75        static Byteplex * ByteplexFactory(Entity_Info * e);
76        static Byteplex * ByteplexFactory(Entity_Info * e, FILE * inputfile);
77        static Byteplex * ByteplexFactory(Entity_Info * e, unsigned char * buffer_bytes, int buffer_size);
78        virtual void DoByteplex() = 0;
79        virtual void PreparePseudoASCII_Stream() = 0;
80        virtual void InitializeBuffer(unsigned char * src, int lgth) = 0;
81        virtual void AdvanceInputBuffer(int advance_amt) = 0;
82        virtual int UTF8_Length(int name_pos, int lgth)=0;
83        virtual void to_UTF8(int name_pos, int lgth, char * u8_ptr)=0;
84        /* Source code unit buffer. */
85        BytePack * src_buffer;
86        int units_in_buffer;
87
88        /* Pseudo-ASCII stream. */
89        BytePack * x8data;
90
91protected:
92        FILE * infile;
93        int packs_in_buffer;
94        int CopyAndFill(unsigned char * bytes_to_copy, int lgth, int bytes_to_read);
95        void Set_limits(int units_in_buffer);
96
97};
98
99
100/*  The X8_Buffer template class is used for either ASCII- or EBCDIC-
101    based 8-bit code units.
102    The X8_Buffer<ASCII> class includes 7-bit ASCII
103    (with high-order bit 0), the ISO-8859 character sets and UTF-8.
104
105    The family of 8-bit EBCDIC based character sets are processed using
106    the X8_Buffer<EBCDIC> class.
107*/
108
109template <CodeUnit_Base C>
110class X8_Buffer : public Byteplex {
111public:
112        static const CodeUnit_Base Base = C;
113        static const CodeUnit_Size Size = SingleByte;
114        X8_Buffer();
115        ~X8_Buffer();
116       
117        void DoByteplex();
118        void PreparePseudoASCII_Stream();
119        void AdvanceInputBuffer(int advance_amt);
120        void InitializeBuffer(unsigned char * src, int lgth);
121        int UTF8_Length(int name_pos, int lgth);
122        void to_UTF8(int name_pos, int lgth, char * u8_ptr);
123};
124
125class UTF8_Buffer : public Byteplex {
126public:
127        static const CodeUnit_Base Base = ASCII;
128        static const CodeUnit_Size Size = SingleByte;
129        UTF8_Buffer();
130        ~UTF8_Buffer();
131       
132        void DoByteplex();
133        void PreparePseudoASCII_Stream();
134        void AdvanceInputBuffer(int advance_amt);
135        void InitializeBuffer(unsigned char * src, int lgth);
136        int UTF8_Length(int name_pos, int lgth);
137        void to_UTF8(int name_pos, int lgth, char * u8_ptr);
138};
139
140
141/*  UTF-16 and UCS-2 character set families in BE and LE byte orders.
142    The U16LE and U16BE subclasses each provide a distinct byteplexer to
143    produce 2 parallel byte streams for the high and low bytes of each
144    16-bit code unit.  Once byteplexing is complete, a generic pseudoASCII
145    conversion routine can be applied at the U16_Buffer level. */
146
147class U16_Buffer : public Byteplex {
148public:
149        static const CodeUnit_Base Base = ASCII;
150        static const CodeUnit_Size Size = DoubleByte;
151        U16_Buffer();
152        ~U16_Buffer();
153        virtual void DoByteplex() = 0;
154        void PreparePseudoASCII_Stream();
155        void AdvanceInputBuffer(int advance_amt);
156        void Validate_UTF16();
157        void Validate_UCS2();
158        void InitializeBuffer(unsigned char * src, int lgth);
159        int UTF8_Length(int name_pos, int lgth);
160        void to_UTF8(int name_pos, int lgth, char * u8_ptr);
161protected:
162        BytePack * x16hi;
163        BytePack * x16lo;
164};
165
166class U16LE_Buffer : public U16_Buffer {
167public:
168        U16LE_Buffer();
169        void DoByteplex();
170};
171
172class U16BE_Buffer : public U16_Buffer {
173public:
174        U16BE_Buffer();
175        void DoByteplex();
176};
177
178
179/*  UTF-32/UCS-4 character sets in BE, LE, 2143 and 3412 byte orders.
180    Each subclass of U32_Buffer provide a distinct byteplexer to
181    produce the 4 parallel byte streams of Unicode data.  Once
182    byteplexing is complete, a generic pseudoASCII routine can
183    be applied. */
184class U32_Buffer : public Byteplex {
185public:
186        static const CodeUnit_Base Base = ASCII;
187        static const CodeUnit_Size Size = QuadByte;
188        U32_Buffer();
189        ~U32_Buffer();
190        virtual void DoByteplex() = 0;
191        void PreparePseudoASCII_Stream();
192        void AdvanceInputBuffer(int advance_amt);
193        void Validate_UTF32();
194        void InitializeBuffer(unsigned char * src, int lgth);
195        int UTF8_Length(int name_pos, int lgth);
196        void to_UTF8(int name_pos, int lgth, char * u8_ptr);
197protected:
198        BytePack * x32hh;
199        BytePack * x32hl;
200        BytePack * x32lh;
201        BytePack * x32ll;
202};
203
204class U32LE_Buffer : public U32_Buffer {
205public:
206        U32LE_Buffer();
207        void DoByteplex();
208};
209
210class U32BE_Buffer : public U32_Buffer {
211public:
212        U32BE_Buffer();
213        void DoByteplex();
214};
215
216class U32_2143_Buffer : public U32_Buffer {
217public:
218        U32_2143_Buffer();
219        void DoByteplex();
220};
221
222class U32_3412_Buffer : public U32_Buffer {
223public:
224        U32_3412_Buffer();
225        void DoByteplex();
226};
227
228
229inline char * copy_name (char * s, int lgth){           
230        char * d = new char[lgth+1];
231        memcpy(d, s,lgth); 
232        d[lgth] = '\0'; 
233        return d;
234}
235
236#endif
237
238
Note: See TracBrowser for help on using the repository browser.