source: trunk/src/byteplex.h @ 125

Last change on this file since 125 was 103, checked in by cameron, 11 years ago

X8_Buffer<C> to unify 8-bit byteplex classes for ASCII and EBCDIC families.

File size: 6.3 KB
Line 
1/*  byteplex.h - Parallel byte stream module.
2    Copyright (c) 2008,  Robert D. Cameron.
3    Licensed to the public under the Open Software License 3.0.
4    Licensed to International Characters, Inc., under the Academic
5    Free License 3.0.
6
7    This module has as its goal the buffering of XML byte data and
8    transformation of 16-bit and 32-bit code unit data so that the
9    parsing engine is provided a uniform representation based on
10    the concept of an 8-bit pseudo-ASCII representation (x8data).
11
12    A Byteplex object provides buffers for one to six parallel data
13    streams based for an XML input entity, depending on the size of
14    character code units. 
15       1.  In the case of 8-bit code units, a single byte stream
16           consisting of unmodified input data is maintained.
17           x8data = src_buffer
18       2.  In the case of 16-bit code units (UTF-16 and UCS-2 families),
19             (a) the original code unit stream is maintained unmodified,
20             (b) the x16hi byte stream is established for the high byte
21                 of each code unit,
22             (c) the x16lo byte stream is established for the low byte
23                 of each code unit, and
24             (d) x8data is established as the pseudo-ASCII byte stream,
25                 with ASCII code units having their proper 8-bit values,
26                 and all others having bit 0 set to 1.
27       3.  In the case of 32-bit code units (UTF-32 family),
28             (a) the original code unit stream is maintained unmodified,
29             (b) the x32hh byte stream has high bytes of each code unit
30             (c) the x32hl byte stream has second bytes of each code unit
31             (d) the x32lh byte stream has third bytes of each code unit
32             (e) the x32hh byte stream has low bytes of each code unit, and
33             (f) x8data is established as the pseudo-ASCII byte stream,
34                 with ASCII code units having their proper 8-bit values,
35                 and all others having bit 0 set to 1.
36
37    The pseudo-ASCII representation is defined for both ASCII-based
38    and EBCDIC-based character sets such that all characters in
39    the ASCII repertoire (i.e., having Unicode code points from 0x00
40    to 0x7F), are represented as themselves and no non-ASCII character
41    is represented as a character in the ASCII repertoire.
42
43*/
44
45#ifndef BYTEPLEX_H
46#define BYTEPLEX_H
47
48#include "xmlmodel.h"
49#include "../lib/lib_simd.h"
50
51/* The BytePack and the BitBlock are the two fundamental
52   types used by the parabix program for data held in
53   SIMD registers, representing, respectively, the byte-oriented
54   and bit-oriented views of character data.*/
55
56typedef SIMD_type BytePack;
57typedef SIMD_type BitBlock;
58const int PACKSIZE = sizeof(SIMD_type);
59const int BLOCKSIZE = sizeof(SIMD_type) * 8;
60
61/* Define the size of buffer used for lexical analysis/parsing. */
62const int BUFFER_BLOCKS = 1024;
63const int BUFFER_SIZE = BUFFER_BLOCKS * BLOCKSIZE;
64
65/* When working near the end of a buffer, a bytespace test may involve
66   a multibyte literal.  The bytespace buffer must always make available
67   a number of lookahead bytes at least equal to the maximum length of any
68   such literal. */
69
70const int LOOKAHEAD_POSITIONS = 16;
71const int BYTEPLEX_SIZE = BUFFER_SIZE + LOOKAHEAD_POSITIONS;
72
73class Byteplex {
74public:
75        static Byteplex * ByteplexFactory(Entity_Info * e);
76        static Byteplex * ByteplexFactory(Entity_Info * e, FILE * inputfile);
77        static Byteplex * ByteplexFactory(Entity_Info * e, unsigned char * buffer_bytes, int buffer_size);
78        virtual void DoByteplex() = 0;
79        virtual void PreparePseudoASCII_Stream() = 0;
80        virtual void InitializeBuffer(unsigned char * src, int lgth) = 0;
81        virtual void AdvanceInputBuffer(int advance_amt) = 0;
82
83        /* Source code unit buffer. */
84        BytePack * src_buffer;
85        int units_in_buffer;
86
87        /* Pseudo-ASCII stream. */
88        BytePack * x8data;
89
90protected:
91        FILE * infile;
92        int packs_in_buffer;
93        int CopyAndFill(unsigned char * bytes_to_copy, int lgth, int bytes_to_read);
94        void Set_limits(int units_in_buffer);
95
96};
97
98
99/*  The X8_Buffer template class is used for either ASCII- or EBCDIC-
100    based 8-bit code units.
101    The X8_Buffer<ASCII> class includes 7-bit ASCII
102    (with high-order bit 0), the ISO-8859 character sets and UTF-8.
103
104    The family of 8-bit EBCDIC based character sets are processed using
105    the X8_Buffer<EBCDIC> class.
106*/
107
108template <CodeUnit_Base C>
109class X8_Buffer : public Byteplex {
110public:
111        X8_Buffer();
112        ~X8_Buffer();
113       
114        void DoByteplex();
115        void PreparePseudoASCII_Stream();
116        void AdvanceInputBuffer(int advance_amt);
117        void InitializeBuffer(unsigned char * src, int lgth);
118};
119
120
121/*  UTF-16 and UCS-2 character set families in BE and LE byte orders.
122    The U16LE and U16BE subclasses each provide a distinct byteplexer to
123    produce 2 parallel byte streams for the high and low bytes of each
124    16-bit code unit.  Once byteplexing is complete, a generic pseudoASCII
125    conversion routine can be applied at the U16_Buffer level. */
126
127class U16_Buffer : public Byteplex {
128public:
129        U16_Buffer();
130        ~U16_Buffer();
131        virtual void DoByteplex() = 0;
132        void PreparePseudoASCII_Stream();
133        void AdvanceInputBuffer(int advance_amt);
134        void InitializeBuffer(unsigned char * src, int lgth);
135protected:
136        BytePack * x16hi;
137        BytePack * x16lo;
138};
139
140class U16LE_Buffer : public U16_Buffer {
141public:
142        U16LE_Buffer();
143        void DoByteplex();
144};
145
146class U16BE_Buffer : public U16_Buffer {
147public:
148        U16BE_Buffer();
149        void DoByteplex();
150};
151
152
153/*  UTF-32/UCS-4 character sets in BE, LE, 2143 and 3412 byte orders.
154    Each subclass of U32_Buffer provide a distinct byteplexer to
155    produce the 4 parallel byte streams of Unicode data.  Once
156    byteplexing is complete, a generic pseudoASCII routine can
157    be applied. */
158class U32_Buffer : public Byteplex {
159public:
160        U32_Buffer();
161        ~U32_Buffer();
162        virtual void DoByteplex() = 0;
163        void PreparePseudoASCII_Stream();
164        void AdvanceInputBuffer(int advance_amt);
165        void InitializeBuffer(unsigned char * src, int lgth);
166protected:
167        BytePack * x32hh;
168        BytePack * x32hl;
169        BytePack * x32lh;
170        BytePack * x32ll;
171};
172
173class U32LE_Buffer : public U32_Buffer {
174public:
175        U32LE_Buffer();
176        void DoByteplex();
177};
178
179class U32BE_Buffer : public U32_Buffer {
180public:
181        U32BE_Buffer();
182        void DoByteplex();
183};
184
185class U32_2143_Buffer : public U32_Buffer {
186public:
187        U32_2143_Buffer();
188        void DoByteplex();
189};
190
191class U32_3412_Buffer : public U32_Buffer {
192public:
193        U32_3412_Buffer();
194        void DoByteplex();
195};
196
197#endif
Note: See TracBrowser for help on using the repository browser.