source: trunk/src/bitlex.c @ 4

Last change on this file since 4 was 4, checked in by cameron, 11 years ago

Initial import of parabix-0.36

File size: 3.9 KB
Line 
1/*  bitlex - Parabix lexical analysis common routines.
2    Copyright (c) 2007, Robert D. Cameron.
3    Licensed to the public under the Open Software License 3.0.
4    Licensed to International Characters, Inc., under the Academic
5    Free License 3.0.
6
7    These are common routines for all ASCII-family character sets.
8    They are used by the character-set specific Lexer objects
9    found in the charsets directory.
10*/
11#include "bitlex.h"
12
13#include "transpose.h"
14
15Lexer::Lexer(XML_Buffer *b, ParallelStreamSet *p) {
16  xml_buf = b;
17  parsing_engine_data = p;
18};
19
20/* Given the bit[] array of one BitBlock each for the 8 bits of
21   an ASCII-family character representation, compute the parallel
22   lexical item streams needed for XML parsing.
23
24   WARNING: the following is generated code by charset_compiler.py.
25   Do not edit.
26
27*/
28static inline void ComputeLexicalItemBlocks(BitBlock bit[], BitBlock LexItem[]) {
29  BitBlock temp1 = simd_or(bit[0], bit[1]);
30  BitBlock temp2 = simd_and(bit[2], bit[3]);
31  BitBlock temp3 = simd_andc(temp2, temp1);
32  BitBlock temp4 = simd_and(bit[4], bit[5]);
33  BitBlock temp5 = simd_or(bit[6], bit[7]);
34  BitBlock temp6 = simd_andc(temp4, temp5);
35  LexItem[LAngle] = simd_and(temp3, temp6);
36  BitBlock temp7 = simd_andc(bit[6], bit[7]);
37  BitBlock temp8 = simd_and(temp4, temp7);
38  LexItem[RAngle] = simd_and(temp3, temp8);
39  BitBlock temp9 = simd_andc(bit[2], bit[3]);
40  BitBlock temp10 = simd_andc(temp9, temp1);
41  BitBlock temp11 = simd_and(bit[6], bit[7]);
42  BitBlock temp12 = simd_and(temp4, temp11);
43  BitBlock Slash = simd_and(temp10, temp12);
44  BitBlock temp13 = simd_andc(bit[1], bit[0]);
45  BitBlock temp14 = simd_andc(bit[3], bit[2]);
46  BitBlock temp15 = simd_and(temp13, temp14);
47  BitBlock temp16 = simd_andc(bit[7], bit[6]);
48  BitBlock temp17 = simd_and(temp4, temp16);
49  LexItem[RBracket] = simd_and(temp15, temp17);
50  LexItem[Hyphen] = simd_and(temp10, temp17);
51  LexItem[QMark] = simd_and(temp3, temp12);
52  BitBlock Equals = simd_and(temp3, temp17);
53  BitBlock temp18 = simd_andc(bit[5], bit[4]);
54  BitBlock temp19 = simd_and(temp18, temp11);
55  LexItem[SQuote] = simd_and(temp10, temp19);
56  BitBlock temp20 = simd_or(bit[4], bit[5]);
57  BitBlock temp21 = simd_andc(temp7, temp20);
58  LexItem[DQuote] = simd_and(temp10, temp21);
59  BitBlock temp22 = simd_or(temp20, temp5);
60  BitBlock temp23 = simd_andc(temp10, temp22);
61  BitBlock temp24 = simd_or(bit[2], bit[3]);
62  BitBlock temp25 = simd_or(temp1, temp24);
63  BitBlock temp26 = simd_andc(temp17, temp25);
64  BitBlock temp27 = simd_or(temp23, temp26);
65  BitBlock temp28 = simd_andc(bit[4], bit[5]);
66  BitBlock temp29 = simd_and(temp28, temp16);
67  BitBlock temp30 = simd_andc(temp29, temp25);
68  BitBlock temp31 = simd_or(temp27, temp30);
69  BitBlock temp32 = simd_and(temp28, temp7);
70  BitBlock temp33 = simd_andc(temp32, temp25);
71  BitBlock WhiteSpace = simd_or(temp31, temp33);
72  LexItem[NonWS] = simd_not(WhiteSpace);
73  LexItem[NameFollow] = simd_or(simd_or(WhiteSpace, 
74                                        simd_or(Slash, LexItem[RAngle])),
75                                simd_or(Equals, LexItem[QMark]));
76}
77
78
79/* A temporary structure for internal use in ComputeLexicalItemStreams. */
80typedef struct {
81  BitBlock bit[8];
82  BitBlock LexicalItems[LexicalItemCount];
83} LexicalItemBlock;
84
85
86void Lexer::ComputeLexicalItemStreams(int new_blocks) {
87  LexicalItemBlock lx_blk[BUFFER_BLOCKS];
88  for (int i = 0; i < new_blocks; i++) {
89    s2p_bytepack(&(parsing_engine_data->x8data[i * 8]), lx_blk[i].bit);
90    ComputeLexicalItemBlocks(lx_blk[i].bit, lx_blk[i].LexicalItems);
91  }
92#ifdef BUFFER_PROFILING
93  end_BOM_interval(bitstream_timer);
94  start_BOM_interval(lextranspose_timer);
95#endif
96  for (int j = LAngle; j < LexicalItemCount; j++) {
97    for (int i = 0; i < BUFFER_BLOCKS; i++) {
98      parsing_engine_data->item_stream[j][i] = lx_blk[i].LexicalItems[j];
99    }
100  }
101#ifdef BUFFER_PROFILING
102  end_BOM_interval(lextranspose_timer);
103  start_BOM_interval(scanner_timer);
104#endif
105}
Note: See TracBrowser for help on using the repository browser.