source: trunk/symbol_table/main_template.cpp @ 1969

Last change on this file since 1969 was 1969, checked in by ksherdy, 7 years ago

Restructuring.

File size: 7.6 KB
Line 
1/*
2 * Created on: 18-December-2011
3 * Author: Ken Herdy
4 *
5 * Length sorted symbol table main.
6 *
7 * Lookahead versus Lookback
8 *
9 * The current implementation applies bit stream length grouping based on 'end' markers.
10 * In a sense, 'end' markers are precomputed 'lookahead'.
11 * True 'lookahead' would compute the current block and number of 'lookahead' position and
12 * support 'shift back' and to mark the 'start' rather than the 'end' positions of lexical items.
13 *
14 * In any case, the current implementation 'expects' that the previous block will be located in a contiguous
15 * memory location that may be indexed as some negative offset of the base address of the current
16 * block.
17 *
18 * Further, to reduce complexity in processing, although structs of BitBlock types are not stored
19 * contiguously in memory, BitBlock struct members are copied into contiguous memory positions.
20 *
21 * Design Issues
22 *
23 * (1) Max hash table size.
24 * (2) Negative shift values.
25 * (3) Template classes on Length L.
26 *
27 */
28
29#include "transpose.hpp"
30#include "../lib/bitblock.hpp"
31#include "../lib/allocator.hpp"
32#include "../lib/s2p.hpp"
33#include "../lib/perflib/perfsec.h"
34#include "marker_strms.hpp"     // GENERATED HEADER
35#include "hash_strms.hpp"       // GENERATED HEADER
36#include "id_group_strms.hpp"   // GENERATED HEADER
37#include "id_symbol_table.hpp"
38#include <string>
39#include <iostream>
40#include <fstream>
41#include <sstream>
42using namespace std;
43
44#ifdef BUFFER_PROFILING
45    BOM_Table * parser_timer;
46#elif CODE_CLOCKER
47    #define NUM_EVENTS 1
48    int Events[NUM_EVENTS] = {PAPI_TOT_CYC};
49    //int Events[NUM_EVENTS] = {PAPI_L2_DCM};
50    //int Events[NUM_EVENTS] = {PAPI_TOT_CYC, PAPI_BR_MSP};
51    int cal_size = 20;
52    CC * parser_timer = new CC(Events,NUM_EVENTS,cal_size);
53#else
54    void * parser_timer;
55#endif
56
57///////////////////////////////////////////////////////////////////////////
58// Buffer Management
59///////////////////////////////////////////////////////////////////////////
60#define PADDING_BLOCKS 1
61#define PADDING_SIZE BLOCK_SIZE * PADDING_BLOCKS
62#define LOOKBACK_BLOCKS 1
63#define LOOKBACK_SIZE BLOCK_SIZE * LOOKBACK_BLOCKS
64#define SEGMENT_BLOCKS 10
65#define SEGMENT_SIZE BLOCK_SIZE * (SEGMENT_BLOCKS)                                          // (bytes) a multiple of BLOCK_SIZE
66#define SEGMENT_ALLOC_SIZE (LOOKBACK_SIZE + SEGMENT_SIZE + PADDING_SIZE) / sizeof(BitBlock) // (bytes)
67
68// Target symbol type must inherit from AoS_symbol
69typedef struct symbol: public AoS_symbol
70{
71    string parameter_1;
72    string parameter_2;
73    bool parameter_3;
74} symbol;
75
76int main(int argc, char * argv[]) {
77
78    if (argc < 2) {
79            cout << "Usage: " << argv[0] << " <filename>" << endl;
80            exit(-1);
81    }
82
83    stringstream filename;
84    filename << argv[1];
85
86    ifstream is;
87    is.open (filename.str().c_str(), ios::binary);
88
89    if (!is) {
90        cerr << "Error: " << filename << endl;
91        abort();
92    }
93
94    // PERF_SEC_BIND(1);
95    PERF_SEC_INIT(parser_timer);
96
97    ///////////////////////////////////////////////////////////////////////////
98    // Stream Definitions
99    ///////////////////////////////////////////////////////////////////////////
100
101    // Byte Segments - Raw byte streams - With lookback.
102    BitBlock aligned_buffer[SEGMENT_ALLOC_SIZE];
103    uint8_t * lookback = (uint8_t *)aligned_buffer;
104    memset(lookback,0,LOOKBACK_SIZE);
105    uint8_t * raw_buffer = &lookback[LOOKBACK_SIZE];
106
107    // Bit Segments - Hash bit streams - With lookback.
108
109    // hash 0
110    BitBlock aligned_h0[SEGMENT_ALLOC_SIZE/8];
111    BitBlock * lookback_h0 = (BitBlock *) aligned_h0;
112    memset(lookback_h0,0,LOOKBACK_SIZE/BLOCK_SIZE);
113    BitBlock * h0 = &lookback_h0[LOOKBACK_SIZE/BLOCK_SIZE];
114
115    // hash 1
116    BitBlock aligned_h1[SEGMENT_ALLOC_SIZE/8];
117    BitBlock * lookback_h1 = (BitBlock *) aligned_h1;
118    memset(lookback_h1,0,LOOKBACK_SIZE/BLOCK_SIZE);
119    BitBlock * h1 = &lookback_h1[LOOKBACK_SIZE/BLOCK_SIZE];
120
121    // BitSteams - Without lookback
122    Basis_bits basis_bits[SEGMENT_BLOCKS];
123    Markers markers[SEGMENT_BLOCKS];
124    Hash hash[SEGMENT_BLOCKS];
125    Groups groups[SEGMENT_BLOCKS];
126
127    // Symbol Table
128    const uint32_t SYMBOL_COUNT = SEGMENT_SIZE;
129    symbol symbol_ary[SYMBOL_COUNT];
130    id_symbol_table<fast_pool_allocator<1024> > symbol_table;
131
132    is.read ((char *)raw_buffer, SEGMENT_SIZE);
133    uint32_t chars_avail = is.gcount();
134
135    ///////////////////////////////////////////////////////////////////////////
136    // Full Segments
137    ///////////////////////////////////////////////////////////////////////////
138    while (chars_avail >= SEGMENT_SIZE) {
139
140      uint32_t blk;
141      for(blk=0;blk<SEGMENT_BLOCKS;blk++) {
142        s2p_do_block((BytePack *) &raw_buffer[blk*BLOCK_SIZE], basis_bits[blk]);    // transpose
143        markers_do_block(basis_bits[blk], markers[blk]);                            // gen symbol spans, mark starts & follows
144        hash_strms_do_block(basis_bits[blk], hash[blk]);                            // gen hash bit streams
145        identity_group_do_block(markers[blk], groups[blk]);                         // sort marker bit stream (identity)
146      }
147
148//    for(int k=0;k<SEGMENT_BLOCKS;k++) {
149//        cout << "RAW " << string((((char*)&raw_buffer[0])+k*BLOCK_SIZE),BLOCK_SIZE) << endl;
150//    }
151
152      for(int blk=0;blk<SEGMENT_BLOCKS;blk++) { // write contiguous hash bit streams
153        h0[blk] = hash[blk].h0;
154        h1[blk] = hash[blk].h1;
155      }
156
157      PERF_SEC_START(parser_timer);
158      symbol_table.resolve(raw_buffer, groups, h0, h1, SEGMENT_BLOCKS, symbol_ary, SYMBOL_COUNT);
159      PERF_SEC_END(parser_timer, SEGMENT_SIZE);
160
161      // copy loopback bytes
162      memmove(lookback,&raw_buffer[SEGMENT_SIZE-LOOKBACK_SIZE],LOOKBACK_SIZE);
163      // copy loopback bits
164      memmove(lookback_h0,&((uint8_t *)h0)[(SEGMENT_SIZE-LOOKBACK_SIZE)/8],LOOKBACK_SIZE/8);
165      memmove(lookback_h1,&((uint8_t *)h1)[(SEGMENT_SIZE-LOOKBACK_SIZE)/8],LOOKBACK_SIZE/8);
166
167      //lookback_h0[0] = h0[SEGMENT_BLOCKS-1];
168      //lookback_h1[0] = h1[SEGMENT_BLOCKS-1];
169      is.read ((char *)(raw_buffer), SEGMENT_SIZE);
170      chars_avail = is.gcount();
171    }
172
173    /* Resolve Partial Segments */
174    uint32_t remaining = chars_avail;
175
176    ///////////////////////////////////////////////////////////////////////////
177    // Full blocks
178    ///////////////////////////////////////////////////////////////////////////
179    uint32_t blk = 0;
180    while (remaining >= BLOCK_SIZE) {
181        s2p_do_block((BytePack *) &raw_buffer[blk*BLOCK_SIZE], basis_bits[blk]);
182        markers_do_block(basis_bits[blk], markers[blk]);
183        hash_strms_do_block(basis_bits[blk], hash[blk]);
184        identity_group_do_block(markers[blk], groups[blk]);
185        blk++;
186        remaining -= BLOCK_SIZE;
187    }
188
189    ///////////////////////////////////////////////////////////////////////////
190    // Final partial block or any carry
191    ///////////////////////////////////////////////////////////////////////////
192    if (remaining > 0 || @marker_strms_any_carry /*|| hash_strms_any_carry*/) {
193          BitBlock EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining));
194          s2p_do_final_block((BytePack *) &raw_buffer[blk*BLOCK_SIZE], basis_bits[blk], EOF_mask);
195          markers_do_final_block(basis_bits[blk], markers[blk], EOF_mask);
196          hash_strms_do_final_block(basis_bits[blk], hash[blk], EOF_mask);
197          identity_group_do_final_block(markers[blk], groups[blk], EOF_mask);
198          blk++;
199    }
200
201//    for(int k=0;k<blk;k++) {
202//      cout << "RAW " << string((((char*)&raw_buffer[0])+k*BLOCK_SIZE),BLOCK_SIZE) << endl;
203//    }
204
205
206    uint32_t segment_size = blk;
207    for(int blk=0;blk<segment_size;blk++) { // write contiguous hash bit streams
208        h0[blk] = hash[blk].h0;
209        h1[blk] = hash[blk].h1;
210    }
211
212    //PERF_SEC_START(parser_timer);
213    symbol_table.resolve(raw_buffer, groups, h0, h1, blk, symbol_ary, SYMBOL_COUNT);
214    //PERF_SEC_END(parser_timer, chars_avail+1);
215
216    PERF_SEC_DUMP(parser_timer);
217    PERF_SEC_DESTROY(parser_timer);
218
219    is.close();
220
221    return 1;
222}
223
224
Note: See TracBrowser for help on using the repository browser.