source: trunk/symbol_table/main_template.cpp @ 2178

Last change on this file since 2178 was 2178, checked in by ksherdy, 7 years ago

Restructuring.

File size: 8.8 KB
Line 
1/*
2 * Created on: 18-December-2011
3 * Author: Ken Herdy
4 *
5 * A simple test driver.
6 *
7 * Lookahead versus COPYBACK
8 *
9 * The current implementation applies bit stream length grouping based on 'end' markers.
10 * In a sense, 'end' markers are precomputed 'lookahead'.
11 * True 'lookahead' would compute the current block and number of 'lookahead' position and
12 * support 'shift back' and to mark the 'start' rather than the 'end' positions of lexical items.
13 *
14 * In any case, the current implementation 'expects' that the previous block will be located in a contiguous
15 * memory location that may be indexed as some negative offset of the base address of the current
16 * block.
17 *
18 * Further, to reduce complexity in processing, although structs of BitBlock types are not stored
19 * contiguously in memory, BitBlock struct members are copied into contiguous memory positions.
20 *
21 */
22
23#include "../lib/allocator.hpp"
24#include "../lib/bitblock.hpp"
25#include "../lib/bitblock_scan.hpp"
26#include "../lib/buffer.hpp"
27#include "../lib/carryQ.hpp"
28#include "../lib/s2p.hpp"
29#include "../lib/perflib/perfsec.h"
30#include "../src/transpose.hpp"
31
32#include "marker_strms.hpp"     // GENERATED HEADER
33#include "hash_strms.hpp"       // GENERATED HEADER
34#include "group_strms.hpp"      // GENERATED HEADER
35#include <string>
36#include <iostream>
37#include <fstream>
38#include <sstream>
39using namespace std;
40
41#include "../src/symbol_table.hpp"
42
43#ifdef BUFFER_PROFILING
44    BOM_Table * parser_timer;
45#elif CODE_CLOCKER
46    #define NUM_EVENTS 1
47    int Events[NUM_EVENTS] = {PAPI_TOT_CYC};
48    //int Events[NUM_EVENTS] = {PAPI_L2_DCM};
49    //int Events[NUM_EVENTS] = {PAPI_TOT_CYC, PAPI_BR_MSP};
50    int cal_size = 20;
51    CC * parser_timer = new CC(Events,NUM_EVENTS,cal_size);
52#else
53    void * parser_timer;
54#endif
55
56int main(int argc, char * argv[]) {
57
58    if (argc < 2) {
59            cout << "Usage: " << argv[0] << " <filename>" << endl;
60            exit(-1);
61    }
62
63    stringstream filename;
64    filename << argv[1];
65
66    ifstream is;
67    is.open (filename.str().c_str(), ios::binary);
68
69    if (!is) {
70        cerr << "Error: " << filename << endl;
71        abort();
72    }
73
74    // PERF_SEC_BIND(1);
75    PERF_SEC_INIT(parser_timer);
76
77        ///////////////////////////////////////////////////////////////////////////
78        // Buffer Constants
79        ///////////////////////////////////////////////////////////////////////////
80        //
81        // NOTE:    Buffer segments are defined as multiples of BLOCK_SIZE.
82        //              BLOCK_SIZE defined as a number of bytes.
83        #define PADDING_BLOCKS 1
84        #define PADDING_SIZE (BLOCK_SIZE * PADDING_BLOCKS)
85        #define COPYBACK_BLOCKS 3
86        #define COPYBACK_SIZE (BLOCK_SIZE * COPYBACK_BLOCKS)
87        #define LOOKAHEAD_BLOCKS 1
88        #define LOOKAHEAD_SIZE (BLOCK_SIZE * LOOKAHEAD_BLOCKS)
89        #define SEGMENT_BLOCKS  12
90        #define SEGMENT_SIZE (BLOCK_SIZE * SEGMENT_BLOCKS)
91
92        #define BUFFER_SIZE (COPYBACK_SIZE + SEGMENT_SIZE + LOOKAHEAD_SIZE + PADDING_SIZE)
93
94    ///////////////////////////////////////////////////////////////////////////
95    // Parallel Data Streams with optional CopyBack
96    ///////////////////////////////////////////////////////////////////////////
97    uint8_t * COPYBACK;
98    uint8_t * src_buffer;
99    ALLOC_STATIC_ALIGNED_BYTE_BUFFER_WITH_COPYBACK(COPYBACK, src_buffer);
100
101    // hash 0
102    BitBlock * COPYBACK_h0;
103    BitBlock * h0;
104    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_h0, h0);
105
106    // hash 1
107    BitBlock * COPYBACK_h1;
108    BitBlock * h1;
109    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_h1, h1);
110
111    // starts
112    BitBlock * COPYBACK_starts;
113    BitBlock * starts;
114    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_starts, starts);
115
116    // follows_0 - Arbitrary length symbols
117    BitBlock * COPYBACK_follows_0;
118    BitBlock * follows_0;
119    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_follows_0, follows_0);
120
121    ///////////////////////////////////////////////////////////////////////////
122    // Parallel Data Streams - No CopyBack
123    ///////////////////////////////////////////////////////////////////////////
124
125    Basis_bits basis_bits[SEGMENT_BLOCKS];
126    Markers markers[SEGMENT_BLOCKS];
127    Hash hash[SEGMENT_BLOCKS];
128    Groups groups[SEGMENT_BLOCKS];
129
130        ///////////////////////////////////////////////////////////////////////////
131    // Symbol Table
132        ///////////////////////////////////////////////////////////////////////////
133        gid<SEGMENT_SIZE> gids;
134    symbol_table<gid<SEGMENT_SIZE>, fast_pool_allocator<1024> > st;
135
136    is.read ((char *)src_buffer, SEGMENT_SIZE);
137    uint32_t chars_avail = is.gcount();
138
139    #if IDENTITY_TEST
140        cout << ","; // prepend delimeter
141    #endif
142
143    ///////////////////////////////////////////////////////////////////////////
144    // Full Segments
145    ///////////////////////////////////////////////////////////////////////////
146    while (chars_avail >= SEGMENT_SIZE) {
147        uint32_t blk;
148        for(blk=0;blk<SEGMENT_BLOCKS;blk++) {
149        s2p_do_block((BytePack *) &src_buffer[blk*BLOCK_SIZE], basis_bits[blk]);    // transpose
150        markers_do_block(basis_bits[blk], markers[blk]);                            // gen symbol spans, mark starts & follows
151        hash_strms_do_block(basis_bits[blk], hash[blk]);                            // gen hash bit streams
152        identity_group_do_block(markers[blk], groups[blk]);                         // sort marker bit stream (identity)
153    }
154
155    for(int blk=0;blk<SEGMENT_BLOCKS;blk++) {   // write contiguous bit streams
156        h0[blk] = hash[blk].h0;
157        h1[blk] = hash[blk].h1;
158        starts[blk] = groups[blk].starts;
159        follows_0[blk] = groups[blk].follows_0;
160    }
161
162    PERF_SEC_START(parser_timer);
163    st.resolve(src_buffer, groups, starts, follows_0, h0, h1, SEGMENT_BLOCKS, gids);
164    PERF_SEC_END(parser_timer, SEGMENT_SIZE);
165
166    COPY_BACK_BYTE_BUFFER(COPYBACK, src_buffer);
167    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_h0,h0);
168    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_h1,h1);
169    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_starts,starts);
170    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_follows_0,follows_0);
171
172    is.read ((char *)(src_buffer), SEGMENT_SIZE);
173    chars_avail = is.gcount();
174
175    #ifdef IDENTITY_TEST
176    uint32_t blk_offset;
177    for(int blk=0;blk<SEGMENT_BLOCKS;blk++) {
178        blk_offset = blk * BLOCK_SIZE;
179        gid_type gid;
180
181        #ifdef INDEX_AT_STARTS
182            ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].starts));
183        #else
184            ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].follows));
185        #endif
186
187        scanner.scan_to_next();
188        while(!scanner.is_done()) {
189                gid = gids.at[scanner.get_pos() + blk_offset];
190                cout << string((char *)st.get_raw_data(gid), st.get_lgth(gid)) << ",";
191                scanner.scan_to_next();
192        }
193    }
194    #endif
195    }
196
197    /* Resolve Partial Segments */
198    uint32_t remaining = chars_avail;
199
200    ///////////////////////////////////////////////////////////////////////////
201    // Full blocks
202    ///////////////////////////////////////////////////////////////////////////
203    uint32_t blk = 0;
204    while (remaining >= BLOCK_SIZE) {
205        s2p_do_block((BytePack *) &src_buffer[blk*BLOCK_SIZE], basis_bits[blk]);
206        markers_do_block(basis_bits[blk], markers[blk]);
207        hash_strms_do_block(basis_bits[blk], hash[blk]);
208        identity_group_do_block(markers[blk], groups[blk]);
209        blk++;
210        remaining -= BLOCK_SIZE;
211    }
212
213    ///////////////////////////////////////////////////////////////////////////
214    // Final partial block or any carry
215    ///////////////////////////////////////////////////////////////////////////
216    if (remaining > 0 || @marker_strms_any_carry || @group_strms_any_carry /*|| hash_strms_any_carry*/) {
217        BitBlock EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining));
218        s2p_do_final_block((BytePack *) &src_buffer[blk*BLOCK_SIZE], basis_bits[blk], EOF_mask);
219        markers_do_final_block(basis_bits[blk], markers[blk], EOF_mask);
220        hash_strms_do_final_block(basis_bits[blk], hash[blk], EOF_mask);
221        identity_group_do_final_block(markers[blk], groups[blk], EOF_mask);
222        blk++;
223    }
224
225    uint32_t segment_blocks = blk;
226    for(int blk=0;blk<segment_blocks;blk++) { // write contiguous hash bit streams
227        h0[blk] = hash[blk].h0;
228        h1[blk] = hash[blk].h1;
229        starts[blk] = groups[blk].starts;
230        follows_0[blk] = groups[blk].follows_0;
231    }
232
233    // PERF_SEC_BIND(0);
234    PERF_SEC_START(parser_timer);
235    st.resolve(src_buffer, groups, starts, follows_0, h0, h1, segment_blocks, gids);
236    PERF_SEC_END(parser_timer, chars_avail);
237
238    #ifdef IDENTITY_TEST
239    uint32_t blk_offset;
240    for(int blk=0;blk<segment_blocks;blk++) {
241        blk_offset = blk * BLOCK_SIZE;
242        gid_type gid;
243
244        #ifdef INDEX_AT_STARTS
245            ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].starts));
246        #else
247            ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].follows));
248        #endif
249
250        scanner.scan_to_next();
251        while(!scanner.is_done()) {
252                gid = gids.at[scanner.get_pos() + blk_offset];
253                cout << string((char *)st.get_raw_data(gid), st.get_lgth(gid)) << ",";
254                scanner.scan_to_next();
255        }
256    }
257    #endif
258
259    #ifdef GID_TEST
260        cout << st.get_max_gid() << endl;
261    #endif
262
263    PERF_SEC_DUMP(parser_timer);
264    PERF_SEC_DESTROY(parser_timer);
265
266    is.close();
267
268    return 1;
269}
270
271
Note: See TracBrowser for help on using the repository browser.