source: trunk/symbol_table/main_template.cpp @ 2141

Last change on this file since 2141 was 2141, checked in by ksherdy, 7 years ago

Fixed 64 bit machine copyback. Updated default to index at follows pos w/o Makefile preprocessor define.

File size: 8.0 KB
Line 
1/*
2 * Created on: 18-December-2011
3 * Author: Ken Herdy
4 *
5 * A simple test driver.
6 *
7 * Lookahead versus COPYBACK
8 *
9 * The current implementation applies bit stream length grouping based on 'end' markers.
10 * In a sense, 'end' markers are precomputed 'lookahead'.
11 * True 'lookahead' would compute the current block and number of 'lookahead' position and
12 * support 'shift back' and to mark the 'start' rather than the 'end' positions of lexical items.
13 *
14 * In any case, the current implementation 'expects' that the previous block will be located in a contiguous
15 * memory location that may be indexed as some negative offset of the base address of the current
16 * block.
17 *
18 * Further, to reduce complexity in processing, although structs of BitBlock types are not stored
19 * contiguously in memory, BitBlock struct members are copied into contiguous memory positions.
20 *
21 */
22
23#include "../lib/allocator.hpp"
24#include "../lib/bitblock.hpp"
25#include "../lib/bitblock_scan.hpp"
26#include "../lib/carryQ.hpp"
27#include "../lib/s2p.hpp"
28#include "../lib/perflib/perfsec.h"
29#include "transpose.hpp"
30#include "buffer.hpp"
31#include "marker_strms.hpp"     // GENERATED HEADER
32#include "hash_strms.hpp"       // GENERATED HEADER
33#include "group_strms.hpp"      // GENERATED HEADER
34#include <string>
35#include <iostream>
36#include <fstream>
37#include <sstream>
38using namespace std;
39
40#include "symbol_table.hpp"
41
42#ifdef BUFFER_PROFILING
43    BOM_Table * parser_timer;
44#elif CODE_CLOCKER
45    #define NUM_EVENTS 1
46    int Events[NUM_EVENTS] = {PAPI_TOT_CYC};
47    //int Events[NUM_EVENTS] = {PAPI_L2_DCM};
48    //int Events[NUM_EVENTS] = {PAPI_TOT_CYC, PAPI_BR_MSP};
49    int cal_size = 20;
50    CC * parser_timer = new CC(Events,NUM_EVENTS,cal_size);
51#else
52    void * parser_timer;
53#endif
54
55int main(int argc, char * argv[]) {
56
57    if (argc < 2) {
58            cout << "Usage: " << argv[0] << " <filename>" << endl;
59            exit(-1);
60    }
61
62    stringstream filename;
63    filename << argv[1];
64
65    ifstream is;
66    is.open (filename.str().c_str(), ios::binary);
67
68    if (!is) {
69        cerr << "Error: " << filename << endl;
70        abort();
71    }
72
73    // PERF_SEC_BIND(1);
74    PERF_SEC_INIT(parser_timer);
75
76    ///////////////////////////////////////////////////////////////////////////
77    // Parallel Data Streams with optional CopyBack
78    ///////////////////////////////////////////////////////////////////////////
79    uint8_t * COPYBACK;
80    uint8_t * src_buffer;
81//    ALLOC_STATIC_ALIGNED_BYTE_BUFFER(COPYBACK, src_buffer);
82        ATTRIBUTE_SIMD_ALIGN BitBlock aligned_base[BUFFER_SIZE]; 
83        COPYBACK = (uint8_t *)aligned_base; 
84        memset(COPYBACK,0,COPYBACK_SIZE); 
85        src_buffer = &COPYBACK[COPYBACK_SIZE]; 
86
87    // hash 0
88    BitBlock * COPYBACK_h0;
89    BitBlock * h0;
90    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER(COPYBACK_h0, h0);
91
92    // hash 1
93    BitBlock * COPYBACK_h1;
94    BitBlock * h1;
95    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER(COPYBACK_h1, h1);
96
97    // starts
98    BitBlock * COPYBACK_starts;
99    BitBlock * starts;
100    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER(COPYBACK_starts, starts);
101
102    // follows_0 - Arbitrary length symbols
103    BitBlock * COPYBACK_follows_0;
104    BitBlock * follows_0;
105    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER(COPYBACK_follows_0, follows_0);
106
107    ///////////////////////////////////////////////////////////////////////////
108    // Parallel Data Streams - No CopyBack
109    ///////////////////////////////////////////////////////////////////////////
110
111    Basis_bits basis_bits[SEGMENT_BLOCKS];
112    Markers markers[SEGMENT_BLOCKS];
113    Hash hash[SEGMENT_BLOCKS];
114    Groups groups[SEGMENT_BLOCKS];
115
116    // Symbol Table
117    gid<SEGMENT_SIZE> gids;
118    symbol_table<gid<SEGMENT_SIZE>, fast_pool_allocator<1024> > st;
119
120    is.read ((char *)src_buffer, SEGMENT_SIZE);
121    uint32_t chars_avail = is.gcount();
122
123    #if IDENTITY_TEST
124        cout << ","; // prepend delimeter
125    #endif
126
127    ///////////////////////////////////////////////////////////////////////////
128    // Full Segments
129    ///////////////////////////////////////////////////////////////////////////
130    while (chars_avail >= SEGMENT_SIZE) {
131        uint32_t blk;
132        for(blk=0;blk<SEGMENT_BLOCKS;blk++) {
133        s2p_do_block((BytePack *) &src_buffer[blk*BLOCK_SIZE], basis_bits[blk]);    // transpose
134        markers_do_block(basis_bits[blk], markers[blk]);                            // gen symbol spans, mark starts & follows
135        hash_strms_do_block(basis_bits[blk], hash[blk]);                            // gen hash bit streams
136        identity_group_do_block(markers[blk], groups[blk]);                         // sort marker bit stream (identity)
137    }
138
139    for(int blk=0;blk<SEGMENT_BLOCKS;blk++) {   // write contiguous bit streams
140        h0[blk] = hash[blk].h0;
141        h1[blk] = hash[blk].h1;
142        starts[blk] = groups[blk].starts;
143        follows_0[blk] = groups[blk].follows_0;
144    }
145
146    PERF_SEC_START(parser_timer);
147    st.resolve(src_buffer, groups, starts, follows_0, h0, h1, SEGMENT_BLOCKS, gids);
148    PERF_SEC_END(parser_timer, SEGMENT_SIZE);
149
150    COPY_BACK_BYTE_BUFFER(COPYBACK, src_buffer);
151    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_h0,h0);
152    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_h1,h1);
153    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_starts,starts);
154    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_follows_0,follows_0);
155
156    is.read ((char *)(src_buffer), SEGMENT_SIZE);
157    chars_avail = is.gcount();
158
159    #ifdef IDENTITY_TEST
160    uint32_t blk_offset;
161    for(int blk=0;blk<SEGMENT_BLOCKS;blk++) {
162        blk_offset = blk * BLOCK_SIZE;
163        gid_type gid;
164
165        #ifdef INDEX_AT_STARTS
166            ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].starts));
167        #else
168            ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].follows));
169        #endif
170
171        scanner.scan_to_next();
172        while(!scanner.is_done()) {
173                gid = gids.at[scanner.get_pos() + blk_offset];
174                cout << string((char *)st.get_raw_data(gid), st.get_lgth(gid)) << ",";
175                scanner.scan_to_next();
176        }
177    }
178    #endif
179    }
180
181    /* Resolve Partial Segments */
182    uint32_t remaining = chars_avail;
183
184    ///////////////////////////////////////////////////////////////////////////
185    // Full blocks
186    ///////////////////////////////////////////////////////////////////////////
187    uint32_t blk = 0;
188    while (remaining >= BLOCK_SIZE) {
189        s2p_do_block((BytePack *) &src_buffer[blk*BLOCK_SIZE], basis_bits[blk]);
190        markers_do_block(basis_bits[blk], markers[blk]);
191        hash_strms_do_block(basis_bits[blk], hash[blk]);
192        identity_group_do_block(markers[blk], groups[blk]);
193        blk++;
194        remaining -= BLOCK_SIZE;
195    }
196
197    ///////////////////////////////////////////////////////////////////////////
198    // Final partial block or any carry
199    ///////////////////////////////////////////////////////////////////////////
200    if (remaining > 0 || @marker_strms_any_carry || @group_strms_any_carry /*|| hash_strms_any_carry*/) {
201        BitBlock EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining));
202        s2p_do_final_block((BytePack *) &src_buffer[blk*BLOCK_SIZE], basis_bits[blk], EOF_mask);
203        markers_do_final_block(basis_bits[blk], markers[blk], EOF_mask);
204        hash_strms_do_final_block(basis_bits[blk], hash[blk], EOF_mask);
205        identity_group_do_final_block(markers[blk], groups[blk], EOF_mask);
206        blk++;
207    }
208
209    uint32_t segment_blocks = blk;
210    for(int blk=0;blk<segment_blocks;blk++) { // write contiguous hash bit streams
211        h0[blk] = hash[blk].h0;
212        h1[blk] = hash[blk].h1;
213        starts[blk] = groups[blk].starts;
214        follows_0[blk] = groups[blk].follows_0;
215    }
216
217    // PERF_SEC_BIND(0);
218    PERF_SEC_START(parser_timer);
219    st.resolve(src_buffer, groups, starts, follows_0, h0, h1, segment_blocks, gids);
220    PERF_SEC_END(parser_timer, chars_avail);
221
222    #ifdef IDENTITY_TEST
223    uint32_t blk_offset;
224    for(int blk=0;blk<segment_blocks;blk++) {
225        blk_offset = blk * BLOCK_SIZE;
226        gid_type gid;
227
228        #ifdef INDEX_AT_STARTS
229            ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].starts));
230        #else
231            ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].follows));
232        #endif
233
234        scanner.scan_to_next();
235        while(!scanner.is_done()) {
236                gid = gids.at[scanner.get_pos() + blk_offset];
237                cout << string((char *)st.get_raw_data(gid), st.get_lgth(gid)) << ",";
238                scanner.scan_to_next();
239        }
240    }
241    #endif
242
243    #ifdef GID_TEST
244        cout << st.get_max_gid() << endl;
245    #endif
246
247    PERF_SEC_DUMP(parser_timer);
248    PERF_SEC_DESTROY(parser_timer);
249
250    is.close();
251
252    return 1;
253}
254
255
Note: See TracBrowser for help on using the repository browser.