source: trunk/symbol_table/templates/main_template.cpp @ 2279

Last change on this file since 2279 was 2279, checked in by ksherdy, 7 years ago

Update include path.

File size: 8.9 KB
Line 
1/*
2 * Created on: 18-December-2011
3 * Author: Ken Herdy
4 *
5 * A simple test driver.
6 *
7 * Lookahead versus COPYBACK
8 *
9 * The current implementation applies bit stream length grouping based on 'end' markers.
10 * In a sense, 'end' markers are precomputed 'lookahead'.
11 * True 'lookahead' would compute the current block and number of 'lookahead' position and
12 * support 'shift back' and to mark the 'start' rather than the 'end' positions of lexical items.
13 *
14 * In any case, the current implementation 'expects' that the previous block will be located in a contiguous
15 * memory location that may be indexed as some negative offset of the base address of the current
16 * block.
17 *
18 * Further, to reduce complexity in processing, although structs of BitBlock types are not stored
19 * contiguously in memory, BitBlock struct members are copied into contiguous memory positions.
20 *
21 */
22
23#include <simd-lib/allocator.hpp>
24#include <simd-lib/bitblock.hpp>
25#include <simd-lib/bitblock_scan.hpp>
26#include <simd-lib/buffer.hpp>
27#include <simd-lib/carryQ.hpp>
28#include <simd-lib/s2p.hpp>
29#include <simd-lib/perflib/perfsec.h>
30#include "../src/transpose.hpp"
31
32#include "marker_strms.hpp"     // GENERATED HEADER
33#include "hash_strms.hpp"       // GENERATED HEADER
34#include "group_strms.hpp"      // GENERATED HEADER
35#include <string>
36#include <iostream>
37#include <fstream>
38#include <sstream>
39using namespace std;
40
41#define TEST_DELIMETER // ','
42#include "../src/symbol_table.hpp"
43
44#ifdef BUFFER_PROFILING
45    BOM_Table * parser_timer;
46#elif CODE_CLOCKER
47    #define NUM_EVENTS 1
48    int Events[NUM_EVENTS] = {PAPI_TOT_CYC};
49    //int Events[NUM_EVENTS] = {PAPI_L2_DCM};
50    //int Events[NUM_EVENTS] = {PAPI_TOT_CYC, PAPI_BR_MSP};
51    int cal_size = 20;
52    CC * parser_timer = new CC(Events,NUM_EVENTS,cal_size);
53#else
54    void * parser_timer;
55#endif
56
57int main(int argc, char * argv[]) {
58
59    if (argc < 2) {
60            cout << "Usage: " << argv[0] << " <filename>" << endl;
61            exit(-1);
62    }
63
64    stringstream filename;
65    filename << argv[1];
66
67    ifstream is;
68    is.open (filename.str().c_str(), ios::binary);
69
70    if (!is) {
71        cerr << "Error: " << filename << endl;
72        abort();
73    }
74
75    // PERF_SEC_BIND(1);
76    PERF_SEC_INIT(parser_timer);
77
78        ///////////////////////////////////////////////////////////////////////////
79        // Buffer Constants
80        ///////////////////////////////////////////////////////////////////////////
81        //
82        // NOTE:    Buffer segments are defined as multiples of BLOCK_SIZE.
83        //              BLOCK_SIZE defined as a number of bytes.
84        #define PADDING_BLOCKS 1
85        #define PADDING_SIZE (BLOCK_SIZE * PADDING_BLOCKS)
86        #define COPYBACK_BLOCKS 3
87        #define COPYBACK_SIZE (BLOCK_SIZE * COPYBACK_BLOCKS)
88        #define LOOKAHEAD_BLOCKS 1
89        #define LOOKAHEAD_SIZE (BLOCK_SIZE * LOOKAHEAD_BLOCKS)
90        #define SEGMENT_BLOCKS  12
91        #define SEGMENT_SIZE (BLOCK_SIZE * SEGMENT_BLOCKS)
92
93        #define BUFFER_SIZE (COPYBACK_SIZE + SEGMENT_SIZE + LOOKAHEAD_SIZE + PADDING_SIZE)
94
95    ///////////////////////////////////////////////////////////////////////////
96    // Parallel Data Streams with optional CopyBack
97    ///////////////////////////////////////////////////////////////////////////
98    uint8_t * COPYBACK;
99    uint8_t * src_buffer;
100    ALLOC_STATIC_ALIGNED_BYTE_BUFFER_WITH_COPYBACK(COPYBACK, src_buffer);
101
102    // hash 0
103    BitBlock * COPYBACK_h0;
104    BitBlock * h0;
105    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_h0, h0);
106
107    // hash 1
108    BitBlock * COPYBACK_h1;
109    BitBlock * h1;
110    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_h1, h1);
111
112    // starts
113    BitBlock * COPYBACK_starts;
114    BitBlock * starts;
115    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_starts, starts);
116
117    // follows_0 - Arbitrary length symbols
118    BitBlock * COPYBACK_follows_0;
119    BitBlock * follows_0;
120    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_follows_0, follows_0);
121
122    ///////////////////////////////////////////////////////////////////////////
123    // Parallel Data Streams - No CopyBack
124    ///////////////////////////////////////////////////////////////////////////
125
126    Basis_bits basis_bits[SEGMENT_BLOCKS];
127    Markers markers[SEGMENT_BLOCKS];
128    Hash hash[SEGMENT_BLOCKS];
129    Groups groups[SEGMENT_BLOCKS];
130
131        ///////////////////////////////////////////////////////////////////////////
132    // Symbol Table
133        ///////////////////////////////////////////////////////////////////////////
134        gid<SEGMENT_SIZE> gids;
135    symbol_table<gid<SEGMENT_SIZE>, fast_pool_allocator<1024> > st;
136
137    is.read ((char *)src_buffer, SEGMENT_SIZE);
138    uint32_t chars_avail = is.gcount();
139
140    #if IDENTITY_TEST
141        cout << ","; // prepend delimeter
142    #endif
143
144    ///////////////////////////////////////////////////////////////////////////
145    // Full Segments
146    ///////////////////////////////////////////////////////////////////////////
147    while (chars_avail >= SEGMENT_SIZE) {
148                uint32_t blk;
149                for(blk=0;blk<SEGMENT_BLOCKS;blk++) {
150                        s2p_do_block((BytePack *) &src_buffer[blk*BLOCK_SIZE], basis_bits[blk]);    // transpose
151                        markers_do_block(basis_bits[blk], markers[blk]);                            // gen symbol spans, mark starts & follows
152                        hash_strms_do_block(basis_bits[blk], hash[blk]);                            // gen hash bit streams
153                        identity_group_do_block(markers[blk], groups[blk]);                         // sort marker bit stream (identity)
154    }
155
156    for(int blk=0;blk<SEGMENT_BLOCKS;blk++) {   // write contiguous bit streams
157                        h0[blk] = hash[blk].h0;
158                        h1[blk] = hash[blk].h1;
159                        starts[blk] = groups[blk].starts;
160                        follows_0[blk] = groups[blk].follows_0;
161    }
162
163    PERF_SEC_START(parser_timer);
164    st.resolve(src_buffer, groups, starts, follows_0, h0, h1, SEGMENT_BLOCKS, gids);
165    PERF_SEC_END(parser_timer, SEGMENT_SIZE);
166
167    COPY_BACK_BYTE_BUFFER(COPYBACK, src_buffer);
168    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_h0,h0);
169    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_h1,h1);
170    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_starts,starts);
171    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_follows_0,follows_0);
172
173    is.read ((char *)(src_buffer), SEGMENT_SIZE);
174    chars_avail = is.gcount();
175
176    #ifdef IDENTITY_TEST
177    uint32_t blk_offset;
178    for(int blk=0;blk<SEGMENT_BLOCKS;blk++) {
179                        blk_offset = blk * BLOCK_SIZE;
180                        gid_type gid;
181
182                        #ifdef INDEX_AT_STARTS
183                                        ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].starts));
184                        #else
185                                        ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].follows));
186                        #endif
187
188                        scanner.scan_to_next();
189                        while(!scanner.is_done()) {
190                                gid = gids.at[scanner.get_pos() + blk_offset];
191                                cout << string((char *)st.get_raw_data(gid), st.get_lgth(gid)) << ",";
192                                scanner.scan_to_next();
193                        }
194    }
195    #endif
196    }
197
198    /* Resolve Partial Segments */
199    uint32_t remaining = chars_avail;
200
201    ///////////////////////////////////////////////////////////////////////////
202    // Full blocks
203    ///////////////////////////////////////////////////////////////////////////
204    uint32_t blk = 0;
205    while (remaining >= BLOCK_SIZE) {
206                        s2p_do_block((BytePack *) &src_buffer[blk*BLOCK_SIZE], basis_bits[blk]);
207                        markers_do_block(basis_bits[blk], markers[blk]);
208                        hash_strms_do_block(basis_bits[blk], hash[blk]);
209                        identity_group_do_block(markers[blk], groups[blk]);
210                        blk++;
211                        remaining -= BLOCK_SIZE;
212    }
213
214    ///////////////////////////////////////////////////////////////////////////
215    // Final partial block or any carry
216    ///////////////////////////////////////////////////////////////////////////
217    if (remaining > 0 || @marker_strms_any_carry || @group_strms_any_carry /*|| hash_strms_any_carry*/) {
218                        BitBlock EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining));
219                        s2p_do_final_block((BytePack *) &src_buffer[blk*BLOCK_SIZE], basis_bits[blk], EOF_mask);
220                        markers_do_final_block(basis_bits[blk], markers[blk], EOF_mask);
221                        hash_strms_do_final_block(basis_bits[blk], hash[blk], EOF_mask);
222                        identity_group_do_final_block(markers[blk], groups[blk], EOF_mask);
223                        blk++;
224    }
225
226    uint32_t segment_blocks = blk;
227    for(int blk=0;blk<segment_blocks;blk++) { // write contiguous hash bit streams
228                        h0[blk] = hash[blk].h0;
229                        h1[blk] = hash[blk].h1;
230                        starts[blk] = groups[blk].starts;
231                        follows_0[blk] = groups[blk].follows_0;
232    }
233
234    // PERF_SEC_BIND(0);
235    PERF_SEC_START(parser_timer);
236    st.resolve(src_buffer, groups, starts, follows_0, h0, h1, segment_blocks, gids);
237    PERF_SEC_END(parser_timer, chars_avail);
238
239    #ifdef IDENTITY_TEST
240    uint32_t blk_offset;
241    for(int blk=0;blk<segment_blocks;blk++) {
242                blk_offset = blk * BLOCK_SIZE;
243                gid_type gid;
244
245                #ifdef INDEX_AT_STARTS
246                          ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].starts));
247                #else
248                          ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].follows));
249                #endif
250
251                scanner.scan_to_next();
252                while(!scanner.is_done()) {
253                        gid = gids.at[scanner.get_pos() + blk_offset];
254                        cout << string((char *)st.get_raw_data(gid), st.get_lgth(gid)) << ",";
255                        scanner.scan_to_next();
256                }
257    }
258    #endif
259
260    #ifdef GID_TEST
261                        cout << st.get_max_gid() << endl;
262    #endif
263
264    PERF_SEC_DUMP(parser_timer);
265    PERF_SEC_DESTROY(parser_timer);
266
267    is.close();
268
269    return 1;
270}
271
272
Note: See TracBrowser for help on using the repository browser.