source: trunk/symbol_table/templates/main.template.cpp

Last change on this file was 2595, checked in by ksherdy, 7 years ago

Updated LSST templates.

File size: 8.5 KB
Line 
1/*
2 * Created on: 18-December-2011
3 * Author: Ken Herdy
4 *
5 * A simple test driver.
6 *
7 * Lookahead versus COPYBACK
8 *
9 * The current implementation applies bit stream length grouping based on 'end' markers.
10 * In a sense, 'end' markers are precomputed 'lookahead'.
11 * True 'lookahead' would compute the current block and number of 'lookahead' position and
12 * support 'shift back' and to mark the 'start' rather than the 'end' positions of lexical items.
13 *
14 * In any case, the current implementation 'expects' that the previous block will be located in a contiguous
15 * memory location that may be indexed as some negative offset of the base address of the current
16 * block.
17 *
18 * Further, to reduce complexity in processing, although structs of BitBlock types are not stored
19 * contiguously in memory, BitBlock struct members are copied into contiguous memory positions.
20 *
21 */
22
23#include <simd-lib/allocator.hpp>
24#include <simd-lib/bitblock.hpp>
25#include <simd-lib/bitblock_iterator.hpp>
26#include <simd-lib/buffer.hpp>
27#include <simd-lib/carryQ.hpp>
28#include <simd-lib/s2p.hpp>
29#include <simd-lib/perflib/perfsec.h>
30#include "../src/transpose.hpp"
31
32#include "marker_strms.hpp"     // GENERATED HEADER
33#include "hash_strms.hpp"       // GENERATED HEADER
34#include "group_strms.hpp"      // GENERATED HEADER
35#include <string>
36#include <iostream>
37#include <fstream>
38#include <sstream>
39using namespace std;
40
41#define TEST_DELIMETER // ','
42#include "../src/lsst.hpp"
43
44#ifdef BUFFER_PROFILING
45    BOM_Table * parser_timer;
46#elif PAPI
47                #define PAPI_EVENTS_COUNT 2
48                int PAPI_EVENTS[PAPI_EVENTS_COUNT] = {PAPI_TOT_CYC, PAPI_BR_MSP};       
49    CC * parser_timer;
50#else
51    void * parser_timer;
52#endif
53
54int main(int argc, char * argv[]) {
55
56    if (argc < 2) {
57            cout << "Usage: " << argv[0] << " <filename>" << endl;
58            exit(-1);
59    }
60
61    stringstream filename;
62    filename << argv[1];
63
64    ifstream is;
65    is.open (filename.str().c_str(), ios::binary);
66
67    if (!is) {
68        cerr << "Error: " << filename << endl;
69        abort();
70    }
71
72    // PERF_SEC_BIND(1);
73    PERF_SEC_INIT(parser_timer);
74
75        ///////////////////////////////////////////////////////////////////////////
76        // Buffer Constants
77        ///////////////////////////////////////////////////////////////////////////
78        //
79        // NOTE:    Buffer segments are defined as multiples of BLOCK_SIZE.
80        //              BLOCK_SIZE defined as a number of bytes.
81        #define PADDING_BLOCKS 1
82        #define PADDING_SIZE (BLOCK_SIZE * PADDING_BLOCKS)
83        #define COPYBACK_BLOCKS 3
84        #define COPYBACK_SIZE (BLOCK_SIZE * COPYBACK_BLOCKS)
85        #define LOOKAHEAD_BLOCKS 1
86        #define LOOKAHEAD_SIZE (BLOCK_SIZE * LOOKAHEAD_BLOCKS)
87        #define SEGMENT_BLOCKS  12
88        #define SEGMENT_SIZE (BLOCK_SIZE * SEGMENT_BLOCKS)
89
90        #define BUFFER_SIZE (COPYBACK_SIZE + SEGMENT_SIZE + LOOKAHEAD_SIZE + PADDING_SIZE)
91
92    ///////////////////////////////////////////////////////////////////////////
93    // Parallel Data Streams with optional CopyBack
94    ///////////////////////////////////////////////////////////////////////////
95    uint8_t * COPYBACK;
96    uint8_t * src_buffer;
97    ALLOC_STATIC_ALIGNED_BYTE_BUFFER_WITH_COPYBACK(COPYBACK, src_buffer);
98
99    // hash 0
100    BitBlock * COPYBACK_h0;
101    BitBlock * h0;
102    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_h0, h0);
103
104    // hash 1
105    BitBlock * COPYBACK_h1;
106    BitBlock * h1;
107    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_h1, h1);
108
109    // starts
110    BitBlock * COPYBACK_starts;
111    BitBlock * starts;
112    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_starts, starts);
113
114    ///////////////////////////////////////////////////////////////////////////
115    // Parallel Data Streams - No CopyBack
116    ///////////////////////////////////////////////////////////////////////////
117
118    Basis_bits basis_bits[SEGMENT_BLOCKS];
119    Marker marker[SEGMENT_BLOCKS];
120    Hash hash[SEGMENT_BLOCKS];
121    Groups groups[SEGMENT_BLOCKS];
122
123        ///////////////////////////////////////////////////////////////////////////
124    // Symbol Table
125        ///////////////////////////////////////////////////////////////////////////
126        gid<SEGMENT_SIZE> gids;
127    lsst<gid<SEGMENT_SIZE>, fast_pool_allocator<1024> > symbol_table;
128
129    is.read ((char *)src_buffer, SEGMENT_SIZE);
130    uint32_t chars_avail = is.gcount();
131
132    #if IDENTITY_TEST
133        cout << ","; // prepend delimeter
134    #endif
135
136    ///////////////////////////////////////////////////////////////////////////
137    // Full Segments
138    ///////////////////////////////////////////////////////////////////////////
139    while (chars_avail >= SEGMENT_SIZE) {
140                uint32_t blk;
141                for(blk=0;blk<SEGMENT_BLOCKS;blk++) {
142                        s2p_do_block((BytePack *) &src_buffer[blk*BLOCK_SIZE], basis_bits[blk]);    // transpose
143                        marker_do_block(basis_bits[blk], marker[blk]);                              // gen symbol spans, mark starts & follows
144                        hash_strms_do_block(basis_bits[blk], hash[blk]);                            // gen hash bit streams
145                        group_strms_do_block(marker[blk].starts, marker[blk].follows, groups[blk]);                         // sort marker bit stream (identity)
146    }
147
148    for(int blk=0;blk<SEGMENT_BLOCKS;blk++) {   // write contiguous bit streams
149                        h0[blk] = hash[blk].h0;
150                        h1[blk] = hash[blk].h1;
151                        starts[blk] = groups[blk].starts;
152    }
153
154    PERF_SEC_START(parser_timer);
155    symbol_table.resolve(src_buffer, groups, starts, h0, h1, SEGMENT_BLOCKS, gids);
156    PERF_SEC_END(parser_timer, SEGMENT_SIZE);
157
158    COPY_BACK_BYTE_BUFFER(COPYBACK, src_buffer);
159    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_h0,h0);
160    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_h1,h1);
161    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_starts,starts);
162
163    is.read ((char *)(src_buffer), SEGMENT_SIZE);
164    chars_avail = is.gcount();
165
166    #ifdef IDENTITY_TEST
167    uint32_t blk_offset;
168    for(int blk=0;blk<SEGMENT_BLOCKS;blk++) {
169                        blk_offset = blk * BLOCK_SIZE;
170                        gid_type gid;
171
172                        #ifdef INDEX_AT_STARTS
173                                        ForwardScanner<BitBlock, ScanWord> scanner(&(marker[blk].starts));
174                        #else
175                                        ForwardScanner<BitBlock, ScanWord> scanner(&(marker[blk].follows));
176                        #endif
177
178                        scanner.scan_to_next();
179                        while(!scanner.is_done()) {
180                                gid = gids.at[scanner.get_pos() + blk_offset];
181                                cout << string((char *)symbol_table.get_raw_data(gid), symbol_table.get_lgth(gid)) << ",";
182                                scanner.scan_to_next();
183                        }
184    }
185    #endif
186    }
187
188    /* Resolve Partial Segments */
189    uint32_t remaining = chars_avail;
190
191    ///////////////////////////////////////////////////////////////////////////
192    // Full blocks
193    ///////////////////////////////////////////////////////////////////////////
194    uint32_t blk = 0;
195    while (remaining >= BLOCK_SIZE) {
196                        s2p_do_block((BytePack *) &src_buffer[blk*BLOCK_SIZE], basis_bits[blk]);
197                        marker_do_block(basis_bits[blk], marker[blk]);
198                        hash_strms_do_block(basis_bits[blk], hash[blk]);
199                        group_strms_do_block(marker[blk].starts, marker[blk].follows, groups[blk]);
200                        blk++;
201                        remaining -= BLOCK_SIZE;
202    }
203
204    ///////////////////////////////////////////////////////////////////////////
205    // Final partial block or any carry
206    ///////////////////////////////////////////////////////////////////////////
207    if (remaining > 0 || @marker_strms_any_carry || @group_strms_any_carry /*|| hash_strms_any_carry*/) {
208                        const BitBlock EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining));
209                        s2p_do_final_block((BytePack *) &src_buffer[blk*BLOCK_SIZE], basis_bits[blk], EOF_mask);
210                        marker_do_final_block(basis_bits[blk], marker[blk], EOF_mask);
211                        hash_strms_do_final_block(basis_bits[blk], hash[blk], EOF_mask);
212                        group_strms_do_final_block(marker[blk].starts, marker[blk].follows, groups[blk], EOF_mask);
213                        blk++;
214    }
215
216    uint32_t segment_blocks = blk;
217    for(int blk=0;blk<segment_blocks;blk++) { // write contiguous hash bit streams
218                        h0[blk] = hash[blk].h0;
219                        h1[blk] = hash[blk].h1;
220                        starts[blk] = groups[blk].starts;
221    }
222
223    // PERF_SEC_BIND(0);
224    PERF_SEC_START(parser_timer);
225    symbol_table.resolve(src_buffer, groups, starts, h0, h1, segment_blocks, gids);
226    PERF_SEC_END(parser_timer, chars_avail);
227
228    #ifdef IDENTITY_TEST
229    uint32_t blk_offset;
230    for(int blk=0;blk<segment_blocks;blk++) {
231                blk_offset = blk * BLOCK_SIZE;
232                gid_type gid;
233
234                #ifdef INDEX_AT_STARTS
235                          ForwardScanner<BitBlock, ScanWord> scanner(&(marker[blk].starts));
236                #else
237                          ForwardScanner<BitBlock, ScanWord> scanner(&(marker[blk].follows));
238                #endif
239
240                scanner.scan_to_next();
241                while(!scanner.is_done()) {
242                        gid = gids.at[scanner.get_pos() + blk_offset];
243                        cout << string((char *)symbol_table.get_raw_data(gid), symbol_table.get_lgth(gid)) << ",";
244                        scanner.scan_to_next();
245                }
246    }
247    #endif
248
249    #ifdef GID_TEST
250                        cout << symbol_table.get_max_gid() << endl;
251    #endif
252
253    PERF_SEC_DUMP(parser_timer);
254    PERF_SEC_DESTROY(parser_timer);
255
256    is.close();
257
258    return 1;
259}
260
261
Note: See TracBrowser for help on using the repository browser.