source: trunk/symbol_table/main_template.cpp @ 2058

Last change on this file since 2058 was 2058, checked in by ksherdy, 7 years ago

Fixed EOF carry bug in test driver.

File size: 8.7 KB
Line 
1/*
2 * Created on: 18-December-2011
3 * Author: Ken Herdy
4 *
5 * A simple test driver that perform an 'identity' translation of CSV (single) seperate symbol values
6 * and prints results to standard output.
7 *
8 * Length sorted symbol table main.
9 *
10 * Lookahead versus Lookback
11 *
12 * The current implementation applies bit stream length grouping based on 'end' markers.
13 * In a sense, 'end' markers are precomputed 'lookahead'.
14 * True 'lookahead' would compute the current block and number of 'lookahead' position and
15 * support 'shift back' and to mark the 'start' rather than the 'end' positions of lexical items.
16 *
17 * In any case, the current implementation 'expects' that the previous block will be located in a contiguous
18 * memory location that may be indexed as some negative offset of the base address of the current
19 * block.
20 *
21 * Further, to reduce complexity in processing, although structs of BitBlock types are not stored
22 * contiguously in memory, BitBlock struct members are copied into contiguous memory positions.
23 *
24 * Design Considerations.
25 *
26 * (1) Hash table memory allocation.
27 * (2) Negative shift values.
28 * (3) Whether to specialized Hash/Symbol classes on symbol length...
29 *
30 */
31
32#include "transpose.hpp"
33#include "buffer.hpp"
34#include "../lib/bitblock.hpp"
35#include "../lib/allocator.hpp"
36#include "../lib/s2p.hpp"
37#include "../lib/perflib/perfsec.h"
38#include "../lib/bitblock_scan.hpp"
39#include "marker_strms.hpp"     // GENERATED HEADER
40#include "hash_strms.hpp"       // GENERATED HEADER
41#include "group_strms.hpp"      // GENERATED HEADER
42#include "symbol_table.hpp"
43#include <string>
44#include <iostream>
45#include <fstream>
46#include <sstream>
47using namespace std;
48
49#ifdef BUFFER_PROFILING
50    BOM_Table * parser_timer;
51#elif CODE_CLOCKER
52    #define NUM_EVENTS 1
53    int Events[NUM_EVENTS] = {PAPI_TOT_CYC};
54    //int Events[NUM_EVENTS] = {PAPI_L2_DCM};
55    //int Events[NUM_EVENTS] = {PAPI_TOT_CYC, PAPI_BR_MSP};
56    int cal_size = 20;
57    CC * parser_timer = new CC(Events,NUM_EVENTS,cal_size);
58#else
59    void * parser_timer;
60#endif
61
62int main(int argc, char * argv[]) {
63
64    if (argc < 2) {
65            cout << "Usage: " << argv[0] << " <filename>" << endl;
66            exit(-1);
67    }
68
69    stringstream filename;
70    filename << argv[1];
71
72    ifstream is;
73    is.open (filename.str().c_str(), ios::binary);
74
75    if (!is) {
76        cerr << "Error: " << filename << endl;
77        abort();
78    }
79
80    // PERF_SEC_BIND(1);
81    PERF_SEC_INIT(parser_timer);
82
83    ///////////////////////////////////////////////////////////////////////////
84    // Stream Definitions
85    ///////////////////////////////////////////////////////////////////////////
86
87    // Byte Segments - Raw byte streams - With lookback.
88    BitBlock aligned_buffer[SEGMENT_ALLOC_SIZE];
89    uint8_t * lookback = (uint8_t *)aligned_buffer;
90    memset(lookback,0,LOOKBACK_SIZE);
91    uint8_t * raw_buffer = &lookback[LOOKBACK_SIZE];
92
93    // Bit Segments - Hash bit streams - With lookback.
94
95    // hash 0
96    BitBlock aligned_h0[SEGMENT_ALLOC_SIZE/8];
97    BitBlock * lookback_h0 = (BitBlock *) aligned_h0;
98    memset(lookback_h0,0,LOOKBACK_SIZE/BLOCK_SIZE);
99    BitBlock * h0 = &lookback_h0[LOOKBACK_SIZE/BLOCK_SIZE];
100
101    // hash 1
102    BitBlock aligned_h1[SEGMENT_ALLOC_SIZE/8];
103    BitBlock * lookback_h1 = (BitBlock *) aligned_h1;
104    memset(lookback_h1,0,LOOKBACK_SIZE/BLOCK_SIZE);
105    BitBlock * h1 = &lookback_h1[LOOKBACK_SIZE/BLOCK_SIZE];
106
107    // starts
108    BitBlock aligned_starts[SEGMENT_ALLOC_SIZE/8];
109    BitBlock * lookback_starts = (BitBlock *) aligned_starts;
110    memset(lookback_starts,0,LOOKBACK_SIZE/BLOCK_SIZE);
111    BitBlock * starts = &lookback_starts[LOOKBACK_SIZE/BLOCK_SIZE];
112
113    // ends_gte_17
114    BitBlock aligned_ends_gte_17[SEGMENT_ALLOC_SIZE/8];
115    BitBlock * lookback_ends_gte_17 = (BitBlock *) aligned_ends_gte_17;
116    memset(lookback_ends_gte_17,0,LOOKBACK_SIZE/BLOCK_SIZE);
117    BitBlock * ends_gte_17 = &lookback_ends_gte_17[LOOKBACK_SIZE/BLOCK_SIZE];
118
119    // BitSteams - Without lookback
120    Basis_bits basis_bits[SEGMENT_BLOCKS];
121    Markers markers[SEGMENT_BLOCKS];
122    Hash hash[SEGMENT_BLOCKS];
123    Groups groups[SEGMENT_BLOCKS];
124
125    // Symbol Table
126    const uint32_t SYMBOL_COUNT = SEGMENT_SIZE;
127
128    Symbol symbols(SYMBOL_COUNT);
129    symbol_table<Symbol, fast_pool_allocator<1024> > st;
130
131    is.read ((char *)raw_buffer, SEGMENT_SIZE);
132    uint32_t chars_avail = is.gcount();
133
134    ///////////////////////////////////////////////////////////////////////////
135    // Full Segments
136    ///////////////////////////////////////////////////////////////////////////
137    while (chars_avail >= SEGMENT_SIZE) {
138                                uint32_t blk;
139                                for(blk=0;blk<SEGMENT_BLOCKS;blk++) {
140                                s2p_do_block((BytePack *) &raw_buffer[blk*BLOCK_SIZE], basis_bits[blk]);    // transpose
141                                markers_do_block(basis_bits[blk], markers[blk]);                            // gen symbol spans, mark starts & follows
142                                hash_strms_do_block(basis_bits[blk], hash[blk]);                            // gen hash bit streams
143                                identity_group_do_block(markers[blk], groups[blk]);                         // sort marker bit stream (identity)
144      }
145
146      for(int blk=0;blk<SEGMENT_BLOCKS;blk++) { // write contiguous bit streams
147                                h0[blk] = hash[blk].h0;
148                                h1[blk] = hash[blk].h1;
149                                starts[blk] = groups[blk].starts;
150                                ends_gte_17[blk] = groups[blk].ends_gte_17;
151      }
152
153      PERF_SEC_START(parser_timer);
154      st.resolve(raw_buffer, groups, starts, ends_gte_17, h0, h1, SEGMENT_BLOCKS, symbols /*, SYMBOL_COUNT*/);
155      PERF_SEC_END(parser_timer, SEGMENT_SIZE);
156
157      // copy loopback bytes
158      memmove(lookback,&raw_buffer[SEGMENT_SIZE-LOOKBACK_SIZE],LOOKBACK_SIZE);
159      // copy loopback bits
160      memmove(lookback_h0,&((uint8_t *)h0)[(SEGMENT_SIZE-LOOKBACK_SIZE)/8],LOOKBACK_SIZE/8);
161      memmove(lookback_h1,&((uint8_t *)h1)[(SEGMENT_SIZE-LOOKBACK_SIZE)/8],LOOKBACK_SIZE/8);
162
163      memmove(lookback_starts,&((uint8_t *)starts)[(SEGMENT_SIZE-LOOKBACK_SIZE)/8],LOOKBACK_SIZE/8);
164      memmove(lookback_ends_gte_17,&((uint8_t *)ends_gte_17)[(SEGMENT_SIZE-LOOKBACK_SIZE)/8],LOOKBACK_SIZE/8);
165
166      //lookback_h0[0] = h0[SEGMENT_BLOCKS-1];
167      //lookback_h1[0] = h1[SEGMENT_BLOCKS-1];
168      is.read ((char *)(raw_buffer), SEGMENT_SIZE);
169      chars_avail = is.gcount();
170
171      // test
172      uint32_t blk_offset;
173      for(int blk=0;blk<SEGMENT_BLOCKS;blk++) {
174                                blk_offset = blk * BLOCKSIZE;
175                                gid_type gid;
176                                ForwardScanner<BitBlock, scanword_t> fscanner(&(groups[blk].starts));
177
178                                fscanner.scan_to_next();
179                                while(!fscanner.is_done()) {
180                                        gid = symbols.gids[fscanner.get_pos() + blk_offset];
181                                        cout << string((char *)st.get_raw_data(gid), st.get_lgth(gid)) << ",";
182                                        fscanner.scan_to_next();
183                                }
184      }
185    }
186    /* Resolve Partial Segments */
187    uint32_t remaining = chars_avail;
188
189    ///////////////////////////////////////////////////////////////////////////
190    // Full blocks
191    ///////////////////////////////////////////////////////////////////////////
192    uint32_t blk = 0;
193    while (remaining >= BLOCK_SIZE) {
194                        s2p_do_block((BytePack *) &raw_buffer[blk*BLOCK_SIZE], basis_bits[blk]);
195                        markers_do_block(basis_bits[blk], markers[blk]);
196                        hash_strms_do_block(basis_bits[blk], hash[blk]);
197                        identity_group_do_block(markers[blk], groups[blk]);
198                        blk++;
199                        remaining -= BLOCK_SIZE;
200    }
201
202    ///////////////////////////////////////////////////////////////////////////
203    // Final partial block or any carry
204    ///////////////////////////////////////////////////////////////////////////
205    if (remaining > 0 || @marker_strms_any_carry || @group_strms_any_carry /*|| hash_strms_any_carry*/) {
206                        BitBlock EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining));
207                        s2p_do_final_block((BytePack *) &raw_buffer[blk*BLOCK_SIZE], basis_bits[blk], EOF_mask);
208                        markers_do_final_block(basis_bits[blk], markers[blk], EOF_mask);
209                        hash_strms_do_final_block(basis_bits[blk], hash[blk], EOF_mask);
210                        identity_group_do_final_block(markers[blk], groups[blk], EOF_mask);
211                        blk++;
212    }
213
214    uint32_t segment_size = blk;
215    for(int blk=0;blk<segment_size;blk++) { // write contiguous hash bit streams
216                        h0[blk] = hash[blk].h0;
217                        h1[blk] = hash[blk].h1;
218                        starts[blk] = groups[blk].starts;
219                        ends_gte_17[blk] = groups[blk].ends_gte_17;
220    }
221
222    //PERF_SEC_START(parser_timer);
223    st.resolve(raw_buffer, groups, starts, ends_gte_17, h0, h1, segment_size, symbols/*, SYMBOL_COUNT*/);
224    //PERF_SEC_END(parser_timer, chars_avail+1);
225
226    uint32_t blk_offset;
227    for(int blk=0;blk<segment_size;blk++) {
228                        blk_offset = blk * BLOCKSIZE;
229                        gid_type gid;
230                        ForwardScanner<BitBlock, scanword_t> fscanner(&(groups[blk].starts));
231
232                        fscanner.scan_to_next();
233                        while(!fscanner.is_done()) {
234                                gid = symbols.gids[fscanner.get_pos() + blk_offset];
235                                cout << string((char *)st.get_raw_data(gid), st.get_lgth(gid)) << ",";
236                                fscanner.scan_to_next();
237                        }
238
239    }
240
241    PERF_SEC_DUMP(parser_timer);
242    PERF_SEC_DESTROY(parser_timer);
243
244    is.close();
245
246    return 1;
247}
248
249
Note: See TracBrowser for help on using the repository browser.