source: trunk/symtab/main_template.cpp @ 1978

Last change on this file since 1978 was 1929, checked in by ksherdy, 8 years ago

Updated identity symbol table.

File size: 5.2 KB
Line 
1/*
2 * Created on: 18-December-2011
3 * Author: Ken Herdy
4 *
5 * Quick and dirty.
6 *
7 */
8
9
10#include "transpose.hpp"
11#include "../lib/bitblock.hpp"
12#include "../lib/allocator.hpp"
13#include "../lib/s2p.hpp"
14
15// GENERATED
16#include "marker_strms.hpp"
17// GENERATED
18#include "hash_strms.hpp"
19// GENERATED
20#include "id_group_strms.hpp"
21
22#include "id_symbol_table.hpp"
23
24#include <string>
25#include <iostream>
26#include <fstream>
27#include <sstream>
28using namespace std;
29
30typedef struct symbol: public AoS_symbol
31{
32    string name;
33    char d;
34} symbol;
35
36#define PADDING_BLOCKS 1
37#define PADDING_SIZE BLOCK_SIZE * PADDING_BLOCKS
38#define LOOKBACK_BLOCKS 1
39#define LOOKBACK_SIZE BLOCK_SIZE * LOOKBACK_BLOCKS
40#define SEGMENT_BLOCKS 8
41#define SEGMENT_SIZE BLOCK_SIZE * (SEGMENT_BLOCKS) // multiple of BLOCK_SIZE (bytes)
42
43int main(int argc, char * argv[]) {
44
45    if (argc < 2) {
46            cout << "Usage: " << argv[0] << " <filename>" << endl;
47            exit(-1);
48    }
49
50    stringstream filename;
51    filename << argv[1];
52
53    ifstream is;
54    is.open (filename.str().c_str(), ios::binary);
55
56    if (!is) {
57        cerr << "Error: " << filename << endl;
58        abort();
59    }
60
61    /* Byte Buffer */
62    BitBlock aligned_buffer[(LOOKBACK_SIZE + SEGMENT_SIZE + PADDING_SIZE) / sizeof(BitBlock) ];
63    uint8_t * lookback = (uint8_t *)aligned_buffer;
64    uint8_t * raw_buffer = lookback + LOOKBACK_SIZE;
65    memset(lookback,0,LOOKBACK_SIZE);
66
67    /* BitSteams */
68    Basis_bits basis_bits[SEGMENT_BLOCKS];
69    Markers markers[SEGMENT_BLOCKS];
70    Hash hash[SEGMENT_BLOCKS];
71    Groups groups[SEGMENT_BLOCKS];
72
73    /* Symbol Table */
74    const uint32_t SYMBOL_COUNT = SEGMENT_SIZE;
75    symbol symbol_ary[SYMBOL_COUNT];
76    id_symbol_table<fast_pool_allocator<1024> > symbol_table;
77
78    is.read ((char *)raw_buffer, SEGMENT_SIZE);
79    uint32_t chars_avail = is.gcount();
80
81    while (chars_avail >= SEGMENT_SIZE) {
82
83      for(uint32_t blk=0;blk<SEGMENT_BLOCKS;blk++) {
84          s2p_do_block((BytePack *) &raw_buffer[blk*BLOCK_SIZE], basis_bits[blk]);
85          markers_do_block(basis_bits[blk], markers[blk]);
86          hash_strms_do_block(basis_bits[blk], hash[blk]);
87          identity_group_do_block(markers[blk], groups[blk]);
88      }
89
90      symbol_table.resolve(raw_buffer, markers, groups, hash, SEGMENT_BLOCKS, symbol_ary, SYMBOL_COUNT);
91
92      memmove(lookback,raw_buffer+SEGMENT_SIZE-LOOKBACK_SIZE,LOOKBACK_SIZE); /* copy final block to lookback */
93      is.read ((char *)(raw_buffer), SEGMENT_SIZE);
94      chars_avail = is.gcount();
95
96//      for(int k=0;k<SEGMENT_BLOCKS;k++) {
97//        cout << "RAW " << string((((char*)&raw_buffer[0])+k*BLOCK_SIZE),BLOCK_SIZE) << endl;
98//      }
99
100    }
101
102    //PERF_SEC_START(parser_timer);
103    /* Partial Segments */
104    uint32_t remaining = chars_avail;
105
106    /* Full Blocks */
107    uint32_t blk = 0;
108    while (remaining >= BLOCK_SIZE) {
109        s2p_do_block((BytePack *) &raw_buffer[blk*BLOCK_SIZE], basis_bits[blk]);
110        markers_do_block(basis_bits[blk], markers[blk]);
111        hash_strms_do_block(basis_bits[blk], hash[blk]);
112        identity_group_do_block(markers[blk], groups[blk]);
113        blk++;
114        remaining -= BLOCK_SIZE;
115    }
116
117    /* Partial Block or carry */
118    if (remaining > 0 || @marker_strms_any_carry /*|| hash_strms_any_carry*/) {
119          BitBlock EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining)); /* null padding byte */
120          s2p_do_final_block((BytePack *) &raw_buffer[blk*BLOCK_SIZE], basis_bits[blk], EOF_mask);
121          markers_do_final_block(basis_bits[blk], markers[blk], EOF_mask);
122          hash_strms_do_final_block(basis_bits[blk], hash[blk], EOF_mask);
123          identity_group_do_final_block(markers[blk], groups[blk], EOF_mask);
124          blk++;
125    }
126
127    symbol_table.resolve(raw_buffer, markers, groups, hash, blk, symbol_ary, SYMBOL_COUNT);
128
129    /* WARNING */
130    // if(remaining==0 && @markers_any_cary) {
131    // @ _any_carry - equivalent to single byte look ahead
132    // Any case in which we have a partial block, we need to know the boundary of the partial block to know when to store carry information.
133    // Any case in which we must evaluate a bit value at position 'one past a boudary' can be handled within final block logic.
134    // }
135
136    // PBS Modules
137    // Pablo block processing structure.
138    // Four Cases --- Move to xml.dnsdojo.com
139    // do_init_block(), do_block(), do_final_block(), do_all()
140    // Four cases
141    // - do_init_block() restrict to initialization only, do_block() does not execute, do_final_block() executes
142
143    // Process full segments in sub modules, ie. sizeof(BitBlock) * 8 bytes,
144    // do_segment()
145    // - do_segment(uint8_t * buffer, BitBlock * strm_1, ..., BitBlock * strm_k, uint32_t byte_count)
146    //
147    // Handle 'while(segments)', while(full blocks), if(partial or carry) on the main application module.
148
149    // Lookahead/Lookback
150    // Current implementation bit stream length grouping strategy leverage 'end' markers.
151    // 'end' markers in a sense are precomputed 'lookahead'
152    // True 'lookahead' would compute the current block and number of 'lookahead' position to
153    // support 'shift back' and the mark the 'start' rather than the 'end' positions of lexical items.
154
155    // The current implementation 'expects' that the previous block will be located in a contiguous
156    // memory location that may be indexed as some negative offset of the base address of the current
157    // block.
158
159    is.close();
160
161    return 1;
162}
163
164
Note: See TracBrowser for help on using the repository browser.