Changeset 1969 for trunk/symbol_table/main_template.cpp
- Timestamp:
- Mar 25, 2012, 12:11:27 AM (7 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/symbol_table/main_template.cpp
r1960 r1969 3 3 * Author: Ken Herdy 4 4 * 5 * Quick and dirty. 5 * Length sorted symbol table main. 6 * 7 * Lookahead versus Lookback 8 * 9 * The current implementation applies bit stream length grouping based on 'end' markers. 10 * In a sense, 'end' markers are precomputed 'lookahead'. 11 * True 'lookahead' would compute the current block and number of 'lookahead' position and 12 * support 'shift back' and to mark the 'start' rather than the 'end' positions of lexical items. 13 * 14 * In any case, the current implementation 'expects' that the previous block will be located in a contiguous 15 * memory location that may be indexed as some negative offset of the base address of the current 16 * block. 17 * 18 * Further, to reduce complexity in processing, although structs of BitBlock types are not stored 19 * contiguously in memory, BitBlock struct members are copied into contiguous memory positions. 20 * 21 * Design Issues 22 * 23 * (1) Max hash table size. 24 * (2) Negative shift values. 25 * (3) Template classes on Length L. 6 26 * 7 27 */ … … 12 32 #include "../lib/s2p.hpp" 13 33 #include "../lib/perflib/perfsec.h" 14 15 // GENERATED 16 #include "marker_strms.hpp" 17 // GENERATED 18 #include "hash_strms.hpp" 19 // GENERATED 20 #include "id_group_strms.hpp" 21 34 #include "marker_strms.hpp" // GENERATED HEADER 35 #include "hash_strms.hpp" // GENERATED HEADER 36 #include "id_group_strms.hpp" // GENERATED HEADER 22 37 #include "id_symbol_table.hpp" 23 24 38 #include <string> 25 39 #include <iostream> … … 41 55 #endif 42 56 43 typedef struct symbol: public AoS_symbol 44 { 45 string name; 46 char d; 47 } symbol; 48 57 /////////////////////////////////////////////////////////////////////////// 58 // Buffer Management 59 /////////////////////////////////////////////////////////////////////////// 49 60 #define PADDING_BLOCKS 1 50 61 #define PADDING_SIZE BLOCK_SIZE * PADDING_BLOCKS 51 62 #define LOOKBACK_BLOCKS 1 52 63 #define LOOKBACK_SIZE BLOCK_SIZE * LOOKBACK_BLOCKS 53 #define SEGMENT_BLOCKS 8 54 #define SEGMENT_SIZE BLOCK_SIZE * (SEGMENT_BLOCKS) // multiple of BLOCK_SIZE (bytes) 64 #define SEGMENT_BLOCKS 10 65 #define SEGMENT_SIZE BLOCK_SIZE * (SEGMENT_BLOCKS) // (bytes) a multiple of BLOCK_SIZE 66 #define SEGMENT_ALLOC_SIZE (LOOKBACK_SIZE + SEGMENT_SIZE + PADDING_SIZE) / sizeof(BitBlock) // (bytes) 67 68 // Target symbol type must inherit from AoS_symbol 69 typedef struct symbol: public AoS_symbol 70 { 71 string parameter_1; 72 string parameter_2; 73 bool parameter_3; 74 } symbol; 55 75 56 76 int main(int argc, char * argv[]) { … … 75 95 PERF_SEC_INIT(parser_timer); 76 96 77 /* Byte Buffer */ 78 BitBlock aligned_buffer[(LOOKBACK_SIZE + SEGMENT_SIZE + PADDING_SIZE) / sizeof(BitBlock)]; 97 /////////////////////////////////////////////////////////////////////////// 98 // Stream Definitions 99 /////////////////////////////////////////////////////////////////////////// 100 101 // Byte Segments - Raw byte streams - With lookback. 102 BitBlock aligned_buffer[SEGMENT_ALLOC_SIZE]; 79 103 uint8_t * lookback = (uint8_t *)aligned_buffer; 80 104 memset(lookback,0,LOOKBACK_SIZE); 81 uint8_t * raw_buffer = lookback + LOOKBACK_SIZE; 82 83 /* Bit Stream Hash Buffers */ 84 85 // TODO - Verify 86 87 /* h0 */ 88 BitBlock aligned_h0[(LOOKBACK_SIZE + SEGMENT_SIZE + PADDING_SIZE) / sizeof(BitBlock)]; 105 uint8_t * raw_buffer = &lookback[LOOKBACK_SIZE]; 106 107 // Bit Segments - Hash bit streams - With lookback. 108 109 // hash 0 110 BitBlock aligned_h0[SEGMENT_ALLOC_SIZE/8]; 89 111 BitBlock * lookback_h0 = (BitBlock *) aligned_h0; 90 memset(lookback_h0,0, sizeof(BitBlock));91 BitBlock * h0 = (BitBlock *)(lookback_h0 + 1); // TODO - Correct92 93 / * h1 */94 BitBlock aligned_h1[ (LOOKBACK_SIZE + SEGMENT_SIZE + PADDING_SIZE) / sizeof(BitBlock)];112 memset(lookback_h0,0,LOOKBACK_SIZE/BLOCK_SIZE); 113 BitBlock * h0 = &lookback_h0[LOOKBACK_SIZE/BLOCK_SIZE]; 114 115 // hash 1 116 BitBlock aligned_h1[SEGMENT_ALLOC_SIZE/8]; 95 117 BitBlock * lookback_h1 = (BitBlock *) aligned_h1; 96 memset(lookback_h1,0, sizeof(BitBlock));97 BitBlock * h1 = (BitBlock *)(lookback_h1 + 1); // TODO - Correct98 99 / * BitSteams */118 memset(lookback_h1,0,LOOKBACK_SIZE/BLOCK_SIZE); 119 BitBlock * h1 = &lookback_h1[LOOKBACK_SIZE/BLOCK_SIZE]; 120 121 // BitSteams - Without lookback 100 122 Basis_bits basis_bits[SEGMENT_BLOCKS]; 101 123 Markers markers[SEGMENT_BLOCKS]; … … 103 125 Groups groups[SEGMENT_BLOCKS]; 104 126 105 / * Symbol Table */127 // Symbol Table 106 128 const uint32_t SYMBOL_COUNT = SEGMENT_SIZE; 107 129 symbol symbol_ary[SYMBOL_COUNT]; … … 111 133 uint32_t chars_avail = is.gcount(); 112 134 135 /////////////////////////////////////////////////////////////////////////// 136 // Full Segments 137 /////////////////////////////////////////////////////////////////////////// 113 138 while (chars_avail >= SEGMENT_SIZE) { 114 139 115 140 uint32_t blk; 116 141 for(blk=0;blk<SEGMENT_BLOCKS;blk++) { 117 s2p_do_block((BytePack *) &raw_buffer[blk*BLOCK_SIZE], basis_bits[blk]);118 markers_do_block(basis_bits[blk], markers[blk]);119 hash_strms_do_block(basis_bits[blk], hash[blk]);120 identity_group_do_block(markers[blk], groups[blk]);142 s2p_do_block((BytePack *) &raw_buffer[blk*BLOCK_SIZE], basis_bits[blk]); // transpose 143 markers_do_block(basis_bits[blk], markers[blk]); // gen symbol spans, mark starts & follows 144 hash_strms_do_block(basis_bits[blk], hash[blk]); // gen hash bit streams 145 identity_group_do_block(markers[blk], groups[blk]); // sort marker bit stream (identity) 121 146 } 122 147 123 // 148 // for(int k=0;k<SEGMENT_BLOCKS;k++) { 124 149 // cout << "RAW " << string((((char*)&raw_buffer[0])+k*BLOCK_SIZE),BLOCK_SIZE) << endl; 125 // } 126 127 /* Write contiguous hash bit streams */ 128 for(int blk=0;blk<SEGMENT_BLOCKS;blk++) { 129 h0[blk] = hash[blk].h0; 130 h1[blk] = hash[blk].h1; 150 // } 151 152 for(int blk=0;blk<SEGMENT_BLOCKS;blk++) { // write contiguous hash bit streams 153 h0[blk] = hash[blk].h0; 154 h1[blk] = hash[blk].h1; 131 155 } 132 156 … … 135 159 PERF_SEC_END(parser_timer, SEGMENT_SIZE); 136 160 137 // print_register("h1[S]",h1[SEGMENT_BLOCKS-1]); 138 139 memmove(lookback,raw_buffer+SEGMENT_SIZE-LOOKBACK_SIZE,LOOKBACK_SIZE); /* copy final block to lookback */ 140 //memmove(lookback_h0,((uint8_t *)h0)+((SEGMENT_SIZE-LOOKBACK_SIZE)/sizeof(BitBlock)),sizeof(BitBlock)); /* copy final block to lookback */ 141 //memmove(lookback_h1,((uint8_t *)h1)+((SEGMENT_SIZE-LOOKBACK_SIZE)/sizeof(BitBlock)),sizeof(BitBlock)); /* copy final block to lookback */ 142 143 lookback_h0[0] = h0[SEGMENT_BLOCKS-1]; 144 lookback_h1[0] = h1[SEGMENT_BLOCKS-1]; 145 //print_register<BitBlock>("h1[S]",*(BitBlock *)&lookback_h1[0]); 146 147 //exit(1); 148 161 // copy loopback bytes 162 memmove(lookback,&raw_buffer[SEGMENT_SIZE-LOOKBACK_SIZE],LOOKBACK_SIZE); 163 // copy loopback bits 164 memmove(lookback_h0,&((uint8_t *)h0)[(SEGMENT_SIZE-LOOKBACK_SIZE)/8],LOOKBACK_SIZE/8); 165 memmove(lookback_h1,&((uint8_t *)h1)[(SEGMENT_SIZE-LOOKBACK_SIZE)/8],LOOKBACK_SIZE/8); 166 167 //lookback_h0[0] = h0[SEGMENT_BLOCKS-1]; 168 //lookback_h1[0] = h1[SEGMENT_BLOCKS-1]; 149 169 is.read ((char *)(raw_buffer), SEGMENT_SIZE); 150 170 chars_avail = is.gcount(); 151 152 } 153 154 //PERF_SEC_START(parser_timer); 155 /* Partial Segments */ 156 171 } 172 173 /* Resolve Partial Segments */ 157 174 uint32_t remaining = chars_avail; 158 175 159 /* Full Blocks */ 176 /////////////////////////////////////////////////////////////////////////// 177 // Full blocks 178 /////////////////////////////////////////////////////////////////////////// 160 179 uint32_t blk = 0; 161 180 while (remaining >= BLOCK_SIZE) { … … 168 187 } 169 188 170 /* Partial Block or carry */ 189 /////////////////////////////////////////////////////////////////////////// 190 // Final partial block or any carry 191 /////////////////////////////////////////////////////////////////////////// 171 192 if (remaining > 0 || @marker_strms_any_carry /*|| hash_strms_any_carry*/) { 172 BitBlock EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining)); /* null padding byte */193 BitBlock EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining)); 173 194 s2p_do_final_block((BytePack *) &raw_buffer[blk*BLOCK_SIZE], basis_bits[blk], EOF_mask); 174 195 markers_do_final_block(basis_bits[blk], markers[blk], EOF_mask); … … 182 203 // } 183 204 184 /* Write contiguous hash bit streams */ 205 185 206 uint32_t segment_size = blk; 186 187 for(int blk=0;blk<segment_size;blk++) { 207 for(int blk=0;blk<segment_size;blk++) { // write contiguous hash bit streams 188 208 h0[blk] = hash[blk].h0; 189 209 h1[blk] = hash[blk].h1; 190 210 } 191 211 192 PERF_SEC_START(parser_timer);212 //PERF_SEC_START(parser_timer); 193 213 symbol_table.resolve(raw_buffer, groups, h0, h1, blk, symbol_ary, SYMBOL_COUNT); 194 PERF_SEC_END(parser_timer, chars_avail+1); 195 196 /* WARNING */ 197 // if(remaining==0 && @markers_any_cary) { 198 // @ _any_carry - equivalent to single byte look ahead 199 // Any case in which we have a partial block, we need to know the boundary of the partial block to know when to store carry information. 200 // Any case in which we must evaluate a bit value at position 'one past a boudary' can be handled within final block logic. 201 // } 202 203 // PBS Modules 204 // Pablo block processing structure. 205 // Four Cases --- Move to xml.dnsdojo.com 206 // do_init_block(), do_block(), do_final_block(), do_all() 207 // Four cases 208 // - do_init_block() restrict to initialization only, do_block() does not execute, do_final_block() executes 209 210 // Process full segments in sub modules, ie. sizeof(BitBlock) * 8 bytes, 211 // do_segment() 212 // - do_segment(uint8_t * buffer, BitBlock * strm_1, ..., BitBlock * strm_k, uint32_t byte_count) 213 // 214 // Handle 'while(segments)', while(full blocks), if(partial or carry) on the main application module. 215 216 // Lookahead/Lookback 217 // Current implementation bit stream length grouping strategy leverage 'end' markers. 218 // 'end' markers in a sense are precomputed 'lookahead' 219 // True 'lookahead' would compute the current block and number of 'lookahead' position to 220 // support 'shift back' and the mark the 'start' rather than the 'end' positions of lexical items. 221 222 // The current implementation 'expects' that the previous block will be located in a contiguous 223 // memory location that may be indexed as some negative offset of the base address of the current 224 // block. 225 226 // Max hash table size, negative shift values. 227 // Both in symbols that cross boundaries as well as in hash_strategy classes, hard coding. 228 // Template parameters on Length L, 229 // Bits vs. Bytes? 214 //PERF_SEC_END(parser_timer, chars_avail+1); 230 215 231 216 PERF_SEC_DUMP(parser_timer);
Note: See TracChangeset
for help on using the changeset viewer.