Context Navigation

pablo_template_symbol_table.cpp @ 2202

Revision 2202, 13.3 KB (checked in by ksherdy, 13 months ago)
Add #define SYMBOL_TABLE.

Line
1	#include <stdio.h>
2	#include <stdlib.h>
3	#include <errno.h>
4	#include <sys/types.h>
5	#include <sys/stat.h>
6
7	#define LocalCarryDeclare(name, count)\
8	CarryArray<count> name;\
9
10	#include "../lib/bitblock.hpp"
11	#include "../lib/carryQ.hpp"
12	#include "../lib/bitstream_iterator.hpp"
13	#include "../lib/bitblock_iterator.hpp"
14	#include "../lib/s2p.hpp"
15	#include "../lib/perflib/perfsec.h"
16
17	#include <xmldecl.h>
18	#include <namechars.h>
19	#include <TagMatcher.hpp>
20	#include <LineColTracker.hpp>
21	#include <ErrorUtil.h>
22	#include <ErrorTracker.h>
23	#include <XMLTestSuiteError.h>
24
25	#define SYMBOL_TABLE
26
27	#ifdef BUFFER_PROFILING
28	BOM_Table * parser_timer;
29	#elif CODE_CLOCKER
30	//#define NUM_EVENTS 1
31	//int Events[NUM_EVENTS] = {PAPI_TOT_CYC};
32	//int Events[NUM_EVENTS] = {PAPI_L2_DCM};
33	#define NUM_EVENTS 2
34	int Events[NUM_EVENTS] = {PAPI_TOT_CYC, PAPI_BR_MSP};
35	int cal_size = 20;
36	CC * parser_timer = new CC(Events,NUM_EVENTS,cal_size);
37	#else
38	void * parser_timer;
39	#endif
40
41	ErrorTracker error_tracker;
42	BitBlock EOF_mask = simd<1>::constant<1>();
43
44	//////////////////////////////////////////////////////////////////////////////////////////
45	// Buffer Management // WARNING: Do Not update #defines. Results in TagMatcher errors.
46	//////////////////////////////////////////////////////////////////////////////////////////
47	#define PADDING_BLOCKS 0
48	#define PADDING_SIZE (BLOCK_SIZE * PADDING_BLOCKS)
49	#define COPYBACK_BLOCKS 2
50	#define COPYBACK_SIZE (BLOCK_SIZE * COPYBACK_BLOCKS)
51	#define LOOKAHEAD_BLOCKS 1
52	#define LOOKAHEAD_SIZE (BLOCK_SIZE * LOOKAHEAD_BLOCKS)
53	#define SEGMENT_BLOCKS 12 // WARNING: TagMatcher.hpp causes xmlconf test suite failures for SEGMENT_BLOCKS < 3.
54	#define SEGMENT_SIZE (BLOCK_SIZE * SEGMENT_BLOCKS)
55	#define BUFFER_SIZE (COPYBACK_SIZE + SEGMENT_SIZE + LOOKAHEAD_SIZE + PADDING_SIZE)
56
57	//////////////////////////////////////////////////////////////////////////////////////////
58	// @ global depends on 'error_tracker' and 'EOF_mask' definitions.
59	//////////////////////////////////////////////////////////////////////////////////////////
60	@global
61
62	#ifdef SYMBOL_TABLE
63	@marker_strms_global // glue
64	@hash_strms_global
65	@group_strms_global
66	#endif
67
68
69	#ifdef SYMBOL_TABLE
70
71	#include "../lib/symbol_table/src/symbol_table.hpp"
72	#include "../lib/symbol_table/src/gid.hpp"
73
74	#endif
75
76	//////////////////////////////////////////////////////////////////////////////////////////
77	// Headers that depend @ global stream struct types.
78	//////////////////////////////////////////////////////////////////////////////////////////
79	#include "../lib/transpose.hpp"
80	#include <post_process.hpp>
81
82	static void do_process(FILE infile, FILE outfile);
83
84	int main(int argc, char * argv[]) {
85
86	char * infilename, * outfilename;
87	FILE infile, outfile;
88	struct stat fileinfo;
89
90	if (argc < 2) {
91	printf("Usage: %s <filename> [<outputfile>]\n", argv[0]);
92	exit(-1);
93	}
94
95	infilename = argv[1];
96	stat(infilename, &fileinfo);
97	infile = fopen(infilename, "rb");
98	if (!infile) {
99	fprintf(stderr, "Error: cannot open %s for input.\n", infilename);
100	exit(-1);
101	}
102
103	if (argc < 3) outfile = stdout;
104	else {
105	outfilename = argv[2];
106	outfile = fopen(outfilename, "wb");
107	if (!outfile) {
108	fprintf(stderr, "Error: cannot open %s for writing.\n", outfilename);
109	exit(-1);
110	}
111	}
112
113	PERF_SEC_BIND(1);
114
115	PERF_SEC_INIT(parser_timer);
116
117	do_process(infile, outfile);
118
119	PERF_SEC_DUMP(parser_timer);
120
121	PERF_SEC_DESTROY(parser_timer);
122
123	fclose(infile);
124	fclose(outfile);
125
126	return(0);
127	}
128
129	void do_process(FILE infile, FILE outfile) {
130	@decl
131
132	#ifdef SYMBOL_TABLE // @ decl
133	@marker_strms_decl // glue xmlwf / symbol table
134	@hash_strms_decl
135	@group_strms_decl
136	#endif
137
138	LineColTracker tracker;
139	TagMatcher<SEGMENT_SIZE,LOOKAHEAD_SIZE> matcher;
140
141	int block_base=0;
142	int block_pos = 0;
143	int buffer_base=0;
144	int buffer_pos = 0;
145	int chars_avail = 0;
146	int chars_read = 0;
147
148	//////////////////////////////////////////////////////////////////////////////////////////
149	// Buffer Management
150	//////////////////////////////////////////////////////////////////////////////////////////
151	// BitBlock buf[(BUFFER_SIZE)/sizeof(BitBlock)];
152	// uint8_t * src_buffer = (uint8_t *)buf + COPYBACK_SIZE;
153
154	#include "../lib/buffer.hpp"
155	uint8_t * COPYBACK;
156	uint8_t * src_buffer;
157	ALLOC_STATIC_ALIGNED_BYTE_BUFFER_WITH_COPYBACK(COPYBACK, src_buffer);
158
159	#ifdef SYMBOL_TABLE
160	///////////////////////////////////////////////////////////////////////////
161	// Parallel Data Streams with optional CopyBack
162	///////////////////////////////////////////////////////////////////////////
163
164	// hash 0
165	BitBlock * COPYBACK_h0;
166	BitBlock * h0;
167	ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_h0, h0);
168
169	// hash 1
170	BitBlock * COPYBACK_h1;
171	BitBlock * h1;
172	ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_h1, h1);
173
174	// starts
175	BitBlock * COPYBACK_starts;
176	BitBlock * starts;
177	ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_starts, starts);
178
179	// follows_0 - Arbitrary length symbols
180	BitBlock * COPYBACK_follows_0;
181	BitBlock * follows_0;
182	ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_follows_0, follows_0);
183
184	///////////////////////////////////////////////////////////////////////////
185	// Parallel Data Streams - No CopyBack
186	///////////////////////////////////////////////////////////////////////////
187
188	// Basis_bits basis_bits_segment[SEGMENT_BLOCKS];
189	// Markers markers_segment[SEGMENT_BLOCKS];
190	// Hash hash[SEGMENT_BLOCKS];
191	Groups groups_segment[SEGMENT_BLOCKS];
192
193	///////////////////////////////////////////////////////////////////////////
194	// Symbol Table
195	///////////////////////////////////////////////////////////////////////////
196	gid<SEGMENT_SIZE> gids;
197	symbol_table<gid<SEGMENT_SIZE>, fast_pool_allocator<1024> > st;
198	#endif
199
200	//////////////////////////////////////////////////////////////////////////////////////////
201	// XML Validation / Content Model
202	//////////////////////////////////////////////////////////////////////////////////////////
203	chars_read = fread((void *)src_buffer, 1, SEGMENT_SIZE, infile);
204	chars_avail = chars_read;
205	if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
206
207	if(chars_read<4){
208	fprintf(stderr,"File is too short. Not well formed.\n");
209	exit(-1);
210	}
211
212	Entity_Info * e = new Entity_Info;
213	e->AnalyzeSignature((unsigned char *)src_buffer);
214
215	if (e->code_unit_base == ASCII) {
216
217	XML_Decl_Parser<ASCII> decl_parser((unsigned char *)src_buffer);
218
219	decl_parser.ReadXMLInfo(*e);
220
221	if (e->code_unit_size != SingleByte \|\| (e->has_encoding_decl && (!at_UTF_8(e->encoding)))){
222	fprintf(stderr,"Sorry, this xmlwf demo only works for UTF-8.\n");
223	exit(-1);
224	}
225	}
226	else {
227	fprintf(stderr,"Sorry, this xmlwf demo does not process EBCDIC.\n");
228	exit(-1);
229	}
230
231	if (e->content_start != 0) {
232
233	memmove(&src_buffer[0], &src_buffer[e->content_start], chars_avail - e->content_start);
234	buffer_pos = e->content_start;
235	if ((chars_avail-e->content_start) < SEGMENT_SIZE) {
236	chars_read = chars_avail - e->content_start + fread(&src_buffer[chars_avail-e->content_start], 1, e->content_start, infile);
237	chars_avail = chars_read;
238	}
239	if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
240	}
241
242	//////////////////////////////////////////////////////////////////////////////////////////
243	// Read OVERLAP bytes to support post processing validation lookahead.
244	//////////////////////////////////////////////////////////////////////////////////////////
245	chars_read = chars_avail + fread(&src_buffer[chars_avail], 1, LOOKAHEAD_SIZE, infile);
246	chars_avail = chars_read;
247	if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
248
249	@stream_stmts
250
251	#ifdef SYMBOL_TABLE // @ stream_stmts
252	@marker_strms_stream_stmts // glue xmlwf / symbol table
253	@hash_strms_stream_stmts
254	@group_strms_stream_stmts
255	#endif
256
257	//////////////////////////////////////////////////////////////////////////////////////////
258	// Full Segments
259	//////////////////////////////////////////////////////////////////////////////////////////
260	matcher.setSrc((char *)src_buffer);
261	while (chars_avail >= SEGMENT_SIZE) {
262	PERF_SEC_START(parser_timer);
263	for (int blk = 0; blk < SEGMENT_BLOCKS; blk++) {
264	block_base = blk*BLOCK_SIZE;
265	s2p_do_block((BytePack *) &src_buffer[block_base], basis_bits);
266
267	@block_stmts
268
269	#ifdef SYMBOL_TABLE
270	@marker_strms_block_stmts // glue
271	@hash_strms_block_stmts
272	@group_strms_block_stmts
273
274	// for(int blk=0;blk<SEGMENT_BLOCKS;blk++) { // write contiguous bit streams
275	h0[blk] = hash.h0;
276	h1[blk] = hash.h1;
277	starts[blk] = groups.starts;
278	follows_0[blk] = groups.follows_0;
279
280	groups_segment[blk] = groups;
281	//}
282	#endif
283
284	tracker.StoreNewlines(lex.LF);
285	postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base, chars_avail, tracker);
286	matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
287	tracker.AdvanceBlock();
288
289	}
290	#ifdef SYMBOL_TABLE
291	st.resolve(src_buffer, groups_segment, starts, follows_0, h0, h1, SEGMENT_BLOCKS, gids);
292
293	#ifdef DUMP_SYMBOLS
294	uint32_t blk_offset;
295	for(int blk=0;blk<SEGMENT_BLOCKS;blk++) {
296	blk_offset = blk * BLOCK_SIZE;
297	gid_type gid;
298
299	#ifdef INDEX_AT_STARTS
300	ForwardScanner<BitBlock, scanword_t> scanner(&(groups_segment[blk].starts));
301	#else
302	ForwardScanner<BitBlock, scanword_t> scanner(&(groups_segment[blk].follows));
303	#endif
304
305	scanner.scan_to_next();
306	while(!scanner.is_done()) {
307	gid = gids.at[scanner.get_pos() + blk_offset];
308	cout << string((char *)st.get_raw_data(gid), st.get_lgth(gid)) << endl;
309	scanner.scan_to_next();
310	}
311	}
312	#endif
313
314	#endif
315
316	matcher.StreamScan(chars_avail);
317	matcher.Advance_buffer();
318	PERF_SEC_END(parser_timer, chars_avail);
319
320	memmove(src_buffer, &src_buffer[SEGMENT_SIZE], LOOKAHEAD_SIZE);
321	#ifdef SYMBOL_TABLE
322	COPY_BACK_BYTE_BUFFER(COPYBACK, src_buffer);
323	COPY_BACK_BITBLOCK_BUFFER(COPYBACK_h0,h0);
324	COPY_BACK_BITBLOCK_BUFFER(COPYBACK_h1,h1);
325	COPY_BACK_BITBLOCK_BUFFER(COPYBACK_starts,starts);
326	COPY_BACK_BITBLOCK_BUFFER(COPYBACK_follows_0,follows_0);
327	#endif
328
329	chars_read = fread(&src_buffer[LOOKAHEAD_SIZE], 1, SEGMENT_SIZE, infile) + LOOKAHEAD_SIZE;
330	chars_avail = chars_read;
331	if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
332	buffer_pos += chars_avail;
333	buffer_base = buffer_pos;
334	}
335
336	//////////////////////////////////////////////////////////////////////////////////////////
337	// Final Partial Segment
338	//////////////////////////////////////////////////////////////////////////////////////////
339	PERF_SEC_START(parser_timer);
340
341	block_pos = 0;
342	int remaining = chars_avail;
343
344	/* Full Blocks */
345	uint32_t blk = 0;
346	while (remaining >= BLOCK_SIZE) {
347	block_base = block_pos;
348	s2p_do_block((BytePack *) &src_buffer[block_pos], basis_bits);
349	@block_stmts
350
351	#ifdef SYMBOL_TABLE
352	@marker_strms_block_stmts // glue
353	@hash_strms_block_stmts
354	@group_strms_block_stmts
355
356
357	h0[blk] = hash.h0;
358	h1[blk] = hash.h1;
359	starts[blk] = groups.starts;
360	follows_0[blk] = groups.follows_0;
361
362	groups_segment[blk].follows = groups.follows;
363
364	#endif
365
366	tracker.StoreNewlines(lex.LF);
367	postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base, chars_avail, tracker);
368	matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
369	tracker.AdvanceBlock();
370	block_pos += BLOCK_SIZE;
371	remaining -= BLOCK_SIZE;
372	blk++;
373	}
374	block_base = block_pos;
375
376	/* Partial Block or Any Carry */
377	if (remaining > 0 \|\| @any_carry
378	#ifdef SYMBOL_TABLE
379	\|\| @marker_strms_any_carry \|\| @group_strms_any_carry \|\| @hash_strms_any_carry
380	#endif
381	) {
382	EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining));
383	s2p_do_final_block((BytePack *) &src_buffer[block_pos], basis_bits, EOF_mask);
384	@final_block_stmts
385
386	#ifdef SYMBOL_TABLE
387	@marker_strms_final_block_stmts // glue
388	@hash_strms_final_block_stmts
389	@group_strms_final_block_stmts
390
391	h0[blk] = hash.h0;
392	h1[blk] = hash.h1;
393	starts[blk] = groups.starts;
394	follows_0[blk] = groups.follows_0;
395
396	groups_segment[blk] = groups;
397	#endif
398
399	tracker.StoreNewlines(lex.LF);
400	postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base, chars_avail, tracker);
401	matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
402	tracker.AdvanceBlock();
403	blk++;
404	}
405
406	buffer_pos += chars_avail;
407	buffer_base = buffer_pos;
408
409	matcher.StreamScan(chars_avail);
410	matcher.Advance_buffer();
411
412	#ifdef SYMBOL_TABLE
413	uint32_t segment_blocks = blk;
414	st.resolve(src_buffer, groups_segment, starts, follows_0, h0, h1, segment_blocks, gids);
415
416	#ifdef DUMP_SYMBOLS
417	uint32_t blk_offset;
418	for(int blk=0;blk<segment_blocks;blk++) {
419	blk_offset = blk * BLOCK_SIZE;
420	gid_type gid;
421
422	#ifdef INDEX_AT_STARTS
423	ForwardScanner<BitBlock, scanword_t> scanner(&(groups_segment[blk].starts));
424	#else
425	ForwardScanner<BitBlock, scanword_t> scanner(&(groups_segment[blk].follows));
426	#endif
427
428	scanner.scan_to_next();
429	while(!scanner.is_done()) {
430	gid = gids.at[scanner.get_pos() + blk_offset];
431	cout << string((char *)st.get_raw_data(gid), st.get_lgth(gid)) << endl;
432	scanner.scan_to_next();
433	}
434	}
435
436	#endif
437
438	#endif
439
440	if (matcher.depth != 0) {
441	fprintf(stderr, "tag matching error (depth %i) at position %i\n", matcher.depth, buffer_base);
442	exit(-1);
443	}
444	PERF_SEC_END(parser_timer, chars_avail);
445	}
446

Note: See TracBrowser for help on using the browser.

Context Navigation

root/proto/parabix2/pablo_template_symbol_table.cpp @ 2202

Download in other formats: