Context Navigation

pablo_template_symbol_table.cpp @ 2211

Revision 2211, 13.5 KB (checked in by ksherdy, 13 months ago)
Updated carry array. Added assert_0_error macro to symbol table template.

Line
1	#include <stdio.h>
2	#include <stdlib.h>
3	#include <errno.h>
4	#include <sys/types.h>
5	#include <sys/stat.h>
6
7	#define LocalCarryDeclare(name, count)\
8	CarryArray<count, 0> name;\
9
10	#include "../lib/bitblock.hpp"
11	#include "../lib/carryQ.hpp"
12	#include "../lib/bitstream_iterator.hpp"
13	#include "../lib/bitblock_iterator.hpp"
14	#include "../lib/s2p.hpp"
15	#include "../lib/perflib/perfsec.h"
16
17	#include <xmldecl.h>
18	#include <namechars.h>
19	#include <TagMatcher.hpp>
20	#include <LineColTracker.hpp>
21	#include <ErrorUtil.h>
22	#include <ErrorTracker.h>
23	#include <XMLTestSuiteError.h>
24
25	// Define the mappings for pablo.assert_0(strm, errkind) statements which
26	// compile to the the form assert_0_error(errkind, strm)
27	#define assert_0_error(errkind, errstrm) error_tracker.NoteError(errkind, errstrm);
28
29	#define SYMBOL_TABLE
30
31	#ifdef BUFFER_PROFILING
32	BOM_Table * parser_timer;
33	#elif CODE_CLOCKER
34	//#define NUM_EVENTS 1
35	//int Events[NUM_EVENTS] = {PAPI_TOT_CYC};
36	//int Events[NUM_EVENTS] = {PAPI_L2_DCM};
37	#define NUM_EVENTS 2
38	int Events[NUM_EVENTS] = {PAPI_TOT_CYC, PAPI_BR_MSP};
39	int cal_size = 20;
40	CC * parser_timer = new CC(Events,NUM_EVENTS,cal_size);
41	#else
42	void * parser_timer;
43	#endif
44
45	ErrorTracker error_tracker;
46	BitBlock EOF_mask = simd<1>::constant<1>();
47
48	//////////////////////////////////////////////////////////////////////////////////////////
49	// Buffer Management // WARNING: Do Not update #defines. Results in TagMatcher errors.
50	//////////////////////////////////////////////////////////////////////////////////////////
51	#define PADDING_BLOCKS 0
52	#define PADDING_SIZE (BLOCK_SIZE * PADDING_BLOCKS)
53	#define COPYBACK_BLOCKS 2
54	#define COPYBACK_SIZE (BLOCK_SIZE * COPYBACK_BLOCKS)
55	#define LOOKAHEAD_BLOCKS 1
56	#define LOOKAHEAD_SIZE (BLOCK_SIZE * LOOKAHEAD_BLOCKS)
57	#define SEGMENT_BLOCKS 12 // WARNING: TagMatcher.hpp causes xmlconf test suite failures for SEGMENT_BLOCKS < 3.
58	#define SEGMENT_SIZE (BLOCK_SIZE * SEGMENT_BLOCKS)
59	#define BUFFER_SIZE (COPYBACK_SIZE + SEGMENT_SIZE + LOOKAHEAD_SIZE + PADDING_SIZE)
60
61	//////////////////////////////////////////////////////////////////////////////////////////
62	// @ global depends on 'error_tracker' and 'EOF_mask' definitions.
63	//////////////////////////////////////////////////////////////////////////////////////////
64	@global
65
66	#ifdef SYMBOL_TABLE
67	@marker_strms_global // glue
68	@hash_strms_global
69	@group_strms_global
70	#endif
71
72
73	#ifdef SYMBOL_TABLE
74
75	#include "../lib/symbol_table/src/symbol_table.hpp"
76	#include "../lib/symbol_table/src/gid.hpp"
77
78	#endif
79
80	//////////////////////////////////////////////////////////////////////////////////////////
81	// Headers that depend @ global stream struct types.
82	//////////////////////////////////////////////////////////////////////////////////////////
83	#include "../lib/transpose.hpp"
84	#include <post_process.hpp>
85
86	static void do_process(FILE infile, FILE outfile);
87
88	int main(int argc, char * argv[]) {
89
90	char * infilename, * outfilename;
91	FILE infile, outfile;
92	struct stat fileinfo;
93
94	if (argc < 2) {
95	printf("Usage: %s <filename> [<outputfile>]\n", argv[0]);
96	exit(-1);
97	}
98
99	infilename = argv[1];
100	stat(infilename, &fileinfo);
101	infile = fopen(infilename, "rb");
102	if (!infile) {
103	fprintf(stderr, "Error: cannot open %s for input.\n", infilename);
104	exit(-1);
105	}
106
107	if (argc < 3) outfile = stdout;
108	else {
109	outfilename = argv[2];
110	outfile = fopen(outfilename, "wb");
111	if (!outfile) {
112	fprintf(stderr, "Error: cannot open %s for writing.\n", outfilename);
113	exit(-1);
114	}
115	}
116
117	PERF_SEC_BIND(1);
118
119	PERF_SEC_INIT(parser_timer);
120
121	do_process(infile, outfile);
122
123	PERF_SEC_DUMP(parser_timer);
124
125	PERF_SEC_DESTROY(parser_timer);
126
127	fclose(infile);
128	fclose(outfile);
129
130	return(0);
131	}
132
133	void do_process(FILE infile, FILE outfile) {
134	@decl
135
136	#ifdef SYMBOL_TABLE // @ decl
137	@marker_strms_decl // glue xmlwf / symbol table
138	@hash_strms_decl
139	@group_strms_decl
140	#endif
141
142	LineColTracker tracker;
143	TagMatcher<SEGMENT_SIZE,LOOKAHEAD_SIZE> matcher;
144
145	int block_base=0;
146	int block_pos = 0;
147	int buffer_base=0;
148	int buffer_pos = 0;
149	int chars_avail = 0;
150	int chars_read = 0;
151
152	//////////////////////////////////////////////////////////////////////////////////////////
153	// Buffer Management
154	//////////////////////////////////////////////////////////////////////////////////////////
155	// BitBlock buf[(BUFFER_SIZE)/sizeof(BitBlock)];
156	// uint8_t * src_buffer = (uint8_t *)buf + COPYBACK_SIZE;
157
158	#include "../lib/buffer.hpp"
159	uint8_t * COPYBACK;
160	uint8_t * src_buffer;
161	ALLOC_STATIC_ALIGNED_BYTE_BUFFER_WITH_COPYBACK(COPYBACK, src_buffer);
162
163	#ifdef SYMBOL_TABLE
164	///////////////////////////////////////////////////////////////////////////
165	// Parallel Data Streams with optional CopyBack
166	///////////////////////////////////////////////////////////////////////////
167
168	// hash 0
169	BitBlock * COPYBACK_h0;
170	BitBlock * h0;
171	ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_h0, h0);
172
173	// hash 1
174	BitBlock * COPYBACK_h1;
175	BitBlock * h1;
176	ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_h1, h1);
177
178	// starts
179	BitBlock * COPYBACK_starts;
180	BitBlock * starts;
181	ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_starts, starts);
182
183	// follows_0 - Arbitrary length symbols
184	BitBlock * COPYBACK_follows_0;
185	BitBlock * follows_0;
186	ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_follows_0, follows_0);
187
188	///////////////////////////////////////////////////////////////////////////
189	// Parallel Data Streams - No CopyBack
190	///////////////////////////////////////////////////////////////////////////
191
192	// Basis_bits basis_bits_segment[SEGMENT_BLOCKS];
193	// Markers markers_segment[SEGMENT_BLOCKS];
194	// Hash hash[SEGMENT_BLOCKS];
195	Groups groups_segment[SEGMENT_BLOCKS];
196
197	///////////////////////////////////////////////////////////////////////////
198	// Symbol Table
199	///////////////////////////////////////////////////////////////////////////
200	gid<SEGMENT_SIZE> gids;
201	symbol_table<gid<SEGMENT_SIZE>, fast_pool_allocator<1024> > st;
202	#endif
203
204	//////////////////////////////////////////////////////////////////////////////////////////
205	// XML Validation / Content Model
206	//////////////////////////////////////////////////////////////////////////////////////////
207	chars_read = fread((void *)src_buffer, 1, SEGMENT_SIZE, infile);
208	chars_avail = chars_read;
209	if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
210
211	if(chars_read<4){
212	fprintf(stderr,"File is too short. Not well formed.\n");
213	exit(-1);
214	}
215
216	Entity_Info * e = new Entity_Info;
217	e->AnalyzeSignature((unsigned char *)src_buffer);
218
219	if (e->code_unit_base == ASCII) {
220
221	XML_Decl_Parser<ASCII> decl_parser((unsigned char *)src_buffer);
222
223	decl_parser.ReadXMLInfo(*e);
224
225	if (e->code_unit_size != SingleByte \|\| (e->has_encoding_decl && (!at_UTF_8(e->encoding)))){
226	fprintf(stderr,"Sorry, this xmlwf demo only works for UTF-8.\n");
227	exit(-1);
228	}
229	}
230	else {
231	fprintf(stderr,"Sorry, this xmlwf demo does not process EBCDIC.\n");
232	exit(-1);
233	}
234
235	if (e->content_start != 0) {
236
237	memmove(&src_buffer[0], &src_buffer[e->content_start], chars_avail - e->content_start);
238	buffer_pos = e->content_start;
239	if ((chars_avail-e->content_start) < SEGMENT_SIZE) {
240	chars_read = chars_avail - e->content_start + fread(&src_buffer[chars_avail-e->content_start], 1, e->content_start, infile);
241	chars_avail = chars_read;
242	}
243	if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
244	}
245
246	//////////////////////////////////////////////////////////////////////////////////////////
247	// Read OVERLAP bytes to support post processing validation lookahead.
248	//////////////////////////////////////////////////////////////////////////////////////////
249	chars_read = chars_avail + fread(&src_buffer[chars_avail], 1, LOOKAHEAD_SIZE, infile);
250	chars_avail = chars_read;
251	if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
252
253	@stream_stmts
254
255	#ifdef SYMBOL_TABLE // @ stream_stmts
256	@marker_strms_stream_stmts // glue xmlwf / symbol table
257	@hash_strms_stream_stmts
258	@group_strms_stream_stmts
259	#endif
260
261	//////////////////////////////////////////////////////////////////////////////////////////
262	// Full Segments
263	//////////////////////////////////////////////////////////////////////////////////////////
264	matcher.setSrc((char *)src_buffer);
265	while (chars_avail >= SEGMENT_SIZE) {
266	PERF_SEC_START(parser_timer);
267	for (int blk = 0; blk < SEGMENT_BLOCKS; blk++) {
268	block_base = blk*BLOCK_SIZE;
269	s2p_do_block((BytePack *) &src_buffer[block_base], basis_bits);
270
271	@block_stmts
272
273	#ifdef SYMBOL_TABLE
274	@marker_strms_block_stmts // glue
275	@hash_strms_block_stmts
276	@group_strms_block_stmts
277
278	// for(int blk=0;blk<SEGMENT_BLOCKS;blk++) { // write contiguous bit streams
279	h0[blk] = hash.h0;
280	h1[blk] = hash.h1;
281	starts[blk] = groups.starts;
282	follows_0[blk] = groups.follows_0;
283
284	groups_segment[blk] = groups;
285	//}
286	#endif
287
288	tracker.StoreNewlines(lex.LF);
289	postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base, chars_avail, tracker);
290	matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
291	tracker.AdvanceBlock();
292
293	}
294	#ifdef SYMBOL_TABLE
295	st.resolve(src_buffer, groups_segment, starts, follows_0, h0, h1, SEGMENT_BLOCKS, gids);
296
297	#ifdef DUMP_SYMBOLS
298	uint32_t blk_offset;
299	for(int blk=0;blk<SEGMENT_BLOCKS;blk++) {
300	blk_offset = blk * BLOCK_SIZE;
301	gid_type gid;
302
303	#ifdef INDEX_AT_STARTS
304	ForwardScanner<BitBlock, scanword_t> scanner(&(groups_segment[blk].starts));
305	#else
306	ForwardScanner<BitBlock, scanword_t> scanner(&(groups_segment[blk].follows));
307	#endif
308
309	scanner.scan_to_next();
310	while(!scanner.is_done()) {
311	gid = gids.at[scanner.get_pos() + blk_offset];
312	cout << string((char *)st.get_raw_data(gid), st.get_lgth(gid)) << endl;
313	scanner.scan_to_next();
314	}
315	}
316	#endif
317
318	#endif
319
320	matcher.StreamScan(chars_avail);
321	matcher.Advance_buffer();
322	PERF_SEC_END(parser_timer, chars_avail);
323
324	memmove(src_buffer, &src_buffer[SEGMENT_SIZE], LOOKAHEAD_SIZE);
325	#ifdef SYMBOL_TABLE
326	COPY_BACK_BYTE_BUFFER(COPYBACK, src_buffer);
327	COPY_BACK_BITBLOCK_BUFFER(COPYBACK_h0,h0);
328	COPY_BACK_BITBLOCK_BUFFER(COPYBACK_h1,h1);
329	COPY_BACK_BITBLOCK_BUFFER(COPYBACK_starts,starts);
330	COPY_BACK_BITBLOCK_BUFFER(COPYBACK_follows_0,follows_0);
331	#endif
332
333	chars_read = fread(&src_buffer[LOOKAHEAD_SIZE], 1, SEGMENT_SIZE, infile) + LOOKAHEAD_SIZE;
334	chars_avail = chars_read;
335	if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
336	buffer_pos += chars_avail;
337	buffer_base = buffer_pos;
338	}
339
340	//////////////////////////////////////////////////////////////////////////////////////////
341	// Final Partial Segment
342	//////////////////////////////////////////////////////////////////////////////////////////
343	PERF_SEC_START(parser_timer);
344
345	block_pos = 0;
346	int remaining = chars_avail;
347
348	/* Full Blocks */
349	uint32_t blk = 0;
350	while (remaining >= BLOCK_SIZE) {
351	block_base = block_pos;
352	s2p_do_block((BytePack *) &src_buffer[block_pos], basis_bits);
353	@block_stmts
354
355	#ifdef SYMBOL_TABLE
356	@marker_strms_block_stmts // glue
357	@hash_strms_block_stmts
358	@group_strms_block_stmts
359
360
361	h0[blk] = hash.h0;
362	h1[blk] = hash.h1;
363	starts[blk] = groups.starts;
364	follows_0[blk] = groups.follows_0;
365
366	groups_segment[blk].follows = groups.follows;
367
368	#endif
369
370	tracker.StoreNewlines(lex.LF);
371	postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base, chars_avail, tracker);
372	matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
373	tracker.AdvanceBlock();
374	block_pos += BLOCK_SIZE;
375	remaining -= BLOCK_SIZE;
376	blk++;
377	}
378	block_base = block_pos;
379
380	/* Partial Block or Any Carry */
381	if (remaining > 0 \|\| @any_carry
382	#ifdef SYMBOL_TABLE
383	\|\| @marker_strms_any_carry \|\| @group_strms_any_carry \|\| @hash_strms_any_carry
384	#endif
385	) {
386	EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining));
387	s2p_do_final_block((BytePack *) &src_buffer[block_pos], basis_bits, EOF_mask);
388	@final_block_stmts
389
390	#ifdef SYMBOL_TABLE
391	@marker_strms_final_block_stmts // glue
392	@hash_strms_final_block_stmts
393	@group_strms_final_block_stmts
394
395	h0[blk] = hash.h0;
396	h1[blk] = hash.h1;
397	starts[blk] = groups.starts;
398	follows_0[blk] = groups.follows_0;
399
400	groups_segment[blk] = groups;
401	#endif
402
403	tracker.StoreNewlines(lex.LF);
404	postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base, chars_avail, tracker);
405	matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
406	tracker.AdvanceBlock();
407	blk++;
408	}
409
410	buffer_pos += chars_avail;
411	buffer_base = buffer_pos;
412
413	matcher.StreamScan(chars_avail);
414	matcher.Advance_buffer();
415
416	#ifdef SYMBOL_TABLE
417	uint32_t segment_blocks = blk;
418	st.resolve(src_buffer, groups_segment, starts, follows_0, h0, h1, segment_blocks, gids);
419
420	#ifdef DUMP_SYMBOLS
421	uint32_t blk_offset;
422	for(int blk=0;blk<segment_blocks;blk++) {
423	blk_offset = blk * BLOCK_SIZE;
424	gid_type gid;
425
426	#ifdef INDEX_AT_STARTS
427	ForwardScanner<BitBlock, scanword_t> scanner(&(groups_segment[blk].starts));
428	#else
429	ForwardScanner<BitBlock, scanword_t> scanner(&(groups_segment[blk].follows));
430	#endif
431
432	scanner.scan_to_next();
433	while(!scanner.is_done()) {
434	gid = gids.at[scanner.get_pos() + blk_offset];
435	cout << string((char *)st.get_raw_data(gid), st.get_lgth(gid)) << endl;
436	scanner.scan_to_next();
437	}
438	}
439
440	#endif
441
442	#endif
443
444	if (matcher.depth != 0) {
445	fprintf(stderr, "tag matching error (depth %i) at position %i\n", matcher.depth, buffer_base);
446	exit(-1);
447	}
448	PERF_SEC_END(parser_timer, chars_avail);
449	}
450

Note: See TracBrowser for help on using the browser.

Context Navigation

root/proto/parabix2/pablo_template_symbol_table.cpp @ 2211

Download in other formats: