source: proto/SymbolTable/dictionary_pbgs_div.py @ 1666

Last change on this file since 1666 was 1666, checked in by vla24, 8 years ago

SymbolTable?: Added natural language dictionary templates that uses symbol table

File size: 3.6 KB
Line 
1# -*- coding: utf-8 -*-
2#
3# parabix2_compilable.py
4#
5# Parallel XML Parsing with Bitstream Addition
6#
7# - Complete prototype for all bitstream computations in Parabix2
8# - Optimized for compilation
9# - Separate compilation
10
11# Robert D. Cameron
12# July 29, 2010
13#
14
15#import bitutil
16
17class Basis_bits():
18    bit_0 = 0
19    bit_1 = 0
20    bit_2 = 0
21    bit_3 = 0
22    bit_4 = 0
23    bit_5 = 0
24    bit_6 = 0
25    bit_7 = 0
26
27class Dictionary ():
28    Words = 0
29    Word_starts = 0
30    Word_ends = 0
31    Word_ends_1_to_2 = 0
32    Word_ends_3_to_4 = 0
33    Word_ends_5_to_6 = 0
34    Word_ends_7_to_8 = 0
35    Word_ends_9_to_10 = 0
36    Word_ends_11_to_12 = 0
37    Word_ends_13_to_14 = 0
38    Word_ends_15_to_16 = 0
39    Word_remaining_ends = 0
40
41class Hash_data():
42    Hash_value = 0
43
44def Classify_bytes(basis_bits, dictionary):
45    temp1 = (basis_bits.bit_6 & basis_bits.bit_7)
46    temp2 = (basis_bits.bit_5 | temp1)
47    temp3 = (basis_bits.bit_4 & temp2)
48    temp4 = (~temp3)
49    temp5 = (basis_bits.bit_4 | basis_bits.bit_5)
50    temp6 = (basis_bits.bit_6 | basis_bits.bit_7)
51    temp7 = (temp5 | temp6)
52    temp8 = ((basis_bits.bit_3 & temp4)|(~(basis_bits.bit_3) & temp7))
53    temp9 = (basis_bits.bit_1 &~ basis_bits.bit_0)
54    temp10 = (temp8 & temp9)
55    temp11 = (basis_bits.bit_0 | basis_bits.bit_1)
56    temp12 = (basis_bits.bit_2 & basis_bits.bit_3)
57    temp13 = (temp12 &~ temp11)
58    temp14 = (basis_bits.bit_5 | basis_bits.bit_6)
59    temp15 = (basis_bits.bit_4 & temp14)
60    temp16 = (temp13 &~ temp15)
61    dictionary.Words = (temp10 | temp16)
62
63    temp = pablo.Advance(dictionary.Words) ^ dictionary.Words
64    dictionary.Word_starts = temp & dictionary.Words
65    dictionary.Word_ends = temp & ~dictionary.Words
66
67def Form_Length_Group_Bitstreams(dictionary):
68
69    starts = dictionary.Word_starts
70    ends = dictionary.Word_ends
71
72    temp = ends | pablo.Advance(ends)
73
74    # Group symbols of length 1 and 2
75    start_2 = pablo.Advance(pablo.Advance(starts))
76    dictionary.Word_ends_1_to_2 =  start_2 & temp
77
78    # Group symbols of length 3 and 4
79    start_4 = pablo.Advance(pablo.Advance(start_2 & ~temp))
80    dictionary.Word_ends_3_to_4 =  start_4 & temp
81
82    # Group symbols of length 5 and 6
83    start_6 = pablo.Advance(pablo.Advance(start_4 & ~temp))
84    dictionary.Word_ends_5_to_6 =  start_6 & temp
85
86    # Group symbols of length 7 and 8
87    start_8 = pablo.Advance(pablo.Advance(start_6 & ~temp))
88    dictionary.Word_ends_7_to_8 =  start_8 & temp
89
90    # Group symbols of length 9 and 10
91    start_10 = pablo.Advance(pablo.Advance(start_8 & ~temp))
92    dictionary.Word_ends_9_to_10 =  start_10 & temp
93
94    # Group symbols of length 11 and 12
95    start_12 = pablo.Advance(pablo.Advance(start_10 & ~temp))
96    dictionary.Word_ends_11_to_12 =  start_12 & temp
97
98    # Group symbols of length 13 and 14
99    start_14 = pablo.Advance(pablo.Advance(start_12 & ~temp))
100    dictionary.Word_ends_13_to_14 =  start_14 & temp
101
102    # Group symbols of length 15 and 16
103    start_16 = pablo.Advance(pablo.Advance(start_14 & ~temp))
104    dictionary.Word_ends_15_to_16 =  start_16 & temp
105
106    # Group symbols of length 17 and longer
107    dictionary.Word_remaining_ends = start_16 & ~dictionary.Word_ends_15_to_16
108
109def Compute_Hash_Value_Bitstream(hash_data, basis_bits):
110    hash_data.Hash_value = basis_bits.bit_2 ^ basis_bits.bit_4 ^ basis_bits.bit_6
111    #hash_data.Hash_value = basis_bits.bit_3 ^ basis_bits.bit_5 ^ basis_bits.bit_7
112
113def Main(basis_bits, dictionary, hash_data):
114    Classify_bytes(basis_bits, dictionary)
115
116    # These methods are needed to do Paralel Bitstream Based Length Sorting
117    Form_Length_Group_Bitstreams(dictionary)
118    Compute_Hash_Value_Bitstream(hash_data, basis_bits);
119               
Note: See TracBrowser for help on using the repository browser.