source: proto/lz4d/lz4d_e_d.py @ 5539

Last change on this file since 5539 was 5539, checked in by xwa163, 22 months ago

lz4d_prototype: add prototype for sequential_pablo_kernel, modify lz4d prototype to use sequential_pablo_kernel

File size: 4.1 KB
Line 
1import sys
2from Basis_Bits import Basis_bits
3import sequential_pablo
4import pablo
5from deposit_compressed_kernel import DepositCompressedKernel
6from extract_e1_m0_kernel import ExtractE1M0Kernel
7from bitwise_op_helper import *
8
9# Kernels
10def extract_blocks(file_content):
11    offset = 0
12    block_data = []
13    uncompressed_mask = 0
14
15    offset += 4  #Magicc Number
16
17    flag = ord(file_content[offset])
18
19    has_content_checksum = bit_test(flag, 2)
20    has_content_size = bit_test(flag, 3)
21    has_block_checksum = bit_test(flag, 4)
22
23    frame_descriptor_size = 3
24    if has_content_size:
25        frame_descriptor_size = 11
26    offset += frame_descriptor_size
27
28    # Process Data Block
29    while True:
30        blockSize = 0
31        for i in range(4):
32            blockSize += (ord(file_content[offset + i]) << (i * 8))
33            pass
34
35        if blockSize == 0:
36            # End Mark
37            break
38
39        highestIndex = 4 * 8 - 1
40        isCompressed = not bit_test(blockSize, highestIndex)
41        realBlockSize = bit_clear(blockSize, highestIndex)
42
43        offset = offset + 4
44
45        if realBlockSize == 0: #End Mark
46            break
47
48        block_start = offset
49
50        offset = offset + realBlockSize
51
52        block_end = offset
53
54        block_data.append((isCompressed, block_start, block_end))
55       
56        if not isCompressed: uncompressed_mask |= (1 << block_end) - (1 << block_start)
57
58        if has_block_checksum:
59            offset = offset + 4
60
61    # Ignore Content Check Sum
62    return block_data, uncompressed_mask
63
64def deposit_uncompressed(file_content, uncompressed_block_data, outputStrBuffer):
65    for (uncompressed_start_pos, block_length, output_pos) in uncompressed_block_data:
66        uncompressed_end_pos = uncompressed_start_pos + block_length
67        outputStrBuffer[output_pos: output_pos + block_length] = list(
68            file_content[uncompressed_start_pos: uncompressed_end_pos]
69        )
70
71'''
72Main
73'''
74
75if __name__ == '__main__':
76    if len(sys.argv) < 3:
77        print("""
78Usage:
79python lz4d_e_d.py [input_file_path] [output_file_path]
80        """)
81    else:
82        # Prepare Input
83        inputFile = sys.argv[1]
84        outputFile = sys.argv[2]
85        with open(inputFile, "rb") as f:
86            file_content = f.read()
87        basis_bits = Basis_bits()
88        sequential_pablo.transpose_streams(file_content, basis_bits)
89
90
91        # Extract start and end marker of compressed and uncompressed blocks
92        block_data, uncompressed_mask = extract_blocks(file_content)
93        #print block_data
94
95        # Imitate ParabixCharacterClassKernelBuilder to produce extenders stream
96        extender_bits = basis_bits.bit_0 & basis_bits.bit_1 & basis_bits.bit_2 & basis_bits.bit_3 & basis_bits.bit_4 & basis_bits.bit_5 & basis_bits.bit_6 & basis_bits.bit_7
97
98        # Extract E1  and M0 marker streams as wells as data for uncompressed blocks and total length
99        extract_e1_m0_k = ExtractE1M0Kernel({
100            'file_content' : file_content,
101            'block_data' : block_data,
102            'extender' : extender_bits
103        })
104        E1_marker, M0_marker, uncompressed_data, output_length = extract_e1_m0_k.process()
105
106        # Use Array of char to imitate output string buffer of fixed length
107        outputStrBuffer = ["" for i in range(output_length)]
108
109        # D_marker Will Be Calculated by a pablo kernel
110        d_marker = ~(M0_marker | pablo.Advance(M0_marker))
111
112        # Deposit compressed and uncompressed blocks
113        # deposit_compressed(file_content, block_data, E1_marker, M0_marker, outputStrBuffer)
114        deposit_compressed_k = DepositCompressedKernel({
115            'file_content' : file_content,
116            'block_data' : block_data,
117            'e1_marker': E1_marker,
118            'm0_marker': M0_marker,
119            'd_marker' : d_marker
120        }, {
121            'outputStrBuffer': outputStrBuffer
122        })
123        deposit_compressed_k.process()
124        deposit_uncompressed(file_content, uncompressed_data, outputStrBuffer)
125
126        outputStr = ''.join(outputStrBuffer)
127        # print(outputStr)
128        with open(outputFile, 'w') as f:
129            f.write(outputStr)
130
Note: See TracBrowser for help on using the repository browser.