Changeset 2098 for trunk


Ignore:
Timestamp:
May 10, 2012, 5:49:33 PM (7 years ago)
Author:
ksherdy
Message:

Adding mixed log2 div2 approach.

Location:
trunk/symbol_table
Files:
1 added
9 edited

Legend:

Unmodified
Added
Removed
  • trunk/symbol_table/Makefile

    r2096 r2098  
    2929DIV2_GROUP_STRMS = div2_group_strms.py
    3030LOGBASE2_GROUP_STRMS = logbase2_group_strms.py
     31DIV2_LOGBASE2_GROUP_STRMS = div2_logbase2_group_strms.py
    3132
    3233GROUP_PREFIX=@group_strms_
     
    4445
    4546logbase2: markers hash logbase2_group src/main.cpp
     47        python $(PABLO_COMPILER) $(MARKER_PABLO_SRC) -t $(MAIN_OUTFILE) -l $(MARKER_PREFIX) -o $(MAIN_OUTFILE) $(PABLO_ADD_DEBUG)
     48
     49div2_logbase2: markers hash div2_logbase2_group src/main.cpp
    4650        python $(PABLO_COMPILER) $(MARKER_PABLO_SRC) -t $(MAIN_OUTFILE) -l $(MARKER_PREFIX) -o $(MAIN_OUTFILE) $(PABLO_ADD_DEBUG)
    4751
     
    6468        python $(PABLO_COMPILER) $(LOGBASE2_GROUP_STRMS) -t $(MAIN_TEMPLATE) -l $(GROUP_PREFIX) -o $(MAIN_OUTFILE) $(PABLO_ADD_DEBUG)
    6569
     70div2_logbase2_group: logbase2_group_strms.py group_strms_template.hpp main_template.cpp
     71        python $(PABLO_COMPILER) $(DIV2_LOGBASE2_GROUP_STRMS) -t $(GROUP_TEMPLATE) -l $(GROUP_PREFIX) -o $(GROUP_OUTFILE) $(PABLO_ADD_DEBUG)
     72        python $(PABLO_COMPILER) $(DIV2_LOGBASE2_GROUP_STRMS) -t $(MAIN_TEMPLATE) -l $(GROUP_PREFIX) -o $(MAIN_OUTFILE) $(PABLO_ADD_DEBUG)
     73
    6674clean:
    6775        rm -f $(MARKER_OUTFILE) $(HASH_OUTFILE) $(GROUP_OUTFILE) $(MAIN_OUTFILE)
  • trunk/symbol_table/demo_strms.py

    r2097 r2098  
    33import div2_group_strms
    44import logbase2_group_strms
     5import div2_logbase2_group_strms
    56import sys
    67
     
    117118
    118119    # Demo_div2(u8data, basis_bits)
    119     Demo_logbase2(u8data, basis_bits)
     120    # Demo_logbase2(u8data, basis_bits)
     121    Demo_div2_logbase2(u8data, basis_bits)
    120122
  • trunk/symbol_table/logbase2_group_strms.py

    r2097 r2098  
    2828    # Group symbols of length 3 to 4
    2929    temp = starts_1 | starts_2
    30     #temp32 = pablo.Advance32(temp)
    31     starts_3_to_4 = pablo.Advance(pablo.Advance(temp))
    32     #starts_3_to_4 = interpose32 (temp, temp32, 2)
     30    temp32 = pablo.Advance32(temp)
     31    #starts_3_to_4 = pablo.Advance(pablo.Advance(temp))
     32    starts_3_to_4 = interpose32 (temp, temp32, 2)
    3333    groups.follows_4 = starts_3_to_4 & remaining_follows
    3434    remaining_follows = remaining_follows & ~groups.follows_4
     
    3636    # Group symbols of length 5 to 8
    3737    temp = temp | starts_3_to_4
    38     #temp32 = pablo.Advance32(temp)
    39     starts_5_to_8 = pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(temp))))
    40     #starts_5_to_8 = interpose32 (temp, temp32, 4)
     38    temp32 = pablo.Advance32(temp)
     39    #starts_5_to_8 = pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(temp))))
     40    starts_5_to_8 = interpose32 (temp, temp32, 4)
    4141    groups.follows_8 = starts_5_to_8 & remaining_follows
    4242    remaining_follows = remaining_follows & ~groups.follows_8
     
    4444    # Group symbols of length 9 to 16
    4545    temp = temp | starts_5_to_8
    46     #temp32 = pablo.Advance32 (temp)
    47     starts_9_16 = pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(temp))))))))
    48     #starts_9_16 = interpose32 (temp, temp32, 8)
     46    temp32 = pablo.Advance32 (temp)
     47    #starts_9_16 = pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(temp))))))))
     48    starts_9_16 = interpose32 (temp, temp32, 8)
    4949    groups.follows_16 = starts_9_16 & remaining_follows
    5050    remaining_follows = remaining_follows & ~groups.follows_16
  • trunk/symbol_table/main_template.cpp

    r2097 r2098  
    2929 *
    3030 */
     31
     32#undef IDISA_ALWAYS_INLINE
    3133
    3234#include "transpose.hpp"
  • trunk/symbol_table/src/Makefile

    r2095 r2098  
    1 CC= g++ $(CFLAGS) -O3 # -g
     1CC= g++ $(CFLAGS) #-fno-inline -O3 # -g
    22SSE=-msse2 #-mssse3 #-msse4.2
    33CFLAGS= $(SSE)
     
    1717TEST_DST_DIR = ../test/out
    1818
     19# Perf builds
    1920id: basis_bits.hpp buffer.hpp byte_pool.hpp hash_strms.hpp  hash_table.hpp ../lib/hash.hpp  group_strms.hpp  symbol_table.hpp  main.cpp  marker_strms.hpp symbol_table.hpp transpose.hpp
    2021        $(CC) -o id main.cpp $(AFLAGS) -DID_STRATEGY -DBUFFER_PROFILING #-DID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG #-DHASH_TABLE_HPP_DEBUG
     
    2425
    2526logbase2: basis_bits.hpp buffer.hpp byte_pool.hpp hash_strms.hpp  hash_table.hpp ../lib/hash.hpp  group_strms.hpp  symbol_table.hpp  main.cpp  marker_strms.hpp symbol_table.hpp transpose.hpp
    26         $(CC) -o logbase2 main.cpp $(AFLAGS) -DLOG2_STRATEGY -DBUFFER_PROFILING #-DID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG  #-DHASH_TABLE_HPP_DEBUG
     27        $(CC) -o logbase2 main.cpp $(AFLAGS) -DLOGBASE2_STRATEGY -DBUFFER_PROFILING #-DID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG  #-DHASH_TABLE_HPP_DEBUG
    2728
     29div2_logbase2: basis_bits.hpp buffer.hpp byte_pool.hpp hash_strms.hpp  hash_table.hpp ../lib/hash.hpp  group_strms.hpp  symbol_table.hpp  main.cpp  marker_strms.hpp symbol_table.hpp transpose.hpp
     30        $(CC) -o div2_logbase2 main.cpp $(AFLAGS) -DDIV2_LOGBASE2_STRATEGY -DBUFFER_PROFILING -DID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG  #-DHASH_TABLE_HPP_DEBUG
     31
     32# Diff Tests
    2833id_diff_test: id
    2934        $(CC) -o id main.cpp $(AFLAGS) -DID_STRATEGY -DIDENTITY_TEST
     
    3742
    3843logbase2_diff_test: logbase2
    39         $(CC) -o logbase2 main.cpp $(AFLAGS) -DLOG2_STRATEGY -DIDENTITY_TEST
     44        $(CC) -o logbase2 main.cpp $(AFLAGS) -DLOGBASE2_STRATEGY -DIDENTITY_TEST
    4045        python $(TEST_SCRIPT) logbase2 -d
    4146        diff -rq --exclude=".svn" $(TEST_SRC_DIR) $(TEST_DST_DIR) || exit 0
    4247
     48div2_logbase2_diff_test: div2_logbase2
     49        $(CC) -o div2_logbase2 main.cpp $(AFLAGS) -DDIV2_LOGBASE2_STRATEGY -DIDENTITY_TEST
     50        python $(TEST_SCRIPT) div2_logbase2 -d
     51        diff -rq --exclude=".svn" $(TEST_SRC_DIR) $(TEST_DST_DIR) || exit 0
     52
     53# GID Tests
    4354id_gid_test:
    4455        $(CC) -o id main.cpp $(AFLAGS) -DID_STRATEGY -DGID_TEST
     
    5061
    5162logbase2_gid_test:
    52         $(CC) -o logbase2 main.cpp $(AFLAGS) -DLOG2_STRATEGY -DGID_TEST
     63        $(CC) -o logbase2 main.cpp $(AFLAGS) -DLOGBASE2_STRATEGY -DGID_TEST
    5364        python $(TEST_SCRIPT) logbase2 -g
    5465
     66div2_logbase2_gid_test:
     67        $(CC) -o div2_logbase2 main.cpp $(AFLAGS) -DDIV2_LOGBASE2_STRATEGY -DGID_TEST
     68        python $(TEST_SCRIPT) div2_logbase2 -g
     69
    5570clean:
    56         rm -Rf id div2 logbase2 $(TEST_DST_DIR)
     71        rm -Rf id div2 logbase2 div2_logbase2 $(TEST_DST_DIR)
    5772
     73
     74
     75# valgrind --tool=callgrind --callgrind-out-file=./callgrind.out ./logbase2 ../test/in/\(1_1000_10\)_\(2_1000_10\)_\(3_1000_10\)_\(4_1000_10\)_\(5_1000_10\)_\(6_1000_10\)_\(7_1000_10\)_\(8_1000_10\)_\(9_1000_10\)_\(10_1000_10\)_\(11_1000_10\)_\(12_1000_10\)_\(13_1000_10\)_\(14_1000_10\)_\(15_1000_10\)_\(16_1000_10\)_\(17_1000_10\)_\(18_1000_10\)_\(19_1000_10\)_1_1.test
  • trunk/symbol_table/src/hash_table.hpp

    r2092 r2098  
    5050
    5151public:
    52     hash_table(const uint32_t table_size=1024, const uint32_t resize_chain_lgth=2) {
     52    hash_table(const uint32_t table_size=4096, const uint32_t resize_chain_lgth=2) {
    5353        #ifdef HASH_TABLE_HPP_DEBUG
    5454            lookups = 0;
  • trunk/symbol_table/src/symbol_table.hpp

    r2097 r2098  
    215215                            BIT_HASH_FIXED(div2,bit,14);
    216216                            BIT_HASH_FIXED(div2,bit,16);
    217                         #elif LOG2_STRATEGY
     217                        #elif LOGBASE2_STRATEGY
    218218                            BYTE_HASH_VARIABLE(logbase2,byte,1);
    219219                            BYTE_HASH_VARIABLE(logbase2,byte,2);
    220220                            BYTE_HASH_VARIABLE(logbase2,byte,4);
    221221                            BIT_HASH_VARIABLE(logbase2,bit,8);
     222                            BIT_HASH_VARIABLE(logbase2,bit,16);
     223                        #elif DIV2_LOGBASE2_STRATEGY
     224                            BYTE_HASH_FIXED(div2,byte,2);
     225                            BYTE_HASH_FIXED(div2,byte,4);
     226                            BYTE_HASH_FIXED(div2,byte,6);
     227                            BIT_HASH_FIXED(div2,bit,8);
     228//                      BYTE_HASH_FIXED(id,byte,1);
     229//                      BYTE_HASH_FIXED(id,byte,2);
     230//                      BYTE_HASH_FIXED(id,byte,3);
     231//                      BYTE_HASH_FIXED(id,byte,4);
     232//                      BYTE_HASH_FIXED(id,byte,5);
     233//                      BYTE_HASH_FIXED(id,byte,6);
     234//                      BYTE_HASH_FIXED(id,byte,7);
     235//                      BIT_HASH_FIXED(id,bit,8);
    222236                            BIT_HASH_VARIABLE(logbase2,bit,16);
    223237                        #endif
     
    276290            div2_hash_table<14, div2, bit, ALLOCATOR> hash_table_14;
    277291            div2_hash_table<16, div2, bit, ALLOCATOR> hash_table_16;
    278         #elif LOG2_STRATEGY
     292        #elif LOGBASE2_STRATEGY
    279293            logbase2_hash_table<1, logbase2, byte, ALLOCATOR> hash_table_1;
    280294            logbase2_hash_table<2, logbase2, byte, ALLOCATOR> hash_table_2;
     
    282296            logbase2_hash_table<8, logbase2, bit, ALLOCATOR> hash_table_8;
    283297            logbase2_hash_table<16, logbase2, bit, ALLOCATOR> hash_table_16;
     298        #elif DIV2_LOGBASE2_STRATEGY
     299            div2_hash_table<2, div2, byte, ALLOCATOR> hash_table_2;
     300            div2_hash_table<4, div2, byte, ALLOCATOR> hash_table_4;
     301            div2_hash_table<6, div2, byte, ALLOCATOR> hash_table_6;
     302            div2_hash_table<8, div2, bit, ALLOCATOR> hash_table_8;
     303//      id_hash_table<1, id, byte, ALLOCATOR> hash_table_1;
     304//      id_hash_table<2, id, byte, ALLOCATOR> hash_table_2;
     305//      id_hash_table<3, id, byte, ALLOCATOR> hash_table_3;
     306//      id_hash_table<4, id, byte, ALLOCATOR> hash_table_4;
     307//      id_hash_table<5, id, byte, ALLOCATOR> hash_table_5;
     308//      id_hash_table<6, id, byte, ALLOCATOR> hash_table_6;
     309//      id_hash_table<7, id, byte, ALLOCATOR> hash_table_7;
     310//      id_hash_table<8, id, bit, ALLOCATOR> hash_table_8;
     311            logbase2_hash_table<16, logbase2, bit, ALLOCATOR> hash_table_16;
    284312        #else
    285             #error "Length group strategy not specified. #define {ID_STRATEGY,DIV2_STRATEGY,LOG2_STRATEGY}."
     313            #error "Length group strategy not specified. #define {ID_STRATEGY,DIV2_STRATEGY,LOGBASE2_STRATEGY,DIV2_LOGBASE2_STRATEGY}."
    286314        #endif
    287315
  • trunk/symbol_table/symbol_table.pro

    r2096 r2098  
    5454    test/uniform/[1_100_10][2_100_10][3_100_10][4_100_10][5_100_10][6_100_10][7_100_10][8_100_10][9_100_10][10_100_10][11_100_10][12_100_10][13_100_10][14_100_10][15_100_10][16_100_10][17_100_10][18_100_10][19_100_10]_1_1.test \
    5555    test/gen_test_file.py \
    56     logbase2_group_strms.py
     56    logbase2_group_strms.py \
     57    div2_logbase2_group_strms.py
    5758HEADERS += marker_strms_template.hpp \
    5859    marker_strms.hpp \
  • trunk/symbol_table/test/gen_test_file.py

    r2094 r2098  
    6969        min_lgth = 1
    7070        max_lgth = 20
    71         occurences = 1000
    72         unique = 10
     71        occurences = 1
     72        unique = 1
    7373
    7474        gen_test_file(min_lgth,max_lgth,occurences,unique)
Note: See TracChangeset for help on using the changeset viewer.