Changeset 3225 for proto/xmlschema


Ignore:
Timestamp:
May 29, 2013, 12:52:13 PM (6 years ago)
Author:
shiyangy
Message:
 
File:
1 edited

Legend:

Unmodified
Added
Removed
  • proto/xmlschema/validation_template_onevec.cpp

    r2219 r3225  
    1 #define USE_FUNCTION_TEMPLATES
    2 #define TEMPLATED_SIMD_LIB
     1
     2#include <simd-lib/bitblock.hpp>
     3#include <simd-lib/carryQ.hpp>
     4#include <simd-lib/pabloSupport.hpp>
     5
     6#define LocalCarryDeclare(name, count)\
     7CarryArray<count, 0> name;\
     8
     9#define assert_0_error(errkind, errstrm)
     10
     11BitBlock EOF_mask = simd<1>::constant<1>();
     12
     13//#define USE_FUNCTION_TEMPLATES
     14//#define TEMPLATED_SIMD_LIB
    315
    416#define DEBUG 1
    5 #define BLOCK_SIZE (sizeof(SIMD_type) * 8)
    6 #define SEGMENT_BLOCKS 12
    7 #define BUFFER_SIZE (BLOCK_SIZE * SEGMENT_BLOCKS)
    8 #define OVERLAP_BUFSIZE 2*(sizeof(SIMD_type))
     17//#define BLOCK_SIZE (sizeof(SIMD_type) * 8)
     18//#define SEGMENT_BLOCKS 12
     19//#define BUFFER_SIZE (BLOCK_SIZE * SEGMENT_BLOCKS)
     20//#define OVERLAP_BUFSIZE 2*(sizeof(SIMD_type))
    921
    1022#define VECTOR_WIDTH 8
     
    1628#include <sys/types.h>
    1729#include <sys/stat.h>
    18 #include "../lib/lib_simd.h"
    19 #include "../lib/symtab/pbgs_identity_symbol_table.h"
     30#include <limits.h>
     31
     32#include <simd-lib/s2p.hpp>
     33#include <simd-lib/buffer.hpp>
     34#include <simd-lib/bitblock_iterator.hpp>
     35#include <simd-lib/perflib/perfsec.h>
     36
     37//#include "../lib/lib_simd.h"
     38#include "../symtab/pbgs_identity_symbol_table.h"
     39
    2040#include <stack>
    2141#include <map>
     42#include <vector>
    2243
    2344#include <queue>
     
    2546
    2647typedef long ScanBlock;
    27 typedef SIMD_type BytePack;
    28 typedef SIMD_type BitBlock;
     48typedef BytePack BitBlock;
    2949
    3050using namespace std;
    3151
    32 #include "../lib/carryQ.h"
    33 #include "xmldecl.h"
    34 #include "xml_error.c"
    35 #include "xmldecl.c"
    36 #include "namechars.h"
    37 
    38 #include "../lib/perflib/perfsec.h"
    39 #include "../lib/s2p.h"
    40 
    41 #include "TagMatcher.h"
    42 #include "LineColTracker.h"
     52//#include "../lib/carryQ.h"
     53#include <xmldecl.h>
     54//#include "xml_error.c"
     55//#include "xmldecl.c"
     56#include <namechars.h>
     57#include <LineColTracker.hpp>
     58#include <XMLTestSuiteError.h>
     59
     60// Define the mappings for pablo.assert_0(strm, errkind) statements which
     61// compile to the the form assert_0_error(errkind, strm)
     62#include <ErrorTracker.h>
     63
     64ErrorTracker error_tracker;
     65#define assert_0_error(errkind, errstrm) error_tracker.NoteError(errkind, errstrm);
     66
     67#ifndef STL_ALIGNED_VECTOR
     68#include <TagMatcher.hpp>
     69#endif
     70
     71#ifdef STL_ALIGNED_VECTOR // experimental, comment out TagMatcher code to use STL aligned memory vector
     72#warning "Compiling with experimental STL_ALIGNED_VECTOR."
     73#include <vector>
     74#include <simd-lib/stl_aligned_allocator.hpp>
     75#endif
    4376
    4477#ifdef BUFFER_PROFILING
     
    5689#endif
    5790
    58 int block_base=0;
    59 int buffer_base=0;
     91//////////////////////////////////////////////////////////////////////////////////////////
     92// Buffer Management // WARNING: Do Not update #defines. Results in TagMatcher errors.
     93//////////////////////////////////////////////////////////////////////////////////////////
     94#define PADDING_BLOCKS 0
     95#define PADDING_SIZE (BLOCK_SIZE * PADDING_BLOCKS)
     96#define COPYBACK_BLOCKS 2
     97#define COPYBACK_SIZE (BLOCK_SIZE * COPYBACK_BLOCKS)
     98#define LOOKAHEAD_BLOCKS 1
     99#define LOOKAHEAD_SIZE (BLOCK_SIZE * LOOKAHEAD_BLOCKS)
     100#define SEGMENT_BLOCKS  12 // WARNING: TagMatcher.hpp causes xmlconf test suite failures for SEGMENT_BLOCKS < 3.
     101#define SEGMENT_SIZE (BLOCK_SIZE * SEGMENT_BLOCKS)
     102#define BUFFER_SIZE (COPYBACK_SIZE + SEGMENT_SIZE + LOOKAHEAD_SIZE + PADDING_SIZE)
     103
     104
     105
     106//int block_base=0;
     107//int buffer_base=0;
    60108int buffer_last;
    61 char * source;
     109FILE *gid_writein;
     110#define ELEMENT_BUFFER_SIZE 30
     111char element_name_buffer[ELEMENT_BUFFER_SIZE];
     112int cross_buffer_flag = 0;
     113//char * source;
    62114LineColTracker tracker;
    63115
     116static inline int cfzl(long x){
     117        unsigned long ux = x;
     118        if (x<0){
     119                ux = ULONG_MAX + x + 1;
     120        }
     121        if (ux==0)
     122                return 8*sizeof(long);
     123        int n;
     124        n = 0;
     125        while (1){
     126                if ((ux%2) == 1){
     127                        if(n==64){
     128                                cout << ux <<endl;
     129                        }
     130                        return n;               
     131                }
     132                n ++;
     133                ux = ux >> 1;
     134                //cout << "x = " << x << endl;
     135        }
     136}
     137
     138static inline void print_gid_vec();
     139/*
     140static inline int cfzl(long x){
     141 if (x ==0)
     142  return 32;
     143 int n=0;
     144 if ((x & 0xFFFF0000) == 0) { n += 16; x =x << 16;} //1111 1111 1111 1111 0000 0000 0000 0000 // 16 bits from left are zero! so we omit 16left bits
     145 if ((x & 0xFF000000) == 0){ n = n +  8; x = x <<  8;} // 8 left bits are 0
     146 if ((x & 0xF0000000) ==0){ n = n +  4; x = x <<  4;} // 4 left bits are 0
     147 if ((x & 0xC0000000) == 0){ n =n +  2, x = x <<  2;}  // 110000....0 2 left bits are zero
     148 if ((x & 0x80000000) == 0){n = n +  1, x = x <<  1;} // first left bit is zero
     149 return n;
     150
     151}
     152*/
    64153static inline void ReportError(const char * error_msg, int error_pos_in_block) {
    65154  int error_line, error_column;
     
    67156  fprintf(stderr, "%s at line %i, column %i\n", error_msg, error_line, error_column);
    68157}
    69 
     158/*
    70159class ErrorTracker {
    71160public:
     
    92181  int noted_pos_in_block;
    93182};
    94 
    95 
    96 TagMatcher matcher;
    97 BitBlock EOF_mask = simd_const_1(1);
    98 
    99 ErrorTracker error_tracker;
     183*/
     184
     185
     186//BitBlock EOF_mask = simd_const_1(1);
     187
     188//ErrorTracker error_tracker;
    100189BitBlock elem_starts;
    101190int previous_block_last_elem_start;
     
    112201static inline int ScanBackwardPos(BitBlock * block, int pos)
    113202{
     203        cout << "pos " << pos << endl;
    114204    BitBlock s = block[0];
    115     BitBlock temp = simd_and( s, simd_not(simd<128>::sll(simd<1>::constant<1>(), sisd_from_int(pos))) );
    116 
    117     if (bitblock_has_bit(temp))
     205    if (bitblock::any(block[0])){
     206        print_register("zero",block[0]);
     207        print_register("t",simd_not(simd<128>::sll(simd<1>::constant<1>(), convert(pos))) );
     208        }
     209    BitBlock temp = simd_and( s, simd_not(simd<128>::sll(simd<1>::constant<1>(), convert(pos))) );
     210//      print_register("zero",simd_not(simd<128>::sll(simd<1>::constant<1>());
     211                print_register("temp",temp);
     212    if (bitblock::any(temp))
    118213    {
    119214        // sizeof (BitBlock)*8 - cbzl( s & ~(~0 << pos)) - 1;
    120         return BLOCK_SIZE - count_backward_zeroes (temp) - 1;
     215//      cout << "block_size" << BLOCK_SIZE << endl;
     216        return BLOCK_SIZE - count_reverse_zeroes(temp) - 1;
    121217    }
    122218    else
     
    193289*/
    194290
    195 inline char ElemEnd_grouping(int pos) {
     291inline char ElemEnd_grouping(char *source,int block_base,int pos) {
    196292    int end = block_base + pos;
    197     int start = ScanBackwardPos (&elem_starts, pos) + block_base;
     293    int start = ScanBackwardPos(&elem_starts, pos) + block_base;
     294    ///////////////////////
     295    //start could be negitive
     296    //the element name falls in two buffers
     297    ///////////////////////
     298    char *name = source + start;
     299    char name_buffer[50];
     300   
     301    if (start<0){
     302        memmove (&name_buffer[0],&element_name_buffer[ELEMENT_BUFFER_SIZE+start],-start);       
     303        memmove (&element_name_buffer[ELEMENT_BUFFER_SIZE+start],&source[0],50+start);
     304        name = name_buffer;
     305    }
     306//    cout << end << "  " << start << endl;
    198307    int lgth = end - start;
    199308    int hashvalue = compute_hash_value(lgth, start - block_base);
    200309    char gid = 0;
     310
    201311
    202312//    if (lgth < 32)
     
    271381    symbol[lgth] ='\0';
    272382    printf ("%s | start: %i[%i] | end: %i[%i] | lgth: %i | hashvalue: %i | gid: %i | symbol: %s\n", __FUNCTION__, start, start - block_base, end, end - block_base, lgth, hashvalue, gid, symbol);
     383    fprintf (gid_writein,"%s | start: %i[%i] | end: %i[%i] | lgth: %i | hashvalue: %i | gid: %i | symbol: %s\n", __FUNCTION__, start, start - block_base, end, end - block_base, lgth, hashvalue, gid, symbol);
    273384#endif
    274385    return gid;
     
    276387
    277388//template <int L>
    278 static inline int StreamScanLengthGrouping(ScanBlock * stream, ScanBlock * stream2, int blk_count) {
     389static inline int StreamScanLengthGrouping(char *source,int block_base,ScanBlock * stream, ScanBlock * stream2, int blk_count) {
    279390    int blk;
    280391    int block_pos = 0;
     392//    cout << "cfzl():" << cfzl() << endl;
     393//      long test = 6;
     394//      test = (test & (test - 1));
     395//      cout << test << endl;
    281396    for (blk = 0; blk < blk_count; blk++) {
     397        cout << "blk:" << blk << endl;
     398   
    282399                ScanBlock s = stream[blk];
    283400                ScanBlock s2 = stream2[blk];
     401//              cout << s << "  " << s2 << endl;
     402//              print_register("s:",((BitBlock*)stream)[0]);
    284403                while(s | s2) {
     404                        cout << "cfzl(s):" << cfzl(s)<<endl;
    285405                    int end_pos = cfzl(s) + block_pos;
     406//                  cout << "s2 " << s2 << endl;
    286407                    int end_tag = cfzl(s2) + block_pos;
    287408//              cout << "cfzl(s):" << cfzl(s)<<endl;
     
    289410//                      cout << (end_tag < end_pos && s2 != 0)<< "||" <<(s == 0 && s2 != 0)  << endl;
    290411                        if ((end_tag < end_pos && s2 != 0) | (s == 0 && s2 != 0)){
    291 //                      cout << "closing tag" << endl;
     412                        cout << "here" << endl;
     413                        cout << "closing tag" << endl;
    292414//                      cout << "endtag:" << end_tag << " " << end_pos<<endl;
    293415//                      cout << "1"<< endl;
     
    296418//                      cout << elem_vec_stack.top()<<endl;
    297419                                if(!(gid_vec.empty())){
    298                                         if(gid_vec.back() != 0){
     420//                                      if(gid_vec.back() != 0){
    299421                                                gid_vec.push_back(0);
    300                                         }
     422//                                      }
    301423                                }
    302                         s2 = s2 & (s2 - 1);
     424                                cout << "before" << s2 << endl;
     425                                s2 = s2 & (s2 - 1);
     426                                cout << "after" << s2 << endl;
     427//                      cout << "s2 = " << s2 << endl;
    303428                }else{
    304 //                      cout << " end_pos == " << end_pos<<endl;
    305                         char gid = ElemEnd_grouping(end_pos);
    306                         s = s & (s-1);  // clear rightmost bit.
     429                        cout << " end_pos == " << end_pos<<endl;
     430                        cout << " end_tag_pos == " << end_tag << endl;
     431                        char gid = ElemEnd_grouping(source,block_base,end_pos);
     432                        if (s!=0){
     433                                s = s & (s-1);  // clear rightmost bit.
     434                        }
     435                        //else{
     436                        //      s2 = s2 & (s2 - 1);
     437                        //}
    307438                                if (gid != 0){
    308439                                        gid_vec.push_back(gid);                 
    309440                                }
    310 
     441                                cout << "here1" << endl;
    311442                }
    312443                }
    313444                block_pos += 8 * sizeof(ScanBlock);
     445                print_gid_vec();
    314446    }
    315447    return 0;
     
    343475*/
    344476}
    345 
     477/*
    346478static inline int NameStrt_check(int pos) {
    347479        int block_pos = block_base + pos;
     
    388520        unsigned char* s = (unsigned char*)&source[block_pos];
    389521        if (!(at_Ref_gt<ASCII>(s)||at_Ref_lt<ASCII>(s)||at_Ref_amp<ASCII>(s)||at_Ref_quot<ASCII>(s)||at_Ref_apos<ASCII>(s))){
     522                        cout << pos << endl;
    390523              ReportError("Undefined reference", pos);
    391524              exit(-1);
     
    470603        return 0;
    471604}
     605*/
    472606
    473607
     
    475609@global
    476610
    477 
    478 
     611#include <simd-lib/transpose.hpp>
     612#include <post_process.hpp>
     613/*
    479614static inline void s2p_do_block(BytePack U8[], Basis_bits & basis_bits) {
    480615  s2p(U8[0], U8[1], U8[2], U8[3], U8[4], U8[5], U8[6], U8[7],
     
    493628  basis_bits.bit_7 = simd_and(basis_bits.bit_7, EOF_mask);
    494629}
    495 
     630*/
    496631static inline void print_basis_bits(Basis_bits & basis_bits){
    497         print_bit_block("bit0:",basis_bits.bit_0);
    498         print_bit_block("bit1:",basis_bits.bit_1);
    499         print_bit_block("bit2:",basis_bits.bit_2);
    500         print_bit_block("bit3:",basis_bits.bit_3);
    501         print_bit_block("bit4:",basis_bits.bit_4);
    502         print_bit_block("bit5:",basis_bits.bit_5);
    503         print_bit_block("bit6:",basis_bits.bit_6);
    504         print_bit_block("bit7:",basis_bits.bit_7);                                     
     632        print_register("bit0:",basis_bits.bit_0);
     633        print_register("bit1:",basis_bits.bit_1);
     634        print_register("bit2:",basis_bits.bit_2);
     635        print_register("bit3:",basis_bits.bit_3);
     636        print_register("bit4:",basis_bits.bit_4);
     637        print_register("bit5:",basis_bits.bit_5);
     638        print_register("bit6:",basis_bits.bit_6);
     639        print_register("bit7:",basis_bits.bit_7);                                       
    505640}
    506641
     
    536671                cout << endl;
    537672                if (remaining !=0){
    538                         BitBlock EOF_mask = sisd_srl(simd_const_1(1),sisd_from_int(BLOCK_SIZE-remaining));
     673                        BitBlock EOF_mask = bitblock::srl(simd<1>::constant<1>(),convert(BLOCK_SIZE-remaining));
    539674                        s2p_do_final_block((BytePack*)(&(gid_vec[i])), temp_basis_bits,EOF_mask);
    540675                        print_elem_vec(&(gid_vec),lgth);
     
    587722}
    588723
     724static inline void vectoring(Tag_Callouts & tag_Callouts, Hash_data & hash_data,char *source,int block_base){
     725
     726        elem_starts = tag_Callouts.ElemName_starts;
     727    hashvalues[1] = hash_data.Hash_value;
     728   
     729        StreamScanLengthGrouping(source,block_base,(ScanBlock *) &tag_Callouts.ElemName_ends,(ScanBlock *) &tag_Callouts.Tag_closing, sizeof(BitBlock)/sizeof(ScanBlock));
     730       
     731        // Store the last starting position in case we hit boundary case
     732    previous_block_last_elem_start = - count_reverse_zeroes(elem_starts);
     733
     734    //copy current hash value data as previous one.
     735    memmove (&hashvalues[0], &hashvalues[1], 16);
     736}
     737/*
    589738static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, Hash_data & hash_data, int chars_avail){
    590739
     
    689838
    690839
    691 
     840/*
    692841    // Store the last starting position in case we hit boundary case
    693     previous_block_last_elem_start = - count_backward_zeroes (elem_starts);
     842    previous_block_last_elem_start = - count_reverse_zeroes(elem_starts);
    694843
    695844    //copy current hash value data as previous one.
    696845    memmove (&hashvalues[0], &hashvalues[1], 16);
    697846
    698     if (bitblock_has_bit(simd_or(check_streams.non_ascii_name_starts, check_streams.non_ascii_names))) {
     847    if (bitblock::any(simd_or(check_streams.non_ascii_name_starts, check_streams.non_ascii_names))) {
    699848      StreamScan((ScanBlock *) &check_streams.non_ascii_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), NameStrt_check);
    700849      StreamScan((ScanBlock *) &check_streams.non_ascii_names, sizeof(BitBlock)/sizeof(ScanBlock), Name_check);
    701850    }
    702851
    703     if (bitblock_has_bit(ctCDPI_Callouts.PI_name_starts)){
     852    if (bitblock::any(ctCDPI_Callouts.PI_name_starts)){
    704853      StreamScan((ScanBlock *) &ctCDPI_Callouts.PI_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), PIName_check);
    705854    }
    706855
    707     if (bitblock_has_bit(ctCDPI_Callouts.CD_starts)){
     856    if (bitblock::any(ctCDPI_Callouts.CD_starts)){
    708857      StreamScan((ScanBlock *) &ctCDPI_Callouts.CD_starts, sizeof(BitBlock)/sizeof(ScanBlock), CD_check);
    709858    }
    710859
    711     if (bitblock_has_bit(ref_Callouts.GenRef_starts)){
     860    if (bitblock::any(ref_Callouts.GenRef_starts)){
    712861      StreamScan((ScanBlock *) &ref_Callouts.GenRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), GenRef_check);
    713862    }
    714863
    715     if (bitblock_has_bit(ref_Callouts.DecRef_starts)){
     864    if (bitblock::any(ref_Callouts.DecRef_starts)){
    716865      StreamScan((ScanBlock *) &ref_Callouts.DecRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), DecRef_check);
    717866    }
    718867
    719     if (bitblock_has_bit(ref_Callouts.HexRef_starts)){
     868    if (bitblock::any(ref_Callouts.HexRef_starts)){
    720869      StreamScan((ScanBlock *) &ref_Callouts.HexRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), HexRef_check);
    721870    }
    722871
    723     if (bitblock_has_bit(check_streams.att_refs)){
     872    if (bitblock::any(check_streams.att_refs)){
    724873      StreamScan((ScanBlock *) &check_streams.att_refs, sizeof(BitBlock)/sizeof(ScanBlock), AttRef_check);
    725874    }
    726875
    727     error_tracker.If_Error_Report_First();
    728 
     876//    error_tracker.If_Error_Report_First();
     877/*
    729878    matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
    730879    tracker.AdvanceBlock();
    731 }
     880    */
     881    /*
     882}
     883*/
    732884
    733885static inline void print_GIDS()
     
    747899void do_process(FILE *infile, FILE *outfile) {
    748900
     901        ///////////////
     902        //gid file
     903        ///////////////
     904        gid_writein = fopen("gid.out","w");
    749905@decl
    750   int buf_pos = 0;
    751   int block_pos = 0;
    752   int errpos = 0;
    753   int chars_avail = 0;
    754   int check_pos = 0;
    755   int chars_read = 0;
    756   BytePack buf[(BUFFER_SIZE+BLOCK_SIZE+OVERLAP_BUFSIZE*2)/sizeof(SIMD_type)];
    757 
    758   char * srcbuf = ((char *) buf) + OVERLAP_BUFSIZE;
     906        #ifndef STL_ALIGNED_VECTOR
     907                TagMatcher<SEGMENT_SIZE,LOOKAHEAD_SIZE> matcher;
     908        #endif
     909        ///////////////////////////////////////////////////
     910        //preload element_IDs into symbol table from schema
     911        ///////////////////////////////////////////////////
     912        FILE *element_ID_in = fopen("../test/element_ID","r");
     913        char symble[50];
     914        int index;
     915        if(element_ID_in == NULL){
     916                printf("Error opening element_ID file.\n");
     917                exit(0);
     918        }
     919        hashvalues[1] = hash_data.Hash_value;
     920        while(fscanf(element_ID_in,"%s %d\n",&symble[0],&index)==2){
     921                int lgth = strlen(symble);
     922                int hashvalue = compute_hash_value(lgth,1);
     923                cout << symble <<" "<< lgth<<" "<<hashvalue << endl;
     924                int gid = 0;
     925                switch (lgth)
     926        {
     927                        case 1:
     928                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<1>(symble, hashvalue);
     929                        break;
     930                        case 2:
     931                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<2>(symble, hashvalue);
     932                        break;
     933                        case 3:
     934                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<3>(symble, hashvalue);
     935                        break;
     936                        case 4:
     937                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<4>(symble, hashvalue);
     938                        break;
     939                        case 5:
     940                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<5>(symble, hashvalue);
     941                        break;
     942                        case 6:
     943                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<6>(symble, hashvalue);
     944                        break;
     945                        case 7:
     946                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<7>(symble, hashvalue);
     947                        break;
     948                        case 8:
     949                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<8>(symble, hashvalue);
     950                        break;
     951                        case 9:
     952                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<9>(symble, hashvalue);
     953                        break;
     954                        case 10:
     955                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<10>(symble, hashvalue);
     956                        break;
     957                        case 11:
     958                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<11>(symble, hashvalue);
     959                        break;
     960                        case 12:
     961                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<12>(symble, hashvalue);
     962                        break;
     963                        case 13:
     964                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<13>(symble, hashvalue);
     965                        break;
     966                        case 14:
     967                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<14>(symble, hashvalue);
     968                        break;
     969                        case 15:
     970                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<15>(symble, hashvalue);
     971                        break;
     972                        case 16:
     973                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<16>(symble, hashvalue);
     974                        break;
     975                        default:
     976                        gid = pbgs_symbol_table.Lookup_or_Insert_Name(symble, hashvalue, lgth);
     977                        break;         
     978                }
     979                cout << "gid=" << gid << endl;
     980        }       
     981        fclose(element_ID_in);
     982        return;
     983
     984        int buffer_base = 0;
     985        int block_base = 0;
     986        int buffer_pos = 0;
     987        int block_pos = 0;
     988        int errpos = 0;
     989        int chars_avail = 0;
     990        int check_pos = 0;
     991        int chars_read = 0;
     992 
     993        //////////////////////////////////////////////////////////////////////////////////////////
     994    // Buffer Management
     995    //////////////////////////////////////////////////////////////////////////////////////////
     996                //      BitBlock buf[(BUFFER_SIZE)/sizeof(BitBlock)];
     997                //      uint8_t * src_buffer = (uint8_t *)buf + COPYBACK_SIZE;
     998
     999                #ifdef STL_ALIGNED_VECTOR
     1000                        std::vector<uint8_t, AAllocator<uint8_t> > src_buffer;
     1001                        src_buffer.reserve(BUFFER_SIZE);
     1002                #else
     1003                        uint8_t * COPYBACK;
     1004                        uint8_t * src_buffer;
     1005                        ALLOC_STATIC_ALIGNED_BYTE_BUFFER_WITH_COPYBACK(COPYBACK, src_buffer);
     1006                #endif
     1007  /*
     1008  BytePack buf[(BUFFER_SIZE+BLOCK_SIZE+LOOKAHEAD_SIZE*2)/sizeof(SIMD_type)];
     1009
     1010  char * srcbuf = ((char *) buf) + LOOKAHEAD_SIZE;
    7591011  buffer_base = buf_pos;
    7601012  source = srcbuf;
    761   chars_read = fread((void *)srcbuf, 1, BUFFER_SIZE + OVERLAP_BUFSIZE, infile);
     1013  */
     1014  chars_read = fread((void *)&src_buffer[0], 1, SEGMENT_SIZE, infile);
    7621015  chars_avail = chars_read;
    763   if (chars_avail > BUFFER_SIZE) chars_avail = BUFFER_SIZE;
    764 
    765   matcher.setSrc(srcbuf);
     1016  if (chars_avail > BUFFER_SIZE) chars_avail = SEGMENT_SIZE;
    7661017
    7671018  if(chars_read<4){
     
    7711022
    7721023  Entity_Info * e = new Entity_Info;
    773   e->AnalyzeSignature((unsigned char *)srcbuf);
     1024  e->AnalyzeSignature((unsigned char *)&src_buffer[0]);
    7741025
    7751026  if (e->code_unit_base == ASCII) {
    7761027
    777     XML_Decl_Parser<ASCII> decl_parser((unsigned char *)srcbuf);
     1028    XML_Decl_Parser<ASCII> decl_parser((unsigned char *)&src_buffer[0]);
    7781029
    7791030    decl_parser.ReadXMLInfo(*e);
     
    7901041
    7911042  if (e->content_start != 0) {
    792         memmove(&srcbuf[0], &srcbuf[e->content_start], chars_read - e->content_start);
    793         buf_pos = e->content_start;
    794         buffer_base = buf_pos;
    795         if (chars_avail == BUFFER_SIZE) {
    796                 chars_read = chars_read - e->content_start +
    797                              fread(&srcbuf[chars_read-e->content_start], 1, e->content_start, infile);
    798                 chars_avail = chars_read;
    799                 if (chars_avail > BUFFER_SIZE) chars_avail = BUFFER_SIZE;
    800         }
    801         else {
    802           chars_read -=e->content_start;
    803           chars_avail -=e->content_start;
    804         }
     1043        memmove(&src_buffer[0], &src_buffer[e->content_start], chars_read - e->content_start);
     1044        buffer_pos = e->content_start;
     1045//      buffer_base = buffer_pos;
     1046        if ((chars_avail-e->content_start) < SEGMENT_SIZE) {
     1047                        chars_read = chars_avail - e->content_start + fread(&src_buffer[chars_avail-e->content_start], 1, e->content_start, infile);
     1048                        chars_avail = chars_read;
     1049                }
     1050                if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
    8051051  }
    8061052
     1053        //////////////////////////////////////////////////////////////////////////////////////////
     1054        // Read OVERLAP bytes to support post processing validation lookahead.
     1055        //////////////////////////////////////////////////////////////////////////////////////////
     1056        chars_read = chars_avail + fread(&src_buffer[chars_avail], 1, LOOKAHEAD_SIZE, infile);
     1057        chars_avail = chars_read;
     1058        if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
     1059
    8071060@stream_stmts
     1061
     1062        #ifndef STL_ALIGNED_VECTOR
     1063        matcher.setSrc((char *)&src_buffer[0]);
     1064        #endif
    8081065
    8091066/* Full Buffers */
    8101067    int block_segment_num = 0;
    811     while (chars_avail == BUFFER_SIZE) {
     1068    while (chars_avail >= SEGMENT_SIZE) {
    8121069      PERF_SEC_START(parser_timer);
    8131070      for (int blk = 0; blk < SEGMENT_BLOCKS; blk++) {
    814           block_base = blk*BLOCK_SIZE;
    815           s2p_do_block((BytePack *) &srcbuf[block_base], basis_bits);
    816           @block_stmts
    817           postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, tag_Callouts, hash_data, chars_avail);
     1071        cout << "loops" << ends;
     1072                block_base = blk*BLOCK_SIZE;
     1073        s2p_do_block((BytePack *) &src_buffer[block_base], basis_bits);
     1074     
     1075                @block_stmts
     1076         
     1077                tracker.StoreNewlines(lex.LF);
     1078                cout << "post start" << endl;
     1079//              postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, tag_Callouts, hash_data, chars_avail);
     1080                postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)&src_buffer[0], buffer_base, block_base, chars_avail, tracker);
     1081                vectoring(tag_Callouts,hash_data,(char *)&src_buffer[0],block_base);
     1082                cout << "post done" << endl;
     1083                #ifndef STL_ALIGNED_VECTOR
     1084                matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
     1085                #endif
     1086                tracker.AdvanceBlock();
     1087                cout << "loop" << endl;
    8181088      }
     1089      #ifndef STL_ALIGNED_VECTOR
    8191090      matcher.StreamScan(chars_avail);
    8201091      matcher.Advance_buffer();
    821       tracker.Advance_buffer();
     1092      #endif
     1093      cout << "11111" << endl;
    8221094      PERF_SEC_END(parser_timer, chars_avail);
    823 
     1095          cout << "here" << endl;
    8241096      int bytes_left = chars_read - chars_avail;
    825       memmove(buf, &srcbuf[BUFFER_SIZE - OVERLAP_BUFSIZE], bytes_left + OVERLAP_BUFSIZE);
    826       chars_read = fread(&srcbuf[bytes_left],1, BUFFER_SIZE + OVERLAP_BUFSIZE - bytes_left, infile) + bytes_left;
    827 
     1097      //////////////////
     1098      //element name buffer, for scanning element name backwards
     1099      /////////////////
     1100      memmove(&element_name_buffer[0],&src_buffer[SEGMENT_SIZE-ELEMENT_BUFFER_SIZE],ELEMENT_BUFFER_SIZE);
     1101      cross_buffer_flag = 1;
     1102     
     1103      memmove(&src_buffer[0], &src_buffer[SEGMENT_SIZE], LOOKAHEAD_SIZE);
     1104      chars_read = fread(&src_buffer[LOOKAHEAD_SIZE],1,SEGMENT_SIZE, infile) + LOOKAHEAD_SIZE;
    8281105      chars_avail = chars_read;
    829       if (chars_avail > BUFFER_SIZE) chars_avail = BUFFER_SIZE;
    830       buf_pos += chars_avail;
    831       buffer_base = buf_pos;
     1106      cout << "here" << endl;
     1107      if (chars_avail > SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
     1108      buffer_pos += chars_avail;
     1109      buffer_base = buffer_pos;
    8321110      block_segment_num++;
    8331111
    8341112  }
    835  
    8361113       
    8371114/* Final Partial Buffer */
     
    8461123    while (remaining >= BLOCK_SIZE) {
    8471124          block_base = block_pos;
    848           s2p_do_block((BytePack *) &srcbuf[block_pos], basis_bits);
     1125          s2p_do_block((BytePack *) &src_buffer[block_pos], basis_bits);
     1126         
    8491127          @block_stmts
    850           postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, tag_Callouts, hash_data, chars_avail);
     1128         
     1129          tracker.StoreNewlines(lex.LF);
     1130          cout << "post start1" << endl;
     1131         
     1132//        postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, tag_Callouts, hash_data, chars_avail);
     1133          postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)&src_buffer[0], buffer_base, block_base, chars_avail, tracker);
     1134          vectoring(tag_Callouts,hash_data,(char *)&src_buffer[0],block_base);
     1135          cout << "post end2" << endl;
     1136          #ifndef STL_ALIGNED_VECTOR
     1137          matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
     1138          #endif
     1139          tracker.AdvanceBlock();
    8511140          block_pos += BLOCK_SIZE;
    8521141          remaining -= BLOCK_SIZE;
    8531142    }
    8541143    block_base = block_pos;
     1144        #ifdef USE_ANY_CARRY_TEST
    8551145    if (remaining > 0 || @any_carry) {
    856           EOF_mask = sisd_srl(simd_const_1(1),sisd_from_int(BLOCK_SIZE-remaining));
    857           s2p_do_final_block((BytePack *) &srcbuf[block_pos], basis_bits, EOF_mask);
     1146        #endif
     1147          EOF_mask = bitblock::srl(simd<1>::constant<1>(),convert(BLOCK_SIZE-remaining));
     1148          s2p_do_final_block((BytePack *) &src_buffer[block_pos], basis_bits, EOF_mask);
    8581149          @final_block_stmts
    859           postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, tag_Callouts, hash_data, chars_avail);
    860     }
    861     buf_pos += chars_avail;
    862     buffer_base = buf_pos;
    863 
     1150          cout << "post start3" << endl;
     1151//        postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, tag_Callouts, hash_data, chars_avail);
     1152          postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)&src_buffer[0], buffer_base, block_base, chars_avail, tracker);
     1153          vectoring(tag_Callouts,hash_data,(char *)&src_buffer[0],block_base);
     1154          cout << "post end3" << endl;
     1155          #ifndef STL_ALIGNED_VECTOR
     1156      matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
     1157          #endif
     1158          tracker.AdvanceBlock();
     1159        #ifdef USE_ANY_CARRY_TEST
     1160    }
     1161    #endif
     1162    buffer_pos += chars_avail;
     1163    buffer_base = buffer_pos;
     1164        #ifndef STL_ALIGNED_VECTOR
    8641165    matcher.StreamScan(chars_avail);
    8651166    matcher.Advance_buffer();
    866     tracker.Advance_buffer();
    867 
    868     PERF_SEC_END(parser_timer, chars_avail);
     1167//    tracker.Advance_buffer();
     1168
     1169   
    8691170    if (matcher.depth != 0) {
    8701171      fprintf(stderr, "tag matching error (depth %i) at position %i\n", matcher.depth, buffer_base);
    8711172      exit(-1);
    872     }
     1173     
     1174     
     1175    }
     1176    #endif
     1177    PERF_SEC_END(parser_timer, chars_avail);
    8731178//  print_GIDS();
    8741179//    pbgs_symbol_table.Print_Symbol_Table_Distribution();
     
    9151220print_gid_vec();
    9161221#endif
    917 
     1222        //cout << "validate start" << endl;
    9181223        validate();
    9191224
Note: See TracChangeset for help on using the changeset viewer.