source: trunk/symtab/driver.cxx @ 4035

#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <sched.h>

// Test Repetitions
#define RUNS 1

// stream2runs versions
//#define BRANCH_REDUCTION
#include "../stream2runs/src/stream2runs.h"

// Performance Measurement
#ifdef PAPI
#include "lib/cclib/clocker/cc.h"
#include "lib/cclib/clocker/cc.cxx"
CC * code_clocker;
#endif

// Parabix 1 Symbol Table implementation (STL Hash Map)
#include "symtab.h"

// Length Sorted Symbol Table

#include "library_conversion.h"
#include "ls_symbol_table.h"

#include "symtab.h"
#include "ls_symbol_table_util.h"

#define SIMD_REGISTER_BIT_WIDTH (sizeof(SIMD_type) << 3)
#define REGISTER_BIT_WIDTH (sizeof(void *) << 3)

#ifdef TEMPLATED_SIMD_LIB
#define s2p_step(s0, s1, hi_mask, shift, p0, p1) \
{\
  BitBlock t0, t1;\
  t0 = simd<16>::pack<h,h>(s0, s1);\
  t1 = simd<16>::pack<l,l>(s0, s1);\
  p0 = simd_if(hi_mask, t0, simd<16>::srli<shift>(t1));\
  p1 = simd_if(hi_mask, simd<16>::slli<shift>(t0), t1);\
}
#endif
#ifndef TEMPLATED_SIMD_LIB
#define s2p_step(s0, s1, hi_mask, shift, p0, p1) \
{\
  BitBlock t0, t1;\
  t0 = simd_pack_16_hh(s0, s1);\
  t1 = simd_pack_16_ll(s0, s1);\
  p0 = simd_if(hi_mask, t0, simd_srli_16(t1, shift));\
  p1 = simd_if(hi_mask, simd_slli_16(t0, shift), t1);\
}
#endif

static inline void s2p_bytepack(BytePack s[], BitBlock p[]) {
#ifdef TEMPLATED_SIMD_LIB
    BitBlock mask_2 = simd<2>::himask();
    BitBlock mask_4 = simd<4>::himask();
    BitBlock mask_8 = simd<8>::himask();
#endif
#ifndef TEMPLATED_SIMD_LIB
    BitBlock mask_2 = simd_himask_2;
    BitBlock mask_4 = simd_himask_4;
    BitBlock mask_8 = simd_himask_8;
#endif
    BitBlock bit00224466_0, bit00224466_1, bit00224466_2, bit00224466_3;
    BitBlock bit11335577_0, bit11335577_1, bit11335577_2, bit11335577_3;
    BitBlock bit00004444_0, bit22226666_0, bit00004444_1, bit22226666_1;
    BitBlock bit11115555_0, bit33337777_0, bit11115555_1, bit33337777_1;
#if (BYTE_ORDER == BIG_ENDIAN)
    s2p_step(s[0], s[1], mask_2, 1, bit00224466_0, bit11335577_0);
    s2p_step(s[2], s[3], mask_2, 1, bit00224466_1, bit11335577_1);
    s2p_step(s[4], s[5], mask_2, 1, bit00224466_2, bit11335577_2);
    s2p_step(s[6], s[7], mask_2, 1, bit00224466_3, bit11335577_3);
#endif
#if (BYTE_ORDER == LITTLE_ENDIAN)
    s2p_step(s[7], s[6], mask_2, 1, bit00224466_0, bit11335577_0);
    s2p_step(s[5], s[4], mask_2, 1, bit00224466_1, bit11335577_1);
    s2p_step(s[3], s[2], mask_2, 1, bit00224466_2, bit11335577_2);
    s2p_step(s[1], s[0], mask_2, 1, bit00224466_3, bit11335577_3);
#endif
    s2p_step(bit00224466_0, bit00224466_1, mask_4, 2, bit00004444_0, bit22226666_0);
    s2p_step(bit00224466_2, bit00224466_3, mask_4, 2, bit00004444_1, bit22226666_1);
    s2p_step(bit11335577_0, bit11335577_1, mask_4, 2, bit11115555_0, bit33337777_0);
    s2p_step(bit11335577_2, bit11335577_3, mask_4, 2, bit11115555_1, bit33337777_1);
    s2p_step(bit00004444_0, bit00004444_1, mask_8, 4, p[0], p[4]);
    s2p_step(bit11115555_0, bit11115555_1, mask_8, 4, p[1], p[5]);
    s2p_step(bit22226666_0, bit22226666_1, mask_8, 4, p[2], p[6]);
    s2p_step(bit33337777_0, bit33337777_1, mask_8, 4, p[3], p[7]);
}

#define double_int64_adc(x1, x2, y1, y2, rslt1, rslt2, carry) \
  __asm__  ("sahf\n\t" \
        "adc %[e1], %[z1]\n\t" \
        "adc %[e2], %[z2]\n\t" \
        "lahf\n\t" \
     : [z1] "=r" (rslt1), [z2] "=r" (rslt2), [carryflag] "=a" (carry) \
         : "[z1]" (x1), "[z2]" (x2), \
           [e1] "r" (y1), [e2] "r" (y2), \
           "[carryflag]" (carry) \
         : "cc")

#define adc128(first, second, carry, sum) \
do\
{\
  union {__m128i bitblock;\
         uint64_t int64[2];} rslt;\
\
  union {__m128i bitblock;\
         uint64_t int64[2];} x;\
\
  union {__m128i bitblock;\
         uint64_t int64[2];} y;\
\
  x.bitblock = first;\
  y.bitblock = second;\
\
  double_int64_adc(x.int64[0], x.int64[1], y.int64[0], y.int64[1], rslt.int64[0], rslt.int64[1], carry);\
  sum = rslt.bitblock;\
}while(0)

#define double_int64_sbb(x1, x2, y1, y2, rslt1, rslt2, carry) \
  __asm__  ("sahf\n\t" \
        "sbb %[e1], %[z1]\n\t" \
        "sbb %[e2], %[z2]\n\t" \
        "lahf\n\t" \
     : [z1] "=r" (rslt1), [z2] "=r" (rslt2), [carryflag] "=a" (carry) \
         : "[z1]" (x1), "[z2]" (x2), \
           [e1] "r" (y1), [e2] "r" (y2), \
           "[carryflag]" (carry) \
         : "cc")

#define sbb128(first, second, carry, sum) \
do\
{ union {__m128i bitblock;\
         uint64_t int64[2];} rslt;\
\
  union {__m128i bitblock;\
         uint64_t int64[2];} x;\
\
  union {__m128i bitblock;\
         uint64_t int64[2];} y;\
\
  x.bitblock = first;\
  y.bitblock = second;\
\
  double_int64_sbb(x.int64[0], x.int64[1], y.int64[0], y.int64[1], \
                   rslt.int64[0], rslt.int64[1], carry);\
  sum = rslt.bitblock;\
}while(0)

// Mark Comma bit positions
static inline BitBlock bytepack2bitblock(BytePack U8[]);
static inline BitBlock bytepack2bitblock(BytePack U8[]) {
                
        BitBlock result;
        BitBlock array_u8bit__5_;
        //BitBlock AllOne = simd_const_1(1);
        //BitBlock AllZero = simd_const_1(0);
        BitBlock array_u8bit__2_;
        BitBlock array_u8bit__3_;
        BitBlock array_u8bit__4_;
        BitBlock _strct_s2iclass__classify_bytes__temp4;
        BitBlock _strct_s2iclass__classify_bytes__temp5;
        BitBlock _strct_s2iclass__classify_bytes__temp2;
        BitBlock _strct_s2iclass__classify_bytes__temp3;
        BitBlock array_u8bit__6_;
        BitBlock _strct_s2iclass__classify_bytes__temp1;
        BitBlock array_u8bit__0_;
        BitBlock array_u8bit__1_;
        BitBlock array_u8bit__7_;       
        
        BitBlock u8[8];
        
        s2p_bytepack(U8,u8);
        array_u8bit__0_ = u8[0];
        array_u8bit__1_ = u8[1];
        array_u8bit__2_ = u8[2];
        array_u8bit__3_ = u8[3];
        array_u8bit__4_ = u8[4];
        array_u8bit__5_ = u8[5];
        array_u8bit__6_ = u8[6];
        array_u8bit__7_ = u8[7];

        _strct_s2iclass__classify_bytes__temp1 = simd_or(array_u8bit__0_,array_u8bit__1_);
        _strct_s2iclass__classify_bytes__temp2 = simd_and(array_u8bit__2_,array_u8bit__3_);
        _strct_s2iclass__classify_bytes__temp3 = simd_andc(_strct_s2iclass__classify_bytes__temp2,_strct_s2iclass__classify_bytes__temp1);
        _strct_s2iclass__classify_bytes__temp4 = simd_or(array_u8bit__5_,array_u8bit__6_);
        _strct_s2iclass__classify_bytes__temp5 = simd_and(array_u8bit__4_,_strct_s2iclass__classify_bytes__temp4);      
        result = simd_andc(_strct_s2iclass__classify_bytes__temp3,_strct_s2iclass__classify_bytes__temp5);
        
        return result;
}

void print_chars(const char * str, size_t length);
void print_chars(const char * str, size_t length) {

        for(int i=0;i<length;i++) {
                printf("%c",str[i]);
        }
        printf("\n");
        
}

void SetCPUAffinity();
void SetCPUAffinity() {

        printf("Setting CPU Affinity...\n");
        
    cpu_set_t mask;
    unsigned int len = sizeof(mask);
    if (sched_getaffinity(0, len, &mask) < 0) {
        perror("sched_getaffinity");
    }

    printf("Original CPU Affinity Mask: %08lx\n", mask.__bits[0]);

    //CPU_CLR(0, &mask); // (CPU 1)
    //CPU_CLR(1, &mask); // (CPU 0)

    if (sched_setaffinity(0, len, &mask) < 0) {
       perror("sched_setaffinity");
    }

    printf("Modified CPU Affinity Mask: %08lx\n", mask.__bits[0]);
}  

int main(int argc, char * argv[]) {
        
  if (argc < 2) {
    printf("Usage: %s <filename> [<outputfile>]\n", argv[0]);
          exit(-1);
  }
  char * filename = argv[1];

  FILE *infile, *outfile;
  infile = fopen(filename, "rb");
  if (!infile) {
      fprintf(stderr, "Error: cannot open %s for input.\n", filename);
      exit(-1);
  }

  if (argc < 3) outfile = stdout;
  else {
    outfile = fopen(argv[2], "wb");
    if (!outfile) {
      fprintf(stderr, "Error: cannot open %s for writing.\n", argv[2]);
      exit(-1);
    }
  }

#ifdef PAPI
    SetCPUAffinity();
        char * src_filename = argv[1];
        char * cmdline = new char[strlen(argv[0]) + strlen(argv[1]) +1 +1]; 
        strcat(cmdline, argv[0]);
        strcat(cmdline," ");
        strcat(cmdline,argv[1]);  
        
        #define NUM_EVENTS 2
        int Events[NUM_EVENTS] = {PAPI_TOT_CYC, PAPI_BR_MSP};
        int cal_size = 1000;
        code_clocker = new CC(Events,NUM_EVENTS,cal_size);
        code_clocker->set_cmd(cmdline);

#endif  
        
  struct stat st;
  stat(filename, &st);
  int filesize = st.st_size;
  size_t bytes = filesize;
 
  bytes += sizeof(SIMD_type);
  
  // allocate byte buffer and pad with trailing zeroes
  unsigned char * byte_buffer = (unsigned char *)simd_new(bytes);  
  
     
  // slurp file
  int chars_read = fread(byte_buffer, sizeof(char), filesize, infile);
  while(chars_read > 0) {
          chars_read = fread(byte_buffer+chars_read, sizeof(char), filesize, infile);
  }
  
  // mask trailing zeroes
  memset(byte_buffer + filesize, 0, sizeof(SIMD_type));  
  
  // allocate bit stream buffer
  int simd_packs = bytes/sizeof(SIMD_type);
  
  #ifdef BRANCH_REDUCTION
    BitBlock * bit_stream_buffer = simd_new(simd_packs + 1); // pad at least an additional general width of bytes 
  #else  
    BitBlock * bit_stream_buffer = simd_new(simd_packs);  
  #endif
    
  if(bit_stream_buffer == NULL) {
      fprintf(stderr, "Error: out of memory.\n");
      exit(-1);   
  }

  cout << "Source Bytes: " << filesize << endl;
  cout << "Allocated Bytes: " << bytes << endl;
  cout << "SIMD Pack Count: " << simd_packs << endl;  
  cout << "SIMD Pack Bytes: " << simd_packs * sizeof(SIMD_type) << endl << endl;  
  
#ifdef PAPI
        code_clocker->start_interval();
#endif          

  for(int i=0;i<RUNS;i++) {

  #ifdef PAPI
        code_clocker->start_interval();
  #endif        
                
  // convert to bit streams
  for(int i=0,j=0;i<simd_packs;i++,j+=SIMD_REGISTER_BIT_WIDTH) {
          bit_stream_buffer[i] = bytepack2bitblock((BytePack *)(&byte_buffer[j]));
  }

  #ifdef BRANCH_REDUCTION
      bit_stream_buffer[simd_packs] = simd<8>::constant<0>(); // initialize the final block
  #endif
  
  #ifdef PAPI
        code_clocker->end_interval(bytes);
  #endif        
  
  }
        
  size_t max_span_count = simd_packs * SIMD_REGISTER_BIT_WIDTH/2;       
  size_t * starts = new size_t[max_span_count];
  size_t * lengths = new size_t[max_span_count];
  size_t span_count = 0;

  size_t bit_stream_buffer_length;
  
//#ifdef PAPI
//      code_clocker->start_interval();
//#endif        
  
#ifdef BRANCH_REDUCTION 
        bit_stream_buffer_length = simd_packs * sizeof(SIMD_type);      
        stream2runs((unsigned char *)bit_stream_buffer, bit_stream_buffer_length, starts, lengths, &span_count);
#else
        bit_stream_buffer_length = simd_packs * SIMD_REGISTER_BIT_WIDTH / REGISTER_BIT_WIDTH;
        stream2runs((size_t *)bit_stream_buffer, bit_stream_buffer_length, starts, lengths, &span_count);       
#endif
                
//#ifdef PAPI
//      code_clocker->end_interval(bytes);
//#endif        
        
//  for(int i =0; i<span_count; i++) {
//      printf("Value (%zu,%zu) = ", starts[i], lengths[i]);
//      print_chars(((char *) byte_buffer) + starts[i], lengths[i]);      
//  }
  
//  for(int i=0;i<RUNS;i++) {
//
//  #ifdef PAPI
//      code_clocker->start_interval();
//  #endif      

  SymbolTable symbol_table;

  int * gids  = new int[span_count];

  int gid = 0;
  for(int i =0; i<span_count; i++) {
    gids[i] = symbol_table.Lookup_or_Insert_Name((char *)(byte_buffer + starts[i]), lengths[i]);
  }

  cout << "STL Symbol Table: " << endl;
  cout << "DOC_IDX_GID[] => ";
  
  for(int i=0;i<span_count;i++) {
           cout << gids[i] << " ";
  }
  cout << endl << endl;
  
  delete [] gids;

//  #ifdef PAPI
//      code_clocker->end_interval(bytes);
//  #endif    
//  
//  }  
        
//  for(int i=0;i<RUNS;i++) {
   

          LSSymbolTable ls_symbol_table;
          for(int i=0; i<span_count; i++) {
            ls_symbol_table.put(byte_buffer + starts[i], lengths[i]);
          }               
          
//        #ifdef PAPI
//                code_clocker->start_interval();
//        #endif    
          
          ls_symbol_table.bind();               
          ls_symbol_table.finalize();   

//        #ifdef PAPI
//                code_clocker->end_interval(bytes);
//        #endif  
          
          cout << "Length Sorted Symbol Table:" << endl;
          ls_symbol_table.display_flattened_symbol_values();
          ls_symbol_table.display_flattened_gids();       
          ls_symbol_table.clear();
//  }
        
  delete [] starts;
  delete [] lengths;
  
  if(byte_buffer != NULL) {
          simd_delete((SIMD_type *)byte_buffer);
  }
  
  if(bit_stream_buffer != NULL) {
          simd_delete(bit_stream_buffer);
  }

  if(infile != NULL) {
          fclose(infile);
  }
  
  if(argc > 3) {
          fclose(outfile);
  }
  
#ifdef PAPI  
        code_clocker->write_xml_file();
        code_clocker->display_system_info();
        code_clocker->display_raw_event_data();
        delete code_clocker;  
#endif 
        
  fprintf(stdout, "Done.\n");
  
  return(0);
}