Changeset 8


Ignore:
Timestamp:
Dec 22, 2007, 7:27:46 PM (11 years ago)
Author:
cameron
Message:

Sentinels

Location:
trunk/src
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • trunk/src/bitlex.c

    r7 r8  
    114114#endif
    115115}
     116#if BYTE_ORDER == BIG_ENDIAN
     117#define sisd_sfl(blk, n) sisd_srl(blk, n)
     118#endif
     119#if BYTE_ORDER == LITTLE_ENDIAN
     120#define sisd_sfl(blk, n) sisd_sll(blk, n)
     121#endif
     122
     123
     124void Lexer::EstablishSentinels(int code_units) {
     125  if (code_units < BUFFER_BLOCKS * BLOCKSIZE + LOOKAHEAD_POSITIONS) {
     126    ((unsigned char *) parsing_engine_data->x8data)[code_units] = '\0';
     127    if (code_units < BUFFER_BLOCKS * BLOCKSIZE) {
     128      BitBlock bitstream_sentinel =
     129        sisd_sfl(simd_const_1(1), sisd_from_int(code_units % BLOCKSIZE));
     130      int lastblk = code_units/BLOCKSIZE;
     131      for (int j = MarkupStart; j < LexicalItemCount; j++) {
     132        parsing_engine_data->item_stream[j][lastblk] =
     133          simd_or(parsing_engine_data->item_stream[j][lastblk], bitstream_sentinel);
     134      }
     135    }
     136  }
     137}
  • trunk/src/bitlex.h

    r7 r8  
    7979   ParallelStreamSet *parsing_engine_data;
    8080   void ComputeLexicalItemStreams(int newblocks);
     81   void EstablishSentinels(int code_units);
    8182};
    8283
  • trunk/src/charsets/ext_ascii_16.c

    r6 r8  
    5959  }
    6060  ComputeLexicalItemStreams(new_blocks);
     61  EstablishSentinels(avail_bytes/X16_CODE_UNIT_BYTES);
    6162  return avail_bytes/X16_CODE_UNIT_BYTES;
    6263}
     
    9495  }
    9596  ComputeLexicalItemStreams(new_blocks);
     97  EstablishSentinels(avail_bytes/X16_CODE_UNIT_BYTES);
    9698  return avail_bytes/X16_CODE_UNIT_BYTES;
    9799}
  • trunk/src/charsets/ext_ascii_8.c

    r7 r8  
    1 /*  ext_ascii_8.c - Lexer object for 8-bit ASCII-based character sets.
    2     Copyright (c) 2007, Robert D. Cameron.
    3     Licensed to the public under the Open Software License 3.0.
    4     Licensed to International Characters, Inc., under the Academic
    5     Free License 3.0.
    6 
    7 */
    81#include "ext_ascii_8.h"
    92
     
    1811
    1912int Ext_ASCII_8_Lexer::AdvanceBuffer(int newpos) {
    20   int new_blocks, new_packs;
    2113  unsigned char * buf_ptr = xml_buf->GetBytePtr(newpos * CODE_UNIT_BYTES);
    22   BytePack * src_data = (BytePack *) buf_ptr;
    2314  int avail_bytes = xml_buf->PrepareBytes(BYTE_STREAM_SIZE);
     15  int new_blocks = min(BUFFER_BLOCKS, (avail_bytes + BLOCKSIZE - 1)/BLOCKSIZE);
     16  int new_packs = new_blocks * 8 + 1;
    2417#ifdef BUFFER_PROFILING
    2518  start_BOM_interval(bitstream_timer);
    2619#endif
    27   if (avail_bytes >= BYTE_STREAM_SIZE) {
    28     new_blocks = BUFFER_BLOCKS;
    29     new_packs = new_blocks * 8 + 1;
    30     for (int pk = 0; pk < new_packs; pk++) {
    31       parsing_engine_data->x8data[pk] = sisd_load_unaligned(&src_data[pk]);
    32     }
    33   }
    34   else {
    35     new_blocks = min(BUFFER_BLOCKS, (avail_bytes + BLOCKSIZE - 1)/BLOCKSIZE);
    36     new_packs = new_blocks * 8 + 1;
    37     for (int pk = 0; pk < new_packs; pk++) {
    38       parsing_engine_data->x8data[pk] = sisd_load_unaligned(&src_data[pk]);
    39     }
    40     ((unsigned char *) parsing_engine_data->x8data)[avail_bytes] = ']';
     20  BytePack * src_data = (BytePack *) buf_ptr;
     21  for (int pk = 0; pk < new_packs; pk++) {
     22    parsing_engine_data->x8data[pk] = sisd_load_unaligned(&src_data[pk]);
    4123  }
    4224  ComputeLexicalItemStreams(new_blocks);
     25  EstablishSentinels(avail_bytes/CODE_UNIT_BYTES);
    4326  return avail_bytes/CODE_UNIT_BYTES;
    4427}
Note: See TracChangeset for help on using the changeset viewer.