Changeset 6032


Ignore:
Timestamp:
May 12, 2018, 11:23:23 PM (5 months ago)
Author:
xwa163
Message:

Add compression pipeline in character_deletion

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/character_deletion.cpp

    r6008 r6032  
    4040using namespace kernel;
    4141
    42 static cl::OptionCategory lz4dFlags("Command Flags", "deletion options");
    43 static cl::opt<char> characterToBeDeleted(cl::Positional, cl::desc("<character to be deletion>"), cl::Required, cl::cat(lz4dFlags));
    44 static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(lz4dFlags));
     42static cl::OptionCategory characterDeletionFlags("Command Flags", "deletion options");
     43static cl::opt<char> characterToBeDeleted(cl::Positional, cl::desc("<character to be deletion>"), cl::Required, cl::cat(characterDeletionFlags));
     44static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(characterDeletionFlags));
     45static cl::opt<bool> useSwizzledDeletion("swizzle-deletion", cl::desc("Use swizzle deletion"), cl::init(false), cl::cat(characterDeletionFlags));
    4546
    4647
     
    5253 * It will delete the character from the input file and then print the output to stdout
    5354 * */
     55
     56StreamSetBuffer * loadBasisBits(ParabixDriver & pxDriver, Value* inputStream, Value* fileSize, int inputBufferBlocks) {
     57    auto & iBuilder = pxDriver.getBuilder();
     58
     59    StreamSetBuffer * ByteStream = pxDriver.addBuffer<SourceBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
     60    StreamSetBuffer * BasisBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), inputBufferBlocks);
     61
     62    kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
     63    sourceK->setInitialArguments({inputStream, fileSize});
     64    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
     65
     66    Kernel * s2pk = pxDriver.addKernelInstance<S2PKernel>(iBuilder, /*aligned = */ true);
     67    pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
     68
     69    return BasisBits;
     70}
     71
     72StreamSetBuffer * generateSwizzledDeletion(ParabixDriver & pxDriver, StreamSetBuffer * BasisBits, int inputBufferBlocks) {
     73    auto & iBuilder = pxDriver.getBuilder();
     74
     75    StreamSetBuffer * const CharacterMarkerBuffer = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), inputBufferBlocks);
     76    Kernel * ccK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "deletionMarker", std::vector<re::CC *>{re::makeCC(characterToBeDeleted)}, 8);
     77
     78    pxDriver.makeKernelCall(ccK, {BasisBits}, {CharacterMarkerBuffer});
     79
     80    StreamSetBuffer * u16Swizzle0 = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(4), inputBufferBlocks, 1);
     81    StreamSetBuffer * u16Swizzle1 = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(4), inputBufferBlocks, 1);
     82    Kernel * delK = pxDriver.addKernelInstance<SwizzledDeleteByPEXTkernel>(iBuilder, 8);
     83    pxDriver.makeKernelCall(delK, {CharacterMarkerBuffer, BasisBits}, {u16Swizzle0, u16Swizzle1});
     84
     85    // Produce unswizzled bit streams
     86    StreamSetBuffer * deletedBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
     87    Kernel * unSwizzleK = pxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
     88
     89    pxDriver.makeKernelCall(unSwizzleK, {u16Swizzle0, u16Swizzle1}, {deletedBits});
     90    return deletedBits;
     91}
     92
     93// TODO: It seems that there are still some bugs in DeleteByPEXTkernel
     94StreamSetBuffer * generateDeletion(ParabixDriver & pxDriver, StreamSetBuffer * BasisBits, int inputBufferBlocks) {
     95    auto & iBuilder = pxDriver.getBuilder();
     96
     97    StreamSetBuffer * const CharacterMarkerBuffer = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), inputBufferBlocks);
     98    Kernel * ccK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "deletionMarker", std::vector<re::CC *>{re::makeCC(characterToBeDeleted)}, 8);
     99    pxDriver.makeKernelCall(ccK, {BasisBits}, {CharacterMarkerBuffer});
     100
     101    StreamSetBuffer * deletedBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
     102    StreamSetBuffer * deletionCounts = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
     103
     104    Kernel * delK = pxDriver.addKernelInstance<DeleteByPEXTkernel>(iBuilder, 64, 8);
     105    pxDriver.makeKernelCall(delK, {BasisBits, CharacterMarkerBuffer}, {deletedBits, deletionCounts});
     106
     107    StreamSetBuffer * compressedBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
     108    Kernel * streamCompressionK = pxDriver.addKernelInstance<StreamCompressKernel>(iBuilder, 64, 8);
     109    pxDriver.makeKernelCall(streamCompressionK, {deletedBits, deletionCounts}, {compressedBits});
     110
     111    return compressedBits;
     112}
     113
     114StreamSetBuffer * generateDeletionByCompression(ParabixDriver & pxDriver, StreamSetBuffer * BasisBits, int inputBufferBlocks) {
     115    auto & iBuilder = pxDriver.getBuilder();
     116
     117    StreamSetBuffer * const CharacterMarkerBuffer = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), inputBufferBlocks);
     118    Kernel * ccK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "deletionMarker", std::vector<re::CC *>{re::subtractCC(re::makeByte(0, 255), re::makeCC(characterToBeDeleted))}, 8);
     119    pxDriver.makeKernelCall(ccK, {BasisBits}, {CharacterMarkerBuffer});
     120
     121    StreamSetBuffer * deletedBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
     122    StreamSetBuffer * deletionCounts = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
     123
     124    Kernel * delK = pxDriver.addKernelInstance<PEXTFieldCompressKernel>(iBuilder, 64, 8);
     125    pxDriver.makeKernelCall(delK, {BasisBits, CharacterMarkerBuffer}, {deletedBits, deletionCounts});
     126
     127    StreamSetBuffer * compressedBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
     128    Kernel * streamCompressionK = pxDriver.addKernelInstance<StreamCompressKernel>(iBuilder, 64, 8);
     129    pxDriver.makeKernelCall(streamCompressionK, {deletedBits, deletionCounts}, {compressedBits});
     130
     131    return compressedBits;
     132}
     133
    54134
    55135int main(int argc, char *argv[]) {
     
    60140    llvm::PrettyStackTraceProgram X(argc, argv);
    61141    llvm_shutdown_obj shutdown;
    62     codegen::ParseCommandLineOptions(argc, argv, {&lz4dFlags, codegen::codegen_flags()});
     142    codegen::ParseCommandLineOptions(argc, argv, {&characterDeletionFlags, codegen::codegen_flags()});
    63143
    64144    const std::string fileName = inputFile;
     
    98178
    99179    // GeneratePipeline
    100     StreamSetBuffer * ByteStream = pxDriver.addBuffer<SourceBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
    101     StreamSetBuffer * BasisBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), inputBufferBlocks);
    102 
    103     kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
    104     sourceK->setInitialArguments({inputStream, fileSize});
    105     pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
    106 
    107     Kernel * s2pk = pxDriver.addKernelInstance<S2PKernel>(iBuilder, /*aligned = */ true);
    108     pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
    109 
    110     StreamSetBuffer * const CharacterMarkerBuffer = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), inputBufferBlocks);
    111     Kernel * ccK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(characterToBeDeleted)}, 8);
    112     pxDriver.makeKernelCall(ccK, {BasisBits}, {CharacterMarkerBuffer});
    113 
    114 
    115     StreamSetBuffer * u16Swizzle0 = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(4), inputBufferBlocks, 1);
    116     StreamSetBuffer * u16Swizzle1 = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(4), inputBufferBlocks, 1);
    117     Kernel * delK = pxDriver.addKernelInstance<SwizzledDeleteByPEXTkernel>(iBuilder, 8);
    118     pxDriver.makeKernelCall(delK, {CharacterMarkerBuffer, BasisBits}, {u16Swizzle0, u16Swizzle1});
    119 
    120     // Produce unswizzled bit streams
    121     StreamSetBuffer * extractedbits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
    122     Kernel * unSwizzleK = pxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
    123 
    124     pxDriver.makeKernelCall(unSwizzleK, {u16Swizzle0, u16Swizzle1}, {extractedbits});
    125 
    126     StreamSetBuffer * const DecompressedByteStream = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), inputBufferBlocks);
     180    StreamSetBuffer * BasisBits = loadBasisBits(pxDriver, inputStream, fileSize, inputBufferBlocks);
     181    StreamSetBuffer * deletedBits = NULL;
     182    if (useSwizzledDeletion) {
     183        deletedBits = generateSwizzledDeletion(pxDriver, BasisBits, inputBufferBlocks);
     184    } else {
     185        deletedBits = generateDeletionByCompression(pxDriver, BasisBits, inputBufferBlocks);
     186    }
     187//    StreamSetBuffer * deletedBits = generateDeletion(pxDriver, BasisBits, inputBufferBlocks);
     188
     189    StreamSetBuffer * const deletedByteStream = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), inputBufferBlocks);
    127190    Kernel * p2sK = pxDriver.addKernelInstance<P2SKernel>(iBuilder);
    128     pxDriver.makeKernelCall(p2sK, {extractedbits}, {DecompressedByteStream});
     191    pxDriver.makeKernelCall(p2sK, {deletedBits}, {deletedByteStream});
    129192
    130193    // --------------------------------------------------------
     
    132195
    133196    Kernel * outK = pxDriver.addKernelInstance<StdOutKernel>(iBuilder, 8);
    134     pxDriver.makeKernelCall(outK, {DecompressedByteStream}, {});
     197    pxDriver.makeKernelCall(outK, {deletedByteStream}, {});
    135198
    136199    pxDriver.generatePipelineIR();
Note: See TracChangeset for help on using the changeset viewer.