Changeset 6033


Ignore:
Timestamp:
May 13, 2018, 10:53:07 AM (3 months ago)
Author:
xwa163
Message:

Add bitstream deposit version of character_deposit pipeline

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/character_deposit.cpp

    r6008 r6033  
    99#include <llvm/IR/Function.h>
    1010#include <llvm/Linker/Linker.h>
    11 #include <llvm/Support/CommandLine.h>
    1211#include <llvm/Support/PrettyStackTrace.h>
    1312#include <llvm/Support/Signals.h>
     
    3130#include <toolchain/cpudriver.h>
    3231#include <iostream>
    33 #include <fstream>
    3432#include <kernels/deletion.h>
    3533#include <kernels/pdep_kernel.h>
     34#include <kernels/bitstream_pdep_kernel.h>
    3635
    3736namespace re { class CC; }
     
    4140using namespace kernel;
    4241
    43 static cl::OptionCategory lz4dFlags("Command Flags", "deletion options");
    44 //static cl::opt<char> characterToBeExtract(cl::Positional, cl::desc("<character to be extracted>"), cl::Required, cl::cat(lz4dFlags));
    45 static cl::opt<char> characterToBeDeposit(cl::Positional, cl::desc("<character to be depositted>"), cl::Required, cl::cat(lz4dFlags));
    46 static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(lz4dFlags));
    47 static cl::opt<std::string> outputFile(cl::Positional, cl::desc("<output file>"), cl::Required, cl::cat(lz4dFlags));
     42static cl::OptionCategory characterDepositFlags("Command Flags", "deletion options");
     43static cl::opt<char> characterToBeDeposit(cl::Positional, cl::desc("<character to be depositted>"), cl::Required, cl::cat(characterDepositFlags));
     44static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(characterDepositFlags));
     45static cl::opt<std::string> outputFile(cl::Positional, cl::desc("<output file>"), cl::Required, cl::cat(characterDepositFlags));
     46static cl::opt<bool> useSwizzledDeposit("swizzle-deposit", cl::desc("Use swizzle deletion"), cl::init(false), cl::cat(characterDepositFlags));
    4847
    4948
     
    5251/*
    5352 * Usage:
    54  *   character_deletion <character to be deleted> <input file name>
    55  * It will delete the character from the input file and then print the output to stdout
     53 *     character_deposit <character to be deposit> <input file name> <output file name>
     54 *
     55 * Assume we have a input file with 6 character:
     56 *     qbwbeb
     57 * Then, the pipeline will first deleted all of character 'b' from input:
     58 *     qbwbeb => qwe
     59 * And then it will deposit the deleted result (qwe) to the original position of b:
     60 *     qwe => \0 q \0 w \0 e (spaces are not included)
     61 * and store the result to output file
    5662 * */
     63
     64StreamSetBuffer * loadBasisBits(ParabixDriver & pxDriver, Value* inputStream, Value* fileSize, int bufferBlocks) {
     65    auto & iBuilder = pxDriver.getBuilder();
     66
     67    StreamSetBuffer * ByteStream = pxDriver.addBuffer<SourceBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
     68    StreamSetBuffer * BasisBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), bufferBlocks);
     69
     70    kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
     71    sourceK->setInitialArguments({inputStream, fileSize});
     72    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
     73    Kernel * s2pk = pxDriver.addKernelInstance<S2PKernel>(iBuilder);
     74    pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
     75
     76    return BasisBits;
     77}
     78
     79StreamSetBuffer * generateSwizzledDeposit(ParabixDriver & pxDriver, StreamSetBuffer * BasisBits, int bufferBlocks) {
     80    auto & iBuilder = pxDriver.getBuilder();
     81
     82    StreamSetBuffer * const CharacterMarkerBuffer = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), bufferBlocks);
     83    Kernel * ccK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(characterToBeDeposit)}, 8);
     84    pxDriver.makeKernelCall(ccK, {BasisBits}, {CharacterMarkerBuffer});
     85
     86
     87    StreamSetBuffer * u16Swizzle0 = pxDriver.addBuffer<CircularCopybackBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferBlocks);
     88    StreamSetBuffer * u16Swizzle1 = pxDriver.addBuffer<CircularCopybackBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferBlocks);
     89    Kernel * delK = pxDriver.addKernelInstance<SwizzledDeleteByPEXTkernel>(iBuilder, 8);
     90    pxDriver.makeKernelCall(delK, {CharacterMarkerBuffer, BasisBits}, {u16Swizzle0, u16Swizzle1});
     91
     92    StreamSetBuffer * depositedSwizzle0 = pxDriver.addBuffer<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferBlocks, 1);
     93    Kernel * pdep0K = pxDriver.addKernelInstance<PDEPkernel>(iBuilder, 4, "pdep0");
     94    pxDriver.makeKernelCall(pdep0K, {CharacterMarkerBuffer, u16Swizzle0}, {depositedSwizzle0});
     95
     96
     97    StreamSetBuffer * depositedSwizzle1 = pxDriver.addBuffer<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferBlocks, 1);
     98    Kernel * pdep1K = pxDriver.addKernelInstance<PDEPkernel>(iBuilder, 4, "pdep1");
     99    pxDriver.makeKernelCall(pdep1K, {CharacterMarkerBuffer, u16Swizzle1}, {depositedSwizzle1});
     100
     101    // Produce unswizzled bit streams
     102    StreamSetBuffer * resultbits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferBlocks);
     103    Kernel * unSwizzleK = pxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
     104
     105    pxDriver.makeKernelCall(unSwizzleK, {depositedSwizzle0, depositedSwizzle1}, {resultbits});
     106    return resultbits;
     107}
     108
     109StreamSetBuffer * generateBitStreamDeposit(ParabixDriver & pxDriver, StreamSetBuffer * BasisBits, int bufferBlocks) {
     110    auto & iBuilder = pxDriver.getBuilder();
     111
     112    StreamSetBuffer * const deletionMarker = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), bufferBlocks);
     113    Kernel * ccK1 = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "deletionMarker", std::vector<re::CC *>{re::subtractCC(re::makeByte(0, 255), re::makeCC(characterToBeDeposit))}, 8);
     114    pxDriver.makeKernelCall(ccK1, {BasisBits}, {deletionMarker});
     115
     116    StreamSetBuffer * const depositMarker = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), bufferBlocks);
     117    Kernel * ccK2 = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(characterToBeDeposit)}, 8);
     118    pxDriver.makeKernelCall(ccK2, {BasisBits}, {depositMarker});
     119
     120    // Deletion
     121    StreamSetBuffer * deletedBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferBlocks);
     122    StreamSetBuffer * deletionCounts = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferBlocks);
     123
     124    Kernel * delK = pxDriver.addKernelInstance<PEXTFieldCompressKernel>(iBuilder, 64, 8);
     125    pxDriver.makeKernelCall(delK, {BasisBits, deletionMarker}, {deletedBits, deletionCounts});
     126
     127    StreamSetBuffer * compressedBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferBlocks);
     128    Kernel * streamCompressionK = pxDriver.addKernelInstance<StreamCompressKernel>(iBuilder, 64, 8);
     129    pxDriver.makeKernelCall(streamCompressionK, {deletedBits, deletionCounts}, {compressedBits});
     130
     131    // Deposit
     132    StreamSetBuffer * depositedBits = pxDriver.addBuffer<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferBlocks, 1);
     133    Kernel * pdepK = pxDriver.addKernelInstance<BitStreamPDEPKernel>(iBuilder, 8);
     134    pxDriver.makeKernelCall(pdepK, {depositMarker, compressedBits}, {depositedBits});
     135
     136    return depositedBits;
     137}
    57138
    58139int main(int argc, char *argv[]) {
     
    63144    llvm::PrettyStackTraceProgram X(argc, argv);
    64145    llvm_shutdown_obj shutdown;
    65     codegen::ParseCommandLineOptions(argc, argv, {&lz4dFlags, codegen::codegen_flags()});
     146    codegen::ParseCommandLineOptions(argc, argv, {&characterDepositFlags, codegen::codegen_flags()});
    66147
    67148    std::string fileName = inputFile;
     
    79160    char *fileBuffer = const_cast<char *>(mappedFile.data());
    80161
    81     const auto bufferSize = codegen::ThreadNum * codegen::SegmentSize;
     162    const auto bufferBlocks = codegen::ThreadNum * codegen::SegmentSize;
    82163
    83164    ParabixDriver pxDriver("character_deletion");
     
    100181    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
    101182
    102 
    103     // GeneratePipeline
    104     StreamSetBuffer * ByteStream = pxDriver.addBuffer<SourceBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
    105     StreamSetBuffer * BasisBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), bufferSize);
    106 
    107     kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
    108     sourceK->setInitialArguments({inputStream, fileSize});
    109     pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
    110     Kernel * s2pk = pxDriver.addKernelInstance<S2PKernel>(iBuilder);
    111     pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
    112 
    113     StreamSetBuffer * const CharacterMarkerBuffer = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), bufferSize);
    114     Kernel * ccK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(characterToBeDeposit)}, 8);
    115     pxDriver.makeKernelCall(ccK, {BasisBits}, {CharacterMarkerBuffer});
    116 
    117 
    118     StreamSetBuffer * u16Swizzle0 = pxDriver.addBuffer<CircularCopybackBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferSize);
    119     StreamSetBuffer * u16Swizzle1 = pxDriver.addBuffer<CircularCopybackBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferSize);
    120     Kernel * delK = pxDriver.addKernelInstance<SwizzledDeleteByPEXTkernel>(iBuilder, 8);
    121     pxDriver.makeKernelCall(delK, {CharacterMarkerBuffer, BasisBits}, {u16Swizzle0, u16Swizzle1});
    122 
    123     StreamSetBuffer * depositedSwizzle0 = pxDriver.addBuffer<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferSize, 1);
    124     Kernel * pdep0K = pxDriver.addKernelInstance<PDEPkernel>(iBuilder, 4, "pdep0");
    125     pxDriver.makeKernelCall(pdep0K, {CharacterMarkerBuffer, u16Swizzle0}, {depositedSwizzle0});
    126 
    127 
    128     StreamSetBuffer * depositedSwizzle1 = pxDriver.addBuffer<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferSize, 1);
    129     Kernel * pdep1K = pxDriver.addKernelInstance<PDEPkernel>(iBuilder, 4, "pdep1");
    130     pxDriver.makeKernelCall(pdep1K, {CharacterMarkerBuffer, u16Swizzle1}, {depositedSwizzle1});
    131 
    132     // Produce unswizzled bit streams
    133     StreamSetBuffer * resultbits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferSize);
    134     Kernel * unSwizzleK = pxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
    135 
    136     pxDriver.makeKernelCall(unSwizzleK, {depositedSwizzle0, depositedSwizzle1}, {resultbits});
    137 
    138     StreamSetBuffer * const ResultBytes = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), bufferSize);
     183    StreamSetBuffer * BasisBits = loadBasisBits(pxDriver, inputStream, fileSize, bufferBlocks);
     184    StreamSetBuffer * resultbits = NULL;
     185    if (useSwizzledDeposit) {
     186        resultbits = generateSwizzledDeposit(pxDriver, BasisBits, bufferBlocks);
     187    } else {
     188        resultbits = generateBitStreamDeposit(pxDriver, BasisBits, bufferBlocks);
     189    }
     190
     191    StreamSetBuffer * const ResultBytes = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), bufferBlocks);
    139192    Kernel * p2sK = pxDriver.addKernelInstance<P2SKernel>(iBuilder);
    140193    pxDriver.makeKernelCall(p2sK, {resultbits}, {ResultBytes});
Note: See TracChangeset for help on using the changeset viewer.