Changeset 6165


Ignore:
Timestamp:
Sep 21, 2018, 12:02:43 PM (3 months ago)
Author:
xwa163
Message:
  1. Small optimization for U8NonFinal Stream in UTF-8 LZ4 Grep
  2. Print total CPU cycles in CPU Counter
Location:
icGREP/icgrep-devel/icgrep
Files:
5 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/lz4/grep/lz4_grep_base_generator.cpp

    r6153 r6165  
    124124}
    125125
     126unsigned LZ4GrepBaseGenerator::calculateTwistWidth(unsigned numOfStreams) {
     127    if (numOfStreams <= 2) {
     128        return numOfStreams;
     129    } else if (numOfStreams <= 4) {
     130        return 4;
     131    } else if (numOfStreams <= 8) {
     132        return 8;
     133    } else {
     134        llvm::report_fatal_error("Twist: Unsupported numOfStreams " + std::to_string(numOfStreams));;
     135    }
     136}
     137
    126138std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> LZ4GrepBaseGenerator::multiplexingGrep(
    127139        re::RE *RE,
     
    157169
    158170    if (utf8CC) {
     171
    159172        re::Seq* seq = re::makeSeq();
    160173        re::RE* targetRe = mRE;
    161174
    162175        bool requireNonFinal = re::isRequireNonFinal(mRE);
    163         std::vector<re::CC*> OriginalUnicodeSets = re::collectCCs(seq, &cc::Unicode, std::set<re::Name *>({re::makeZeroWidth("\\b{g}")}));;
    164 
    165176        linefeedCC = re::makeCC(0x0A);
    166 
    167177        seq->push_back(targetRe);
    168178        seq->push_back(std::move(linefeedCC));
    169         if (requireNonFinal) {
    170             seq->push_back(u8FinalRe);
    171         }
     179
    172180
    173181
    174182        std::vector<re::CC*> UnicodeSets = re::collectCCs(seq, &cc::Unicode, std::set<re::Name *>({re::makeZeroWidth("\\b{g}")}));;
    175 
    176183        mpx = make_unique<cc::MultiplexedAlphabet>("mpx", UnicodeSets);
    177         mRE = transformCCs(mpx.get(), targetRe);
    178 
    179 
    180184        std::vector<re::CC *> mpx_basis = mpx->getMultiplexedCCs();
    181185        auto numOfCharacterClasses = mpx_basis.size();
    182         llvm::errs() << "numOfUnicodeSet:" << UnicodeSets.size() << "\n";
    183 
    184         llvm::errs() << "numOfCharacterClasses:" << numOfCharacterClasses << "\n";
     186        bool mpxContainFinal = false;
     187        {
     188            seq->push_back(u8FinalRe);
     189            std::vector<re::CC*> UnicodeSetsWithU8Final = re::collectCCs(seq, &cc::Unicode, std::set<re::Name *>({re::makeZeroWidth("\\b{g}")}));;
     190            auto u8FinalMpx = make_unique<cc::MultiplexedAlphabet>("mpx", UnicodeSetsWithU8Final);
     191            if (calculateTwistWidth(numOfCharacterClasses + 1) > calculateTwistWidth(u8FinalMpx->getMultiplexedCCs().size())) {
     192                mpxContainFinal = true;
     193                UnicodeSets = UnicodeSetsWithU8Final;
     194                mpx = std::move(u8FinalMpx);
     195                mpx_basis = mpx->getMultiplexedCCs();
     196                numOfCharacterClasses = mpx_basis.size();
     197            }
     198
     199        }
     200        mRE = transformCCs(mpx.get(), targetRe);
     201
     202
     203//        llvm::errs() << "numOfUnicodeSet:" << UnicodeSets.size() << "\n";
     204//        llvm::errs() << "numOfCharacterClasses:" << numOfCharacterClasses << "\n";
    185205        StreamSetBuffer * CharClasses = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(numOfCharacterClasses), baseBufferSize, 1);
    186206
     
    195215            u8NoFinalStream = fakeStreams[1];
    196216        } else {
    197 //            StreamSetBuffer* compressedNonFinalStream = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
    198 //            kernel::Kernel * nonFinalK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, u8NonFinalRe, externalStreamNames, std::vector<cc::Alphabet *>(), cc::BitNumbering::BigEndian);
    199 //            mGrepDriver->makeKernelCall(nonFinalK, {compressedBitStream}, {compressedNonFinalStream});
    200 
    201             auto decompressedStreams = this->decompressBitStreams(compressedByteStream, {CharClasses/*, compressedNonFinalStream*/});
    202             uncompressedCharClasses = decompressedStreams[0];
    203 //            u8NoFinalStream = decompressedStreams[1];
    204 
    205             auto fakeStreams = this->generateFakeStreams(idb, uncompressedCharClasses, std::vector<unsigned>{8});
    206             fakeMatchCopiedBits = fakeStreams[0];
    207 
    208             StreamSetBuffer * u8FinalStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), this->getDefaultBufferBlocks(), 1);
    209             ICGrepKernel * u8FinalGrepK = (ICGrepKernel *)mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, transformCCs(mpx.get(), u8FinalRe), externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian);
    210             u8FinalGrepK->setCachable(false);
    211             mGrepDriver->makeKernelCall(u8FinalGrepK, {fakeMatchCopiedBits, uncompressedCharClasses}, {u8FinalStream});
    212 
    213             u8NoFinalStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), this->getDefaultBufferBlocks(), 1);
    214             Kernel* notK = mGrepDriver->addKernelInstance<LZ4NotKernel>(idb);
    215             mGrepDriver->makeKernelCall(notK, {u8FinalStream}, {u8NoFinalStream});
     217            if (mpxContainFinal) {
     218                auto decompressedStreams = this->decompressBitStreams(compressedByteStream, {CharClasses/*, compressedNonFinalStream*/});
     219                uncompressedCharClasses = decompressedStreams[0];
     220
     221                auto fakeStreams = this->generateFakeStreams(idb, uncompressedCharClasses, std::vector<unsigned>{8});
     222                fakeMatchCopiedBits = fakeStreams[0];
     223
     224                StreamSetBuffer * u8FinalStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), this->getDefaultBufferBlocks(), 1);
     225
     226                re::RE* mpxU8FinalRe = transformCCs(mpx.get(), u8FinalRe);
     227
     228                ICGrepKernel * u8FinalGrepK = (ICGrepKernel *)mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mpxU8FinalRe, externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian);
     229                u8FinalGrepK->setCachable(false);
     230                mGrepDriver->makeKernelCall(u8FinalGrepK, {fakeMatchCopiedBits, uncompressedCharClasses}, {u8FinalStream});
     231
     232                u8NoFinalStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), this->getDefaultBufferBlocks(), 1);
     233                Kernel* notK = mGrepDriver->addKernelInstance<LZ4NotKernel>(idb);
     234                mGrepDriver->makeKernelCall(notK, {u8FinalStream}, {u8NoFinalStream});
     235            } else {
     236                StreamSetBuffer* compressedNonFinalStream = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
     237                kernel::Kernel * nonFinalK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, u8NonFinalRe, externalStreamNames, std::vector<cc::Alphabet *>(), cc::BitNumbering::BigEndian);
     238                mGrepDriver->makeKernelCall(nonFinalK, {compressedBitStream}, {compressedNonFinalStream});
     239
     240                auto decompressedStreams = this->decompressBitStreams(compressedByteStream, {CharClasses, compressedNonFinalStream});
     241                uncompressedCharClasses = decompressedStreams[0];
     242                u8NoFinalStream = decompressedStreams[1];
     243
     244                auto fakeStreams = this->generateFakeStreams(idb, uncompressedCharClasses, std::vector<unsigned>{8});
     245                fakeMatchCopiedBits = fakeStreams[0];
     246            }
    216247        }
    217248    } else {
     
    232263        std::vector<re::CC *> mpx_basis = mpx->getMultiplexedCCs();
    233264        auto numOfCharacterClasses = mpx_basis.size();
    234         llvm::errs() << "numOfUnicodeSet:" << UnicodeSets.size() << "\n";
    235         llvm::errs() << "numOfCharacterClasses:" << numOfCharacterClasses << "\n";
     265//        llvm::errs() << "numOfUnicodeSet:" << UnicodeSets.size() << "\n";
     266//        llvm::errs() << "numOfCharacterClasses:" << numOfCharacterClasses << "\n";
    236267        StreamSetBuffer * CharClasses = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(numOfCharacterClasses), baseBufferSize, 1);
    237268
     
    338369            const unsigned ByteCClimit = 6;
    339370
    340             if (byteTestsWithinLimit(mRE, ByteCClimit)) {
     371            if (false && byteTestsWithinLimit(mRE, ByteCClimit)) {
    341372                LineBreakStream = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    342373                kernel::Kernel * breakK = mGrepDriver->addKernelInstance<kernel::DirectCharacterClassKernelBuilder>(idb, "breakCC", std::vector<re::CC *>{mBreakCC});
  • icGREP/icgrep-devel/icgrep/lz4/grep/lz4_grep_base_generator.h

    r6152 r6165  
    4949            std::vector<unsigned> numOfStreams
    5050    );
     51    unsigned calculateTwistWidth(unsigned numOfStreams);
    5152
    5253private:
  • icGREP/icgrep-devel/icgrep/lz4/grep/lz4_grep_bytestream_generator.cpp

    r6150 r6165  
    2929                                                parabix::StreamSetBuffer *compressedBitStream) {
    3030    return this->decompressBitStreams(compressedByteStream, {compressedBitStream})[0];
    31 }
    32 
    33 unsigned LZ4GrepByteStreamGenerator::calculateTwistWidth(unsigned numOfStreams) {
    34     if (numOfStreams <= 2) {
    35         return numOfStreams;
    36     } else if (numOfStreams <= 4) {
    37         return 4;
    38     } else if (numOfStreams <= 8) {
    39         return 8;
    40     } else {
    41         llvm::report_fatal_error("Twist: Unsupported numOfStreams " + std::to_string(numOfStreams));;
    42     }
    4331}
    4432
  • icGREP/icgrep-devel/icgrep/lz4/grep/lz4_grep_bytestream_generator.h

    r6150 r6165  
    1313
    1414private:
    15     unsigned calculateTwistWidth(unsigned numOfStreams);
     15
    1616    parabix::StreamSetBuffer* twist(const std::unique_ptr<kernel::KernelBuilder> &b,
    1717                                    std::vector<parabix::StreamSetBuffer*> inputStreams,
  • icGREP/icgrep-devel/icgrep/toolchain/pipeline.cpp

    r6047 r6165  
    320320            b->CreateCall(b->GetDprintf(), {b->getInt32(2), stringPtr, fItems, fCycles, b->CreateFDiv(fCycles, fItems), percentage});
    321321        }
     322        const auto formatString = "Total CPU Cycles: %7.2e\n ";
     323        Value * stringPtr = b->CreatePointerCast(b->GetString(formatString), b->getInt8PtrTy());
     324        b->CreateCall(b->GetDprintf(), {b->getInt32(2), stringPtr, fTotalCycle});
    322325    }
    323326   
     
    401404            b->CreateCall(b->GetDprintf(), {b->getInt32(2), stringPtr, fItems, fCycles, b->CreateFDiv(fCycles, fItems), percentage});
    402405        }
     406
     407        const auto formatString = "Total CPU Cycles: %7.2e\n ";
     408        Value * stringPtr = b->CreatePointerCast(b->GetString(formatString), b->getInt8PtrTy());
     409        b->CreateCall(b->GetDprintf(), {b->getInt32(2), stringPtr, fTotalCycle});
    403410    }
    404411
Note: See TracChangeset for help on using the changeset viewer.