Changes to lib/matcher_avx512bw.cpp. All files

lib/matcher_avx512bw.cpp

@@ -61,28 +61,22 @@ void Matcher::simd_init_advance_avx512bw()   {     if (pat_->min_ == 0)       adv_ = &Matcher::simd_advance_chars_avx512bw<2>;-    else if (pat_->min_ < 4)-      adv_ = &Matcher::simd_advance_chars_pma_avx512bw<2>;     else-      adv_ = &Matcher::simd_advance_chars_pmh_avx512bw<2>;+      adv_ = &Matcher::simd_advance_chars_pma_avx512bw<2>;   }   else if (pat_->len_ == 3)   {     if (pat_->min_ == 0)       adv_ = &Matcher::simd_advance_chars_avx512bw<3>;-    else if (pat_->min_ < 4)-      adv_ = &Matcher::simd_advance_chars_pma_avx512bw<3>;     else-      adv_ = &Matcher::simd_advance_chars_pmh_avx512bw<3>;+      adv_ = &Matcher::simd_advance_chars_pma_avx512bw<3>;   }   else if (pat_->bmd_ == 0)   {-    if (pat_->min_ >= 4)-      adv_ = &Matcher::simd_advance_string_pmh_avx512bw;-    else if (pat_->min_ > 0)-      adv_ = &Matcher::simd_advance_string_pma_avx512bw;-    else+    if (pat_->min_ == 0)       adv_ = &Matcher::simd_advance_string_avx512bw;+    else+      adv_ = &Matcher::simd_advance_string_pma_avx512bw;   } } @@ -152,7 +146,7 @@ bool Matcher::simd_advance_chars_pma_avx512bw(size_t loc)   const uint16_t lcp = 0;   const uint16_t lcs = LEN - 1;   const char *chr = pat_->chr_;-  const size_t min = pat_->min_;+  const uint16_t min = pat_->min_;   while (true)   {     const char *s = buf_ + loc + lcp;@@ -171,7 +165,7 @@ bool Matcher::simd_advance_chars_pma_avx512bw(size_t loc)             (LEN == 3 ? s[offset + 1 - lcp] == chr[1] : std::memcmp(s + 1 - lcp + offset, chr + 1, LEN - 2) == 0))         {           size_t k = s - lcp + offset - buf_;-          if (k + LEN + 4 > end_ || pat_->predict_match(&buf_[k + LEN]))+          if (REFLEX_UNLIKELY(k + LEN + Pattern::Const::PM_M > end_) || pat_->predict_match(&buf_[k + LEN]))           {             set_current(k);             return true;@@ -195,73 +189,7 @@ bool Matcher::simd_advance_chars_pma_avx512bw(size_t loc)           (LEN == 3 ? s[1 - lcp] == chr[1] : std::memcmp(s + 1 - lcp, chr + 1, LEN - 1) == 0))       {         size_t k = s - lcp - buf_;-        if (k + LEN + 4 > end_ || pat_->predict_match(&buf_[k + LEN]))-        {-          set_current(k);-          return true;-        }-      }-      ++s;-    }-    loc = s - lcp - buf_;-    set_current_and_peek_more(loc);-    loc = cur_;-    if (loc + LEN + min > end_ && eof_)-      return false;-  }-}--/// Few chars followed by 4 minimal char pattern-template<uint8_t LEN>-bool Matcher::simd_advance_chars_pmh_avx512bw(size_t loc)-{-  const uint16_t lcp = 0;-  const uint16_t lcs = LEN - 1;-  const char *chr = pat_->chr_;-  size_t min = pat_->min_;-  while (true)-  {-    const char *s = buf_ + loc + lcp;-    const char *e = buf_ + end_ + lcp - LEN - min + 1;-    __m512i vlcp = _mm512_set1_epi8(chr[lcp]);-    __m512i vlcs = _mm512_set1_epi8(chr[lcs]);-    while (s <= e - 64)-    {-      __m512i vlcpm = _mm512_loadu_si512(reinterpret_cast<const __m512i*>(s));-      __m512i vlcsm = _mm512_loadu_si512(reinterpret_cast<const __m512i*>(s + lcs - lcp));-      uint64_t mask = _mm512_cmpeq_epi8_mask(vlcp, vlcpm) & _mm512_cmpeq_epi8_mask(vlcs, vlcsm);-      while (REFLEX_UNLIKELY(mask != 0))-      {-        uint32_t offset = ctzl(mask);-        if (LEN == 2 ||-            (LEN == 3 ? s[offset + 1 - lcp] == chr[1] : std::memcmp(s + 1 - lcp + offset, chr + 1, LEN - 2) == 0))-        {-          size_t k = s - lcp + offset - buf_;-          if (k + LEN + min > end_ || pat_->predict_match(&buf_[k + LEN], min))-          {-            set_current(k);-            return true;-          }-        }-        mask &= mask - 1;-      }-      s += 64;-    }-    while (s < e)-    {-      do-        s = static_cast<const char*>(std::memchr(s, chr[lcp], e - s));-      while (s != NULL && s[lcs - lcp] != chr[lcs] && ++s < e);-      if (s == NULL || s >= e)-      {-        s = e;-        break;-      }-      if (LEN == 2 ||-          (LEN == 3 ? s[1 - lcp] == chr[1] : std::memcmp(s + 1 - lcp, chr + 1, LEN - 1) == 0))-      {-        size_t k = s - lcp - buf_;-        if (pat_->predict_match(&buf_[k + LEN], min))+        if (REFLEX_UNLIKELY(k + LEN + Pattern::Const::PM_M > end_) || pat_->predict_match(&buf_[k + LEN]))         {           set_current(k);           return true;@@ -281,7 +209,7 @@ bool Matcher::simd_advance_chars_pmh_avx512bw(size_t loc) bool Matcher::simd_advance_string_avx512bw(size_t loc) {   const char *chr = pat_->chr_;-  const size_t len = pat_->len_;+  const uint16_t len = pat_->len_;   const uint16_t lcp = pat_->lcp_;   const uint16_t lcs = pat_->lcs_;   while (true)@@ -338,72 +266,8 @@ bool Matcher::simd_advance_string_avx512bw(size_t loc) bool Matcher::simd_advance_string_pma_avx512bw(size_t loc) {   const char *chr = pat_->chr_;-  const size_t len = pat_->len_;-  const size_t min = pat_->min_;-  const uint16_t lcp = pat_->lcp_;-  const uint16_t lcs = pat_->lcs_;-  while (true)-  {-    const char *s = buf_ + loc + lcp;-    const char *e = buf_ + end_ + lcp - len - min + 1;-    __m512i vlcp = _mm512_set1_epi8(chr[lcp]);-    __m512i vlcs = _mm512_set1_epi8(chr[lcs]);-    while (s <= e - 64)-    {-      __m512i vlcpm = _mm512_loadu_si512(reinterpret_cast<const __m512i*>(s));-      __m512i vlcsm = _mm512_loadu_si512(reinterpret_cast<const __m512i*>(s + lcs - lcp));-      uint64_t mask = _mm512_cmpeq_epi8_mask(vlcp, vlcpm) & _mm512_cmpeq_epi8_mask(vlcs, vlcsm);-      while (REFLEX_UNLIKELY(mask != 0))-      {-        uint32_t offset = ctzl(mask);-        if (std::memcmp(s - lcp + offset, chr, len) == 0)-        {-          size_t k = s - lcp + offset - buf_;-          if (k + len + 4 > end_ || pat_->predict_match(&buf_[k + len]))-          {-            set_current(k);-            return true;-          }-        }-        mask &= mask - 1;-      }-      s += 64;-    }-    while (s < e)-    {-      do-        s = static_cast<const char*>(std::memchr(s, chr[lcp], e - s));-      while (s != NULL && s[lcs - lcp] != chr[lcs] && ++s < e);-      if (s == NULL || s >= e)-      {-        s = e;-        break;-      }-      if (std::memcmp(s - lcp, chr, len) == 0)-      {-        size_t k = s - lcp - buf_;-        if (k + len + 4 > end_ || pat_->predict_match(&buf_[k + len]))-        {-          set_current(k);-          return true;-        }-      }-      ++s;-    }-    loc = s - lcp - buf_;-    set_current_and_peek_more(loc);-    loc = cur_;-    if (loc + len + min > end_ && eof_)-      return false;-  }-}--/// Implements AVX512BW string search scheme based on http://0x80.pl/articles/simd-friendly-karp-rabin.html-bool Matcher::simd_advance_string_pmh_avx512bw(size_t loc)-{-  const char *chr = pat_->chr_;-  const size_t len = pat_->len_;-  const size_t min = pat_->min_;+  const uint16_t len = pat_->len_;+  const uint16_t min = pat_->min_;   const uint16_t lcp = pat_->lcp_;   const uint16_t lcs = pat_->lcs_;   while (true)@@ -423,7 +287,7 @@ bool Matcher::simd_advance_string_pmh_avx512bw(size_t loc)         if (std::memcmp(s - lcp + offset, chr, len) == 0)         {           size_t k = s - lcp + offset - buf_;-          if (k + len + min > end_ || pat_->predict_match(&buf_[k + len], min))+          if (REFLEX_UNLIKELY(k + len + Pattern::Const::PM_M > end_) || pat_->predict_match(&buf_[k + len]))           {             set_current(k);             return true;@@ -446,7 +310,7 @@ bool Matcher::simd_advance_string_pmh_avx512bw(size_t loc)       if (std::memcmp(s - lcp, chr, len) == 0)       {         size_t k = s - lcp - buf_;-        if (pat_->predict_match(&buf_[k + len], min))+        if (REFLEX_UNLIKELY(k + len + Pattern::Const::PM_M > end_) || pat_->predict_match(&buf_[k + len]))         {           set_current(k);           return true;