Update

Author: jackzhxng

include/pytorch/tokenizers/pcre2_regex.h

@@ -5,12 +5,11 @@
 
 // Define PCRE2 code unit width before including pcre2.h
 #define PCRE2_CODE_UNIT_WIDTH 8
-
-// Third Party
 #include <pcre2.h>
 
-// Local
-#include "regex.h"
+#include <pytorch/tokenizers/regex.h>
+
+namespace tokenizers {
 
 /**
  * @brief PCRE2-based implementation of IRegex.
@@ -50,5 +49,8 @@ class Pcre2Regex : public IRegex {
   pcre2_match_data* match_data_;
   bool is_valid_;
 
-  friend std::unique_ptr<IRegex> createRegex(const std::string& pattern);
-};
+  friend Result<std::unique_ptr<IRegex>> createRegex(
+      const std::string& pattern);
+};
+
+} // namespace tokenizers

src/pcre2_regex.cpp

@@ -1,8 +1,10 @@
-#include "pytorch/tokenizers/pcre2_regex.h"
-
 #include <iostream>
 #include <vector>
 
+#include <pytorch/tokenizers/pcre2_regex.h>
+
+namespace tokenizers {
+
 Pcre2Regex::Pcre2Regex(const std::string& pattern)
     : regex_(nullptr), match_data_(nullptr), is_valid_(false) {
   int error_code;
@@ -108,4 +110,6 @@ bool Pcre2Regex::ok() const {
 
 const pcre2_code* Pcre2Regex::rawRegex() const {
   return regex_;
-}
+}
+
+} // namespace tokenizers

src/regex.cpp

@@ -1,5 +1,6 @@
-#include <pytorch/tokenizers/regex.h>
+#include <pytorch/tokenizers/pcre2_regex.h>
 #include <pytorch/tokenizers/re2_regex.h>
+#include <pytorch/tokenizers/regex.h>
 #include <pytorch/tokenizers/std_regex.h>
 
 #include <re2/re2.h>

test/test_regex.cpp

@@ -1,12 +1,14 @@
 #include <gtest/gtest.h>
 
-#include "pytorch/tokenizers/regex.h"
-#include "pytorch/tokenizers/re2_regex.h"
 #include "pytorch/tokenizers/pcre2_regex.h"
+#include "pytorch/tokenizers/re2_regex.h"
+#include "pytorch/tokenizers/regex.h"
+
+using namespace tokenizers;
 
 // Test basic functionality
 TEST(RegexTest, BasicMatching) {
-  auto regex = createRegex("\\w+");
+  auto regex = TK_UNWRAP_THROW(createRegex("\\w+"));
   ASSERT_TRUE(regex->ok());
 
   std::string text = "Hello world";
@@ -24,9 +26,9 @@ TEST(RegexTest, Pcre2Specific) {
   const std::string pattern = "(?<=@)\\w+";
   Re2Regex re2_regex(pattern);
   ASSERT_FALSE(re2_regex.ok());
-  
+
   // Now verify that the factory function fallsback on a PCRE2 regex
-  auto regex = createRegex(pattern);
+  auto regex = TK_UNWRAP_THROW(createRegex(pattern));
   ASSERT_TRUE(regex->ok());
 
   std::string text = "[email protected]";
@@ -40,20 +42,21 @@ TEST(RegexTest, Pcre2Specific) {
 // This specific pattern is from the Qwen2.5 1.5B pretokenizer.
 // https://huggingface.co/Qwen/Qwen2.5-1.5B/raw/main/tokenizer.json
 TEST(RegexTest, ComplexPatternWithNegativeLookahead) {
-  const std::string complex_pattern = "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+";
-  
+  const std::string complex_pattern =
+      "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+";
+
   // First verify that RE2 cannot handle this pattern
   Re2Regex re2_regex(complex_pattern);
   ASSERT_FALSE(re2_regex.ok());
-  
+
   // Now verify that the factory function fallsback on a PCRE2 regex
-  auto regex = createRegex(complex_pattern);
+  auto regex = TK_UNWRAP_THROW(createRegex(complex_pattern));
   ASSERT_TRUE(regex->ok());
-  
+
   // Test the pattern with some sample text
   std::string text = "Hello's world\n  test";
   auto matches = regex->findAll(text);
-  
+
   // We expect to match:
   // 1. "Hello" (word)
   // 2. "'s" (contraction)
@@ -62,22 +65,22 @@ TEST(RegexTest, ComplexPatternWithNegativeLookahead) {
   // 5. " " (whitespace)
   // 6. " test" (word with leading space)
   ASSERT_EQ(matches.size(), 6);
-  
+
   EXPECT_EQ(matches[0].text, "Hello");
   EXPECT_EQ(matches[0].position, 0);
-  
+
   EXPECT_EQ(matches[1].text, "'s");
   EXPECT_EQ(matches[1].position, 5);
-  
+
   EXPECT_EQ(matches[2].text, " world");
   EXPECT_EQ(matches[2].position, 7);
-  
+
   EXPECT_EQ(matches[3].text, "\n");
   EXPECT_EQ(matches[3].position, 13);
-  
+
   EXPECT_EQ(matches[4].text, " ");
   EXPECT_EQ(matches[4].position, 14);
-  
+
   EXPECT_EQ(matches[5].text, " test");
   EXPECT_EQ(matches[5].position, 15);
-}
+}