update mlua-rs, fix warnings, remove dead code

Author: acheong08

Cargo.lock

@@ -9,7 +9,7 @@ name = "tiktoken_core"
 crate-type = ["cdylib"]
 
 [dependencies]
-mlua = { version = "0.9.7", features = ["serialize", "module"] }
+mlua = { version = "0.10.5", features = ["serialize", "module"] }
 # tiktoken dependencies
 fancy-regex = "0.11.0"
 regex = "1.8.3"

Cargo.toml

@@ -1,4 +1,4 @@
-use base64;
+use base64::{prelude::BASE64_STANDARD, Engine as _};
 use fancy_regex::Regex;
 use mlua::prelude::*;
 use rustc_hash::FxHashMap as HashMap;
@@ -8,10 +8,6 @@ use std::io::{BufRead, BufReader};
 use std::sync::{Arc, Mutex};
 use std::thread;
 
-#[cfg(feature = "multithreading")]
-const MAX_NUM_THREADS: usize = 128;
-
-#[cfg(not(feature = "multithreading"))]
 const MAX_NUM_THREADS: usize = 1;
 
 fn _byte_pair_merge<T>(
@@ -224,7 +220,9 @@ fn new(
     for line in reader.lines() {
         let line = line.unwrap();
         let mut parts = line.split_whitespace();
-        let token = base64::decode(parts.next().unwrap().as_bytes()).unwrap();
+        let token = BASE64_STANDARD
+            .decode(parts.next().unwrap().as_bytes())
+            .unwrap();
         let rank = parts.next().unwrap().parse().unwrap();
         encoder.insert(token, rank);
     }
@@ -262,7 +260,8 @@ fn new(
 }
 
 fn encode(state: &State, text: mlua::String) -> LuaResult<(Vec<usize>, usize, usize)> {
-    let encoded_str = String::from_utf8_lossy(text.as_bytes());
+    let text_bytes = text.as_bytes();
+    let encoded_str = String::from_utf8_lossy(&text_bytes);
     let allowed_special = HashSet::new();
     let max_tokens = None;
     Ok(state
@@ -592,13 +591,6 @@ impl CoreBPENative {
         Err(piece.to_owned())
     }
 
-    fn encode_single_piece(&self, piece: &[u8]) -> Vec<usize> {
-        if let Some(token) = self.encoder.get(piece) {
-            return vec![*token];
-        }
-        byte_pair_encode(piece, &self.encoder)
-    }
-
     // ====================
     // Decoding
     // ====================

src/lib.rs

@@ -1,9 +1,9 @@
 package = "tiktoken_core"
-version = "0.2.3-1"
+version = "0.2.4-1"
 
 source = {
 	url = "git+https://github.com/gptlang/lua-tiktoken",
-	tag = "v0.2.3",
+	tag = "v0.2.4",
 }
 
 description = {