diff --git a/src/lexer/token.rs b/src/lexer/token.rs index 0a6bdf74..f22c52d5 100644 --- a/src/lexer/token.rs +++ b/src/lexer/token.rs @@ -647,6 +647,29 @@ mod benches { }); } + #[bench] + fn bench_clean_token_english_hinted_build(b: &mut Bencher) { + b.iter(|| { + TokenLexerBuilder::from( + TokenLexerMode::NormalizeAndCleanup(Some(Lang::Eng)), + "The quick brown fox jumps over the lazy dog!", + ) + }); + } + + #[bench] + fn bench_clean_token_english_hinted_exhaust(b: &mut Bencher) { + b.iter(|| { + let token_cleaner = TokenLexerBuilder::from( + TokenLexerMode::NormalizeAndCleanup(Some(Lang::Eng)), + "The quick brown fox jumps over the lazy dog!", + ) + .unwrap(); + + token_cleaner.map(|value| value.1).collect::>() + }); + } + #[bench] fn bench_clean_token_chinese_build(b: &mut Bencher) { b.iter(|| { @@ -671,21 +694,21 @@ mod benches { } #[bench] - fn bench_clean_token_english_hinted_build(b: &mut Bencher) { + fn bench_clean_token_japanese_build(b: &mut Bencher) { b.iter(|| { TokenLexerBuilder::from( - TokenLexerMode::NormalizeAndCleanup(Some(Lang::Eng)), - "The quick brown fox jumps over the lazy dog!", + TokenLexerMode::NormalizeAndCleanup(None), + "関西国際空港限定トートバッグ", ) }); } #[bench] - fn bench_clean_token_english_hinted_exhaust(b: &mut Bencher) { + fn bench_clean_token_japanese_exhaust(b: &mut Bencher) { b.iter(|| { let token_cleaner = TokenLexerBuilder::from( - TokenLexerMode::NormalizeAndCleanup(Some(Lang::Eng)), - "The quick brown fox jumps over the lazy dog!", + TokenLexerMode::NormalizeAndCleanup(None), + "関西国際空港限定トートバッグ", ) .unwrap();