From 9814bbb22cc4d455a7f6bb141988ecc201b7ac2c Mon Sep 17 00:00:00 2001 From: Daniel Karbach Date: Thu, 12 Sep 2024 17:21:04 +0200 Subject: [PATCH] fix consecutive tokens --- app/TwitchBot/TokenizedMessage.php | 4 ++-- tests/Unit/TwitchBot/TokenizedMessageTest.php | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/TwitchBot/TokenizedMessage.php b/app/TwitchBot/TokenizedMessage.php index e8465fd..e33ec97 100644 --- a/app/TwitchBot/TokenizedMessage.php +++ b/app/TwitchBot/TokenizedMessage.php @@ -15,7 +15,7 @@ class TokenizedMessage { $this->text = mb_substr($text, mb_strlen($tags['reply-parent-display-name']) + 2); } $this->raw = strtolower(preg_replace('/[^\w]/u', '', $this->text)); - $this->tokens = array_values(array_map('trim', array_filter(preg_split('/\b/u', strtolower($this->text))))); + $this->tokens = array_values(array_filter(array_map('trim', array_filter(preg_split('/\b/u', strtolower($this->text)))))); $this->emoteless = $text; if (isset($this->tags['emotes']) && !empty($this->tags['emotes'])) { @@ -34,7 +34,7 @@ class TokenizedMessage { $this->emoteless = trim(preg_replace('/\s+/u', ' ', $this->emoteless)); } $this->emoteless_raw = strtolower(preg_replace('/[^\w]/u', '', $this->emoteless)); - $this->emoteless_tokens = array_values(array_map('trim', array_filter(preg_split('/\b/u', strtolower($this->emoteless))))); + $this->emoteless_tokens = array_values(array_filter(array_map('trim', array_filter(preg_split('/\b/u', strtolower($this->emoteless)))))); } public static function fromIRC(IRCMessage $msg) { diff --git a/tests/Unit/TwitchBot/TokenizedMessageTest.php b/tests/Unit/TwitchBot/TokenizedMessageTest.php index be5da8d..3678dcc 100644 --- a/tests/Unit/TwitchBot/TokenizedMessageTest.php +++ b/tests/Unit/TwitchBot/TokenizedMessageTest.php @@ -96,4 +96,8 @@ class TokenizedMessageTest extends TestCase { $this->assertTrue(TokenizedMessage::fromString('Willkommen auf Starbase 47')->isSpammy()); } + public function test_tokenizer() { + $this->assertTrue(TokenizedMessage::fromString('@HorstieBot wie viele warps?')->hasConsecutiveTokens(['wie', 'viele'])); + $this->assertFalse(TokenizedMessage::fromString('@HorstieBot wie viele warps?')->hasConsecutiveTokens(['wo', 'wurst'])); + } } -- 2.39.2