From: Daniel Karbach <daniel.karbach@localhorst.tv>
Date: Thu, 12 Sep 2024 15:21:04 +0000 (+0200)
Subject: fix consecutive tokens
X-Git-Url: https://git.localhorst.tv/?a=commitdiff_plain;h=9814bbb22cc4d455a7f6bb141988ecc201b7ac2c;p=alttp.git

fix consecutive tokens
---

diff --git a/app/TwitchBot/TokenizedMessage.php b/app/TwitchBot/TokenizedMessage.php
index e8465fd..e33ec97 100644
--- a/app/TwitchBot/TokenizedMessage.php
+++ b/app/TwitchBot/TokenizedMessage.php
@@ -15,7 +15,7 @@ class TokenizedMessage {
 			$this->text = mb_substr($text, mb_strlen($tags['reply-parent-display-name']) + 2);
 		}
 		$this->raw = strtolower(preg_replace('/[^\w]/u', '', $this->text));
-		$this->tokens = array_values(array_map('trim', array_filter(preg_split('/\b/u', strtolower($this->text)))));
+		$this->tokens = array_values(array_filter(array_map('trim', array_filter(preg_split('/\b/u', strtolower($this->text))))));
 
 		$this->emoteless = $text;
 		if (isset($this->tags['emotes']) && !empty($this->tags['emotes'])) {
@@ -34,7 +34,7 @@ class TokenizedMessage {
 			$this->emoteless = trim(preg_replace('/\s+/u', ' ', $this->emoteless));
 		}
 		$this->emoteless_raw = strtolower(preg_replace('/[^\w]/u', '', $this->emoteless));
-		$this->emoteless_tokens = array_values(array_map('trim', array_filter(preg_split('/\b/u', strtolower($this->emoteless)))));
+		$this->emoteless_tokens = array_values(array_filter(array_map('trim', array_filter(preg_split('/\b/u', strtolower($this->emoteless))))));
 	}
 
 	public static function fromIRC(IRCMessage $msg) {
diff --git a/tests/Unit/TwitchBot/TokenizedMessageTest.php b/tests/Unit/TwitchBot/TokenizedMessageTest.php
index be5da8d..3678dcc 100644
--- a/tests/Unit/TwitchBot/TokenizedMessageTest.php
+++ b/tests/Unit/TwitchBot/TokenizedMessageTest.php
@@ -96,4 +96,8 @@ class TokenizedMessageTest extends TestCase {
 		$this->assertTrue(TokenizedMessage::fromString('Willkommen auf Starbase 47')->isSpammy());
 	}
 
+	public function test_tokenizer() {
+		$this->assertTrue(TokenizedMessage::fromString('@HorstieBot wie viele warps?')->hasConsecutiveTokens(['wie', 'viele']));
+		$this->assertFalse(TokenizedMessage::fromString('@HorstieBot wie viele warps?')->hasConsecutiveTokens(['wo', 'wurst']));
+	}
 }