fix consecutive tokens

diff --git a/app/TwitchBot/TokenizedMessage.php b/app/TwitchBot/TokenizedMessage.php
index e8465fd0bcdf4b0295486c03cefa0f0f81d9dcf5..e33ec9772aef2951ae21d13a03f5fcd1f1752ce4 100644 (file)
--- a/app/TwitchBot/TokenizedMessage.php
+++ b/app/TwitchBot/TokenizedMessage.php
@@ -15,7 +15,7 @@ class TokenizedMessage {
                        $this->text = mb_substr($text, mb_strlen($tags['reply-parent-display-name']) + 2);
                }
                $this->raw = strtolower(preg_replace('/[^\w]/u', '', $this->text));
-               $this->tokens = array_values(array_map('trim', array_filter(preg_split('/\b/u', strtolower($this->text)))));
+               $this->tokens = array_values(array_filter(array_map('trim', array_filter(preg_split('/\b/u', strtolower($this->text))))));
 
                $this->emoteless = $text;
                if (isset($this->tags['emotes']) && !empty($this->tags['emotes'])) {
@@ -34,7 +34,7 @@ class TokenizedMessage {
                        $this->emoteless = trim(preg_replace('/\s+/u', ' ', $this->emoteless));
                }
                $this->emoteless_raw = strtolower(preg_replace('/[^\w]/u', '', $this->emoteless));
-               $this->emoteless_tokens = array_values(array_map('trim', array_filter(preg_split('/\b/u', strtolower($this->emoteless)))));
+               $this->emoteless_tokens = array_values(array_filter(array_map('trim', array_filter(preg_split('/\b/u', strtolower($this->emoteless))))));
        }
 
        public static function fromIRC(IRCMessage $msg) {

diff --git a/tests/Unit/TwitchBot/TokenizedMessageTest.php b/tests/Unit/TwitchBot/TokenizedMessageTest.php
index be5da8d97a6bc7dded646a209384b3c7a9e2c825..3678dccc1fa6f8e20d284e09ba4904406818a5ca 100644 (file)
--- a/tests/Unit/TwitchBot/TokenizedMessageTest.php
+++ b/tests/Unit/TwitchBot/TokenizedMessageTest.php
@@ -96,4 +96,8 @@ class TokenizedMessageTest extends TestCase {
                $this->assertTrue(TokenizedMessage::fromString('Willkommen auf Starbase 47')->isSpammy());
        }
 
+       public function test_tokenizer() {
+               $this->assertTrue(TokenizedMessage::fromString('@HorstieBot wie viele warps?')->hasConsecutiveTokens(['wie', 'viele']));
+               $this->assertFalse(TokenizedMessage::fromString('@HorstieBot wie viele warps?')->hasConsecutiveTokens(['wo', 'wurst']));
+       }
 }