separate chatlib database generation

[alttp.git] / app / TwitchBot / TokenizedMessage.php

diff --git a/app/TwitchBot/TokenizedMessage.php b/app/TwitchBot/TokenizedMessage.php
index 668634af6ef45f887d3360997d7fb85e1e1ee0c4..4b7d758221d71aa9a9a3d9cbfdd9cf3cde77cb73 100644 (file)
--- a/app/TwitchBot/TokenizedMessage.php
+++ b/app/TwitchBot/TokenizedMessage.php
@@ -12,7 +12,7 @@ class TokenizedMessage {
                $this->text = trim($text);
                $this->tags = $tags;
                $this->raw = strtolower(preg_replace('/[^\w]/u', '', $this->text));
-               $this->tokens = array_values(array_map('trim', array_filter(preg_split('/\b/', strtolower($this->text)))));
+               $this->tokens = array_values(array_map('trim', array_filter(preg_split('/\b/u', strtolower($this->text)))));
 
                $this->emoteless = $this->text;
                if (isset($this->tags['emotes']) && !empty($this->tags['emotes'])) {
@@ -22,16 +22,16 @@ class TokenizedMessage {
                                $positions = explode(',', $set[1]);
                                foreach ($positions as $position) {
                                        $coords = explode('-', $position);
-                                       $this->emotes[] = preg_replace('/\d+$/', '', strtolower(substr($this->text, $coords[0], $coords[1] - $coords[0] + 1)));
-                                       for ($i = intval($coords[0]); $i <= intval($coords[1]); ++$i) {
-                                               $this->emoteless[$i] = ' ';
-                                       }
+                                       $emote_text = mb_substr($this->text, $coords[0], $coords[1] - $coords[0] + 1);
+                                       $this->original_emotes[] = $emote_text;
+                                       $this->emotes[] = preg_replace('/\d+$/', '', strtolower($emote_text));
+                                       $this->emoteless = mb_substr($this->emoteless, 0, $coords[0]).str_repeat(' ', $coords[1] - $coords[0] + 1).mb_substr($this->emoteless, $coords[1] + 1);
                                }
                        }
-                       $this->emoteless = trim(preg_replace('/\s+/', ' ', $this->emoteless));
+                       $this->emoteless = trim(preg_replace('/\s+/u', ' ', $this->emoteless));
                }
-               $this->emoteless_raw = strtolower(preg_replace('/[^\w]/', '', $this->emoteless));
-               $this->emoteless_tokens = array_values(array_map('trim', array_filter(preg_split('/\b/', strtolower($this->emoteless)))));
+               $this->emoteless_raw = strtolower(preg_replace('/[^\w]/u', '', $this->emoteless));
+               $this->emoteless_tokens = array_values(array_map('trim', array_filter(preg_split('/\b/u', strtolower($this->emoteless)))));
        }
 
        public static function fromIRC(IRCMessage $msg) {
@@ -79,10 +79,18 @@ class TokenizedMessage {
                return !empty($this->tokens) && $this->tokens[count($this->tokens) - 1] == $text;
        }
 
+       public function getEmotes() {
+               return $this->emotes;
+       }
+
        public function getNumericValue() {
                return intval($this->text);
        }
 
+       public function getOriginalEmotes() {
+               return $this->original_emotes;
+       }
+
        public function hasConsecutiveTokens($tokens) {
                for ($i = 0; $i < count($this->tokens) - count($tokens) + 1; ++$i) {
                        for ($j = 0; $j < count($tokens); ++$j) {
@@ -346,6 +354,7 @@ class TokenizedMessage {
        private $tokens;
 
        private $emotes = [];
+       private $original_emotes = [];
        private $emoteless = '';
        private $emoteless_raw = '';
        private $emoteless_tokens = [];